1 /*
   2  * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
   4  * Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "asm/assembler.hpp"
  28 #include "asm/assembler.inline.hpp"
  29 #include "ci/ciInlineKlass.hpp"
  30 #include "code/compiledIC.hpp"
  31 #include "compiler/disassembler.hpp"
  32 #include "gc/shared/barrierSet.hpp"
  33 #include "gc/shared/barrierSetAssembler.hpp"
  34 #include "gc/shared/cardTable.hpp"
  35 #include "gc/shared/cardTableBarrierSet.hpp"
  36 #include "gc/shared/collectedHeap.hpp"
  37 #include "interpreter/bytecodeHistogram.hpp"
  38 #include "interpreter/interpreter.hpp"
  39 #include "interpreter/interpreterRuntime.hpp"
  40 #include "memory/resourceArea.hpp"
  41 #include "memory/universe.hpp"
  42 #include "oops/accessDecorators.hpp"
  43 #include "oops/compressedKlass.inline.hpp"
  44 #include "oops/compressedOops.inline.hpp"
  45 #include "oops/klass.inline.hpp"
  46 #include "oops/oop.hpp"
  47 #include "oops/resolvedFieldEntry.hpp"
  48 #include "runtime/interfaceSupport.inline.hpp"
  49 #include "runtime/javaThread.hpp"
  50 #include "runtime/jniHandles.inline.hpp"
  51 #include "runtime/sharedRuntime.hpp"
  52 #include "runtime/stubRoutines.hpp"
  53 #include "utilities/globalDefinitions.hpp"
  54 #include "utilities/integerCast.hpp"
  55 #include "utilities/powerOfTwo.hpp"
  56 #ifdef COMPILER2
  57 #include "opto/compile.hpp"
  58 #include "opto/node.hpp"
  59 #include "opto/output.hpp"
  60 #endif
  61 
  62 #ifdef PRODUCT
  63 #define BLOCK_COMMENT(str) /* nothing */
  64 #else
  65 #define BLOCK_COMMENT(str) block_comment(str)
  66 #endif
  67 #define STOP(str) stop(str);
  68 #define BIND(label) bind(label); __ BLOCK_COMMENT(#label ":")
  69 
  70 
  71 
  72 Register MacroAssembler::extract_rs1(address instr) {
  73   assert_cond(instr != nullptr);
  74   return as_Register(Assembler::extract(Assembler::ld_instr(instr), 19, 15));
  75 }
  76 
  77 Register MacroAssembler::extract_rs2(address instr) {
  78   assert_cond(instr != nullptr);
  79   return as_Register(Assembler::extract(Assembler::ld_instr(instr), 24, 20));
  80 }
  81 
  82 Register MacroAssembler::extract_rd(address instr) {
  83   assert_cond(instr != nullptr);
  84   return as_Register(Assembler::extract(Assembler::ld_instr(instr), 11, 7));
  85 }
  86 
  87 uint32_t MacroAssembler::extract_opcode(address instr) {
  88   assert_cond(instr != nullptr);
  89   return Assembler::extract(Assembler::ld_instr(instr), 6, 0);
  90 }
  91 
  92 uint32_t MacroAssembler::extract_funct3(address instr) {
  93   assert_cond(instr != nullptr);
  94   return Assembler::extract(Assembler::ld_instr(instr), 14, 12);
  95 }
  96 
  97 bool MacroAssembler::is_pc_relative_at(address instr) {
  98   // auipc + jalr
  99   // auipc + addi
 100   // auipc + load
 101   // auipc + fload_load
 102   return (is_auipc_at(instr)) &&
 103          (is_addi_at(instr + MacroAssembler::instruction_size) ||
 104           is_jalr_at(instr + MacroAssembler::instruction_size) ||
 105           is_load_at(instr + MacroAssembler::instruction_size) ||
 106           is_float_load_at(instr + MacroAssembler::instruction_size)) &&
 107          check_pc_relative_data_dependency(instr);
 108 }
 109 
 110 // ie:ld(Rd, Label)
 111 bool MacroAssembler::is_load_pc_relative_at(address instr) {
 112   return is_auipc_at(instr) && // auipc
 113          is_ld_at(instr + MacroAssembler::instruction_size) && // ld
 114          check_load_pc_relative_data_dependency(instr);
 115 }
 116 
 117 bool MacroAssembler::is_movptr1_at(address instr) {
 118   return is_lui_at(instr) && // Lui
 119          is_addi_at(instr + MacroAssembler::instruction_size) && // Addi
 120          is_slli_shift_at(instr + MacroAssembler::instruction_size * 2, 11) && // Slli Rd, Rs, 11
 121          is_addi_at(instr + MacroAssembler::instruction_size * 3) && // Addi
 122          is_slli_shift_at(instr + MacroAssembler::instruction_size * 4, 6) && // Slli Rd, Rs, 6
 123          (is_addi_at(instr + MacroAssembler::instruction_size * 5) ||
 124           is_jalr_at(instr + MacroAssembler::instruction_size * 5) ||
 125           is_load_at(instr + MacroAssembler::instruction_size * 5)) && // Addi/Jalr/Load
 126          check_movptr1_data_dependency(instr);
 127 }
 128 
 129 bool MacroAssembler::is_movptr2_at(address instr) {
 130   return is_lui_at(instr) && // lui
 131          is_lui_at(instr + MacroAssembler::instruction_size) && // lui
 132          is_slli_shift_at(instr + MacroAssembler::instruction_size * 2, 18) && // slli Rd, Rs, 18
 133          is_add_at(instr + MacroAssembler::instruction_size * 3) &&
 134          (is_addi_at(instr + MacroAssembler::instruction_size * 4) ||
 135           is_jalr_at(instr + MacroAssembler::instruction_size * 4) ||
 136           is_load_at(instr + MacroAssembler::instruction_size * 4)) && // Addi/Jalr/Load
 137          check_movptr2_data_dependency(instr);
 138 }
 139 
 140 bool MacroAssembler::is_li16u_at(address instr) {
 141   return is_lui_at(instr) && // lui
 142          is_srli_at(instr + MacroAssembler::instruction_size) && // srli
 143          check_li16u_data_dependency(instr);
 144 }
 145 
 146 bool MacroAssembler::is_li32_at(address instr) {
 147   return is_lui_at(instr) && // lui
 148          is_addiw_at(instr + MacroAssembler::instruction_size) && // addiw
 149          check_li32_data_dependency(instr);
 150 }
 151 
 152 bool MacroAssembler::is_lwu_to_zr(address instr) {
 153   assert_cond(instr != nullptr);
 154   return (extract_opcode(instr) == 0b0000011 &&
 155           extract_funct3(instr) == 0b110 &&
 156           extract_rd(instr) == zr);         // zr
 157 }
 158 
 159 uint32_t MacroAssembler::get_membar_kind(address addr) {
 160   assert_cond(addr != nullptr);
 161   assert(is_membar(addr), "no membar found");
 162 
 163   uint32_t insn = Bytes::get_native_u4(addr);
 164 
 165   uint32_t predecessor = Assembler::extract(insn, 27, 24);
 166   uint32_t successor = Assembler::extract(insn, 23, 20);
 167 
 168   return MacroAssembler::pred_succ_to_membar_mask(predecessor, successor);
 169 }
 170 
 171 void MacroAssembler::set_membar_kind(address addr, uint32_t order_kind) {
 172   assert_cond(addr != nullptr);
 173   assert(is_membar(addr), "no membar found");
 174 
 175   uint32_t predecessor = 0;
 176   uint32_t successor = 0;
 177 
 178   MacroAssembler::membar_mask_to_pred_succ(order_kind, predecessor, successor);
 179 
 180   uint32_t insn = Bytes::get_native_u4(addr);
 181   address pInsn = (address) &insn;
 182   Assembler::patch(pInsn, 27, 24, predecessor);
 183   Assembler::patch(pInsn, 23, 20, successor);
 184 
 185   address membar = addr;
 186   Assembler::sd_instr(membar, insn);
 187 }
 188 
 189 static void pass_arg0(MacroAssembler* masm, Register arg) {
 190   if (c_rarg0 != arg) {
 191     masm->mv(c_rarg0, arg);
 192   }
 193 }
 194 
 195 static void pass_arg1(MacroAssembler* masm, Register arg) {
 196   if (c_rarg1 != arg) {
 197     masm->mv(c_rarg1, arg);
 198   }
 199 }
 200 
 201 static void pass_arg2(MacroAssembler* masm, Register arg) {
 202   if (c_rarg2 != arg) {
 203     masm->mv(c_rarg2, arg);
 204   }
 205 }
 206 
 207 static void pass_arg3(MacroAssembler* masm, Register arg) {
 208   if (c_rarg3 != arg) {
 209     masm->mv(c_rarg3, arg);
 210   }
 211 }
 212 
 213 void MacroAssembler::push_cont_fastpath(Register java_thread) {
 214   if (!Continuations::enabled()) return;
 215   Label done;
 216   ld(t0, Address(java_thread, JavaThread::cont_fastpath_offset()));
 217   bleu(sp, t0, done);
 218   sd(sp, Address(java_thread, JavaThread::cont_fastpath_offset()));
 219   bind(done);
 220 }
 221 
 222 void MacroAssembler::pop_cont_fastpath(Register java_thread) {
 223   if (!Continuations::enabled()) return;
 224   Label done;
 225   ld(t0, Address(java_thread, JavaThread::cont_fastpath_offset()));
 226   bltu(sp, t0, done);
 227   sd(zr, Address(java_thread, JavaThread::cont_fastpath_offset()));
 228   bind(done);
 229 }
 230 
 231 int MacroAssembler::align(int modulus, int extra_offset) {
 232   CompressibleScope scope(this);
 233   intptr_t before = offset();
 234   while ((offset() + extra_offset) % modulus != 0) { nop(); }
 235   return (int)(offset() - before);
 236 }
 237 
 238 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
 239   call_VM_base(oop_result, noreg, noreg, nullptr, entry_point, number_of_arguments, check_exceptions);
 240 }
 241 
 242 // Implementation of call_VM versions
 243 
 244 void MacroAssembler::call_VM(Register oop_result,
 245                              address entry_point,
 246                              bool check_exceptions) {
 247   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
 248 }
 249 
 250 void MacroAssembler::call_VM(Register oop_result,
 251                              address entry_point,
 252                              Register arg_1,
 253                              bool check_exceptions) {
 254   pass_arg1(this, arg_1);
 255   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
 256 }
 257 
 258 void MacroAssembler::call_VM(Register oop_result,
 259                              address entry_point,
 260                              Register arg_1,
 261                              Register arg_2,
 262                              bool check_exceptions) {
 263   assert_different_registers(arg_1, c_rarg2);
 264   pass_arg2(this, arg_2);
 265   pass_arg1(this, arg_1);
 266   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
 267 }
 268 
 269 void MacroAssembler::call_VM(Register oop_result,
 270                              address entry_point,
 271                              Register arg_1,
 272                              Register arg_2,
 273                              Register arg_3,
 274                              bool check_exceptions) {
 275   assert_different_registers(arg_1, c_rarg2, c_rarg3);
 276   assert_different_registers(arg_2, c_rarg3);
 277   pass_arg3(this, arg_3);
 278 
 279   pass_arg2(this, arg_2);
 280 
 281   pass_arg1(this, arg_1);
 282   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
 283 }
 284 
 285 void MacroAssembler::call_VM(Register oop_result,
 286                              Register last_java_sp,
 287                              address entry_point,
 288                              int number_of_arguments,
 289                              bool check_exceptions) {
 290   call_VM_base(oop_result, xthread, last_java_sp, nullptr, entry_point, number_of_arguments, check_exceptions);
 291 }
 292 
 293 void MacroAssembler::call_VM(Register oop_result,
 294                              Register last_java_sp,
 295                              address entry_point,
 296                              Register arg_1,
 297                              bool check_exceptions) {
 298   pass_arg1(this, arg_1);
 299   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
 300 }
 301 
 302 void MacroAssembler::call_VM(Register oop_result,
 303                              Register last_java_sp,
 304                              address entry_point,
 305                              Register arg_1,
 306                              Register arg_2,
 307                              bool check_exceptions) {
 308 
 309   assert_different_registers(arg_1, c_rarg2);
 310   pass_arg2(this, arg_2);
 311   pass_arg1(this, arg_1);
 312   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
 313 }
 314 
 315 void MacroAssembler::call_VM(Register oop_result,
 316                              Register last_java_sp,
 317                              address entry_point,
 318                              Register arg_1,
 319                              Register arg_2,
 320                              Register arg_3,
 321                              bool check_exceptions) {
 322   assert_different_registers(arg_1, c_rarg2, c_rarg3);
 323   assert_different_registers(arg_2, c_rarg3);
 324   pass_arg3(this, arg_3);
 325   pass_arg2(this, arg_2);
 326   pass_arg1(this, arg_1);
 327   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
 328 }
 329 
 330 void MacroAssembler::post_call_nop() {
 331   assert(!in_compressible_scope(), "Must be");
 332   assert_alignment(pc());
 333   if (!Continuations::enabled()) {
 334     return;
 335   }
 336   relocate(post_call_nop_Relocation::spec());
 337   InlineSkippedInstructionsCounter skipCounter(this);
 338   nop();
 339   li32(zr, 0);
 340 }
 341 
 342 // these are no-ops overridden by InterpreterMacroAssembler
 343 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {}
 344 void MacroAssembler::check_and_handle_popframe(Register java_thread) {}
 345 
 346 // Calls to C land
 347 //
 348 // When entering C land, the fp, & esp of the last Java frame have to be recorded
 349 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
 350 // has to be reset to 0. This is required to allow proper stack traversal.
 351 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
 352                                          Register last_java_fp,
 353                                          Register last_java_pc) {
 354 
 355   if (last_java_pc->is_valid()) {
 356     sd(last_java_pc, Address(xthread,
 357                              JavaThread::frame_anchor_offset() +
 358                              JavaFrameAnchor::last_Java_pc_offset()));
 359   }
 360 
 361   // determine last_java_sp register
 362   if (!last_java_sp->is_valid()) {
 363     last_java_sp = esp;
 364   }
 365 
 366   // last_java_fp is optional
 367   if (last_java_fp->is_valid()) {
 368     sd(last_java_fp, Address(xthread, JavaThread::last_Java_fp_offset()));
 369   }
 370 
 371   // We must set sp last.
 372   sd(last_java_sp, Address(xthread, JavaThread::last_Java_sp_offset()));
 373 
 374 }
 375 
 376 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
 377                                          Register last_java_fp,
 378                                          address  last_java_pc,
 379                                          Register tmp) {
 380   assert(last_java_pc != nullptr, "must provide a valid PC");
 381 
 382   la(tmp, last_java_pc);
 383   sd(tmp, Address(xthread, JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()));
 384 
 385   set_last_Java_frame(last_java_sp, last_java_fp, noreg);
 386 }
 387 
 388 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
 389                                          Register last_java_fp,
 390                                          Label &L,
 391                                          Register tmp) {
 392   if (L.is_bound()) {
 393     set_last_Java_frame(last_java_sp, last_java_fp, target(L), tmp);
 394   } else {
 395     L.add_patch_at(code(), locator());
 396     IncompressibleScope scope(this); // the label address will be patched back.
 397     set_last_Java_frame(last_java_sp, last_java_fp, pc() /* Patched later */, tmp);
 398   }
 399 }
 400 
 401 void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
 402   // we must set sp to zero to clear frame
 403   sd(zr, Address(xthread, JavaThread::last_Java_sp_offset()));
 404 
 405   // must clear fp, so that compiled frames are not confused; it is
 406   // possible that we need it only for debugging
 407   if (clear_fp) {
 408     sd(zr, Address(xthread, JavaThread::last_Java_fp_offset()));
 409   }
 410 
 411   // Always clear the pc because it could have been set by make_walkable()
 412   sd(zr, Address(xthread, JavaThread::last_Java_pc_offset()));
 413 }
 414 
 415 void MacroAssembler::call_VM_base(Register oop_result,
 416                                   Register java_thread,
 417                                   Register last_java_sp,
 418                                   Label*   return_pc,
 419                                   address  entry_point,
 420                                   int      number_of_arguments,
 421                                   bool     check_exceptions) {
 422    // determine java_thread register
 423   if (!java_thread->is_valid()) {
 424     java_thread = xthread;
 425   }
 426 
 427   // determine last_java_sp register
 428   if (!last_java_sp->is_valid()) {
 429     last_java_sp = esp;
 430   }
 431 
 432   // debugging support
 433   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
 434   assert(java_thread == xthread, "unexpected register");
 435 
 436   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
 437   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
 438 
 439   // push java thread (becomes first argument of C function)
 440   mv(c_rarg0, java_thread);
 441 
 442   // set last Java frame before call
 443   assert(last_java_sp != fp, "can't use fp");
 444 
 445   Label l;
 446   set_last_Java_frame(last_java_sp, fp, return_pc != nullptr ? *return_pc : l, t0);
 447 
 448   // do the call, remove parameters
 449   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments, &l);
 450 
 451   // reset last Java frame
 452   // Only interpreter should have to clear fp
 453   reset_last_Java_frame(true);
 454 
 455    // C++ interp handles this in the interpreter
 456   check_and_handle_popframe(java_thread);
 457   check_and_handle_earlyret(java_thread);
 458 
 459   if (check_exceptions) {
 460     // check for pending exceptions (java_thread is set upon return)
 461     ld(t0, Address(java_thread, in_bytes(Thread::pending_exception_offset())));
 462     Label ok;
 463     beqz(t0, ok);
 464     j(RuntimeAddress(StubRoutines::forward_exception_entry()));
 465     bind(ok);
 466   }
 467 
 468   // get oop result if there is one and reset the value in the thread
 469   if (oop_result->is_valid()) {
 470     get_vm_result_oop(oop_result, java_thread);
 471   }
 472 }
 473 
 474 void MacroAssembler::get_vm_result_oop(Register oop_result, Register java_thread) {
 475   ld(oop_result, Address(java_thread, JavaThread::vm_result_oop_offset()));
 476   sd(zr, Address(java_thread, JavaThread::vm_result_oop_offset()));
 477   verify_oop_msg(oop_result, "broken oop in call_VM_base");
 478 }
 479 
 480 void MacroAssembler::get_vm_result_metadata(Register metadata_result, Register java_thread) {
 481   ld(metadata_result, Address(java_thread, JavaThread::vm_result_metadata_offset()));
 482   sd(zr, Address(java_thread, JavaThread::vm_result_metadata_offset()));
 483 }
 484 
 485 void MacroAssembler::clinit_barrier(Register klass, Register tmp, Label* L_fast_path, Label* L_slow_path) {
 486   assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required");
 487   assert_different_registers(klass, xthread, tmp);
 488 
 489   Label L_fallthrough, L_tmp;
 490   if (L_fast_path == nullptr) {
 491     L_fast_path = &L_fallthrough;
 492   } else if (L_slow_path == nullptr) {
 493     L_slow_path = &L_fallthrough;
 494   }
 495 
 496   // Fast path check: class is fully initialized
 497   lbu(tmp, Address(klass, InstanceKlass::init_state_offset()));
 498   membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
 499   sub(tmp, tmp, InstanceKlass::fully_initialized);
 500   beqz(tmp, *L_fast_path);
 501 
 502   // Fast path check: current thread is initializer thread
 503   ld(tmp, Address(klass, InstanceKlass::init_thread_offset()));
 504 
 505   if (L_slow_path == &L_fallthrough) {
 506     beq(xthread, tmp, *L_fast_path);
 507     bind(*L_slow_path);
 508   } else if (L_fast_path == &L_fallthrough) {
 509     bne(xthread, tmp, *L_slow_path);
 510     bind(*L_fast_path);
 511   } else {
 512     Unimplemented();
 513   }
 514 }
 515 
 516 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
 517   if (!VerifyOops) { return; }
 518 
 519   // Pass register number to verify_oop_subroutine
 520   const char* b = nullptr;
 521   {
 522     ResourceMark rm;
 523     stringStream ss;
 524     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
 525     b = code_string(ss.as_string());
 526   }
 527   BLOCK_COMMENT("verify_oop {");
 528 
 529   push_reg(RegSet::of(ra, t0, t1, c_rarg0), sp);
 530 
 531   mv(c_rarg0, reg); // c_rarg0 : x10
 532   {
 533     // The length of the instruction sequence emitted should not depend
 534     // on the address of the char buffer so that the size of mach nodes for
 535     // scratch emit and normal emit matches.
 536     IncompressibleScope scope(this); // Fixed length
 537     movptr(t0, (address) b);
 538   }
 539 
 540   // Call indirectly to solve generation ordering problem
 541   ld(t1, RuntimeAddress(StubRoutines::verify_oop_subroutine_entry_address()));
 542   jalr(t1);
 543 
 544   pop_reg(RegSet::of(ra, t0, t1, c_rarg0), sp);
 545 
 546   BLOCK_COMMENT("} verify_oop");
 547 }
 548 
 549 // Handle the receiver type profile update given the "recv" klass.
 550 //
 551 // Normally updates the ReceiverData (RD) that starts at "mdp" + "mdp_offset".
 552 // If there are no matching or claimable receiver entries in RD, updates
 553 // the polymorphic counter.
 554 //
 555 // This code expected to run by either the interpreter or JIT-ed code, without
 556 // extra synchronization. For safety, receiver cells are claimed atomically, which
 557 // avoids grossly misrepresenting the profiles under concurrent updates. For speed,
 558 // counter updates are not atomic.
 559 //
 560 void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset) {
 561   assert_different_registers(recv, mdp, t0, t1);
 562 
 563   int base_receiver_offset   = in_bytes(ReceiverTypeData::receiver_offset(0));
 564   int end_receiver_offset    = in_bytes(ReceiverTypeData::receiver_offset(ReceiverTypeData::row_limit()));
 565   int poly_count_offset      = in_bytes(CounterData::count_offset());
 566   int receiver_step          = in_bytes(ReceiverTypeData::receiver_offset(1)) - base_receiver_offset;
 567   int receiver_to_count_step = in_bytes(ReceiverTypeData::receiver_count_offset(0)) - base_receiver_offset;
 568 
 569   // Adjust for MDP offsets. Slots are pointer-sized, so is the global offset.
 570   base_receiver_offset += mdp_offset;
 571   end_receiver_offset  += mdp_offset;
 572   poly_count_offset    += mdp_offset;
 573 
 574 #ifdef ASSERT
 575   // We are about to walk the MDO slots without asking for offsets.
 576   // Check that our math hits all the right spots.
 577   for (uint c = 0; c < ReceiverTypeData::row_limit(); c++) {
 578     int real_recv_offset  = mdp_offset + in_bytes(ReceiverTypeData::receiver_offset(c));
 579     int real_count_offset = mdp_offset + in_bytes(ReceiverTypeData::receiver_count_offset(c));
 580     int offset = base_receiver_offset + receiver_step*c;
 581     int count_offset = offset + receiver_to_count_step;
 582     assert(offset == real_recv_offset, "receiver slot math");
 583     assert(count_offset  == real_count_offset, "receiver count math");
 584   }
 585   int real_poly_count_offset = mdp_offset + in_bytes(CounterData::count_offset());
 586   assert(poly_count_offset == real_poly_count_offset, "poly counter math");
 587 #endif
 588 
 589   // Corner case: no profile table. Increment poly counter and exit.
 590   if (ReceiverTypeData::row_limit() == 0) {
 591     increment(Address(mdp, poly_count_offset), DataLayout::counter_increment);
 592     return;
 593   }
 594 
 595   Register offset = t1;
 596 
 597   Label L_loop_search_receiver, L_loop_search_empty;
 598   Label L_restart, L_found_recv, L_found_empty, L_count_update;
 599 
 600   // The code here recognizes three major cases:
 601   //   A. Fastest: receiver found in the table
 602   //   B. Fast: no receiver in the table, and the table is full
 603   //   C. Slow: no receiver in the table, free slots in the table
 604   //
 605   // The case A performance is most important, as perfectly-behaved code would end up
 606   // there, especially with larger TypeProfileWidth. The case B performance is
 607   // important as well, this is where bulk of code would land for normally megamorphic
 608   // cases. The case C performance is not essential, its job is to deal with installation
 609   // races, we optimize for code density instead. Case C needs to make sure that receiver
 610   // rows are only claimed once. This makes sure we never overwrite a row for another
 611   // receiver and never duplicate the receivers in the list, making profile type-accurate.
 612   //
 613   // It is very tempting to handle these cases in a single loop, and claim the first slot
 614   // without checking the rest of the table. But, profiling code should tolerate free slots
 615   // in the table, as class unloading can clear them. After such cleanup, the receiver
 616   // we need might be _after_ the free slot. Therefore, we need to let at least full scan
 617   // to complete, before trying to install new slots. Splitting the code in several tight
 618   // loops also helpfully optimizes for cases A and B.
 619   //
 620   // This code is effectively:
 621   //
 622   // restart:
 623   //   // Fastest: receiver is already installed
 624   //   for (i = 0; i < receiver_count(); i++) {
 625   //     if (receiver(i) == recv) goto found_recv(i);
 626   //   }
 627   //
 628   //   // Fast: no receiver, but profile is not full
 629   //   for (i = 0; i < receiver_count(); i++) {
 630   //     if (receiver(i) == null) goto found_null(i);
 631   //   }
 632   //
 633   //   // Slow: profile is full, polymorphic case
 634   //   count++;
 635   //   return
 636   //
 637   //   // Slow: try to install receiver
 638   // found_null(i):
 639   //   CAS(&receiver(i), null, recv);
 640   //   goto restart
 641   //
 642   // found_recv(i):
 643   //   *receiver_count(i)++
 644   //
 645 
 646   bind(L_restart);
 647 
 648   // Fastest: receiver is already installed
 649   mv(offset, base_receiver_offset);
 650   bind(L_loop_search_receiver);
 651     add(t0, mdp, offset);
 652     ld(t0, Address(t0));
 653     beq(recv, t0, L_found_recv);
 654   add(offset, offset, receiver_step);
 655   sub(t0, offset, end_receiver_offset);
 656   bnez(t0, L_loop_search_receiver);
 657 
 658   // Fast: no receiver, but profile is not full
 659   mv(offset, base_receiver_offset);
 660   bind(L_loop_search_empty);
 661     add(t0, mdp, offset);
 662     ld(t0, Address(t0));
 663     beqz(t0, L_found_empty);
 664   add(offset, offset, receiver_step);
 665   sub(t0, offset, end_receiver_offset);
 666   bnez(t0, L_loop_search_empty);
 667 
 668   // Slow: Receiver is not found and table is full.
 669   // Increment polymorphic counter instead of receiver slot.
 670   mv(offset, poly_count_offset);
 671   j(L_count_update);
 672 
 673   // Slowest: try to install receiver
 674   bind(L_found_empty);
 675 
 676   // Atomically swing receiver slot: null -> recv.
 677   //
 678   // The update uses CAS, which clobbers t0. Therefore, t1
 679   // is used to hold the destination address. This is safe because the
 680   // offset is no longer needed after the address is computed.
 681   add(t1, mdp, offset);
 682   weak_cmpxchg(/*addr*/ t1, /*expected*/ zr, /*new*/ recv, Assembler::int64,
 683                /*acquire*/ Assembler::relaxed, /*release*/ Assembler::relaxed, /*result*/ t0);
 684 
 685   // CAS success means the slot now has the receiver we want. CAS failure means
 686   // something had claimed the slot concurrently: it can be the same receiver we want,
 687   // or something else. Since this is a slow path, we can optimize for code density,
 688   // and just restart the search from the beginning.
 689   j(L_restart);
 690 
 691   // Found a receiver, convert its slot offset to corresponding count offset.
 692   bind(L_found_recv);
 693   add(offset, offset, receiver_to_count_step);
 694 
 695   // Finally, update the counter
 696   bind(L_count_update);
 697   add(t1, mdp, offset);
 698   increment(Address(t1), DataLayout::counter_increment);
 699 }
 700 
 701 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
 702   if (!VerifyOops) {
 703     return;
 704   }
 705 
 706   const char* b = nullptr;
 707   {
 708     ResourceMark rm;
 709     stringStream ss;
 710     ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
 711     b = code_string(ss.as_string());
 712   }
 713   BLOCK_COMMENT("verify_oop_addr {");
 714 
 715   push_reg(RegSet::of(ra, t0, t1, c_rarg0), sp);
 716 
 717   if (addr.uses(sp)) {
 718     la(x10, addr);
 719     ld(x10, Address(x10, 4 * wordSize));
 720   } else {
 721     ld(x10, addr);
 722   }
 723 
 724   {
 725     // The length of the instruction sequence emitted should not depend
 726     // on the address of the char buffer so that the size of mach nodes for
 727     // scratch emit and normal emit matches.
 728     IncompressibleScope scope(this); // Fixed length
 729     movptr(t0, (address) b);
 730   }
 731 
 732   // Call indirectly to solve generation ordering problem
 733   ld(t1, RuntimeAddress(StubRoutines::verify_oop_subroutine_entry_address()));
 734   jalr(t1);
 735 
 736   pop_reg(RegSet::of(ra, t0, t1, c_rarg0), sp);
 737 
 738   BLOCK_COMMENT("} verify_oop_addr");
 739 }
 740 
 741 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
 742                                          int extra_slot_offset) {
 743   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
 744   int stackElementSize = Interpreter::stackElementSize;
 745   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
 746 #ifdef ASSERT
 747   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
 748   assert(offset1 - offset == stackElementSize, "correct arithmetic");
 749 #endif
 750   if (arg_slot.is_constant()) {
 751     return Address(esp, arg_slot.as_constant() * stackElementSize + offset);
 752   } else {
 753     assert_different_registers(t0, arg_slot.as_register());
 754     shadd(t0, arg_slot.as_register(), esp, t0, exact_log2(stackElementSize));
 755     return Address(t0, offset);
 756   }
 757 }
 758 
 759 #ifndef PRODUCT
 760 extern "C" void findpc(intptr_t x);
 761 #endif
 762 
 763 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[])
 764 {
 765   // In order to get locks to work, we need to fake a in_VM state
 766   if (ShowMessageBoxOnError) {
 767     JavaThread* thread = JavaThread::current();
 768     JavaThreadState saved_state = thread->thread_state();
 769     thread->set_thread_state(_thread_in_vm);
 770 #ifndef PRODUCT
 771     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
 772       ttyLocker ttyl;
 773       BytecodeCounter::print();
 774     }
 775 #endif
 776     if (os::message_box(msg, "Execution stopped, print registers?")) {
 777       ttyLocker ttyl;
 778       tty->print_cr(" pc = 0x%016lx", pc);
 779 #ifndef PRODUCT
 780       tty->cr();
 781       findpc(pc);
 782       tty->cr();
 783 #endif
 784       tty->print_cr(" x0 = 0x%016lx", regs[0]);
 785       tty->print_cr(" x1 = 0x%016lx", regs[1]);
 786       tty->print_cr(" x2 = 0x%016lx", regs[2]);
 787       tty->print_cr(" x3 = 0x%016lx", regs[3]);
 788       tty->print_cr(" x4 = 0x%016lx", regs[4]);
 789       tty->print_cr(" x5 = 0x%016lx", regs[5]);
 790       tty->print_cr(" x6 = 0x%016lx", regs[6]);
 791       tty->print_cr(" x7 = 0x%016lx", regs[7]);
 792       tty->print_cr(" x8 = 0x%016lx", regs[8]);
 793       tty->print_cr(" x9 = 0x%016lx", regs[9]);
 794       tty->print_cr("x10 = 0x%016lx", regs[10]);
 795       tty->print_cr("x11 = 0x%016lx", regs[11]);
 796       tty->print_cr("x12 = 0x%016lx", regs[12]);
 797       tty->print_cr("x13 = 0x%016lx", regs[13]);
 798       tty->print_cr("x14 = 0x%016lx", regs[14]);
 799       tty->print_cr("x15 = 0x%016lx", regs[15]);
 800       tty->print_cr("x16 = 0x%016lx", regs[16]);
 801       tty->print_cr("x17 = 0x%016lx", regs[17]);
 802       tty->print_cr("x18 = 0x%016lx", regs[18]);
 803       tty->print_cr("x19 = 0x%016lx", regs[19]);
 804       tty->print_cr("x20 = 0x%016lx", regs[20]);
 805       tty->print_cr("x21 = 0x%016lx", regs[21]);
 806       tty->print_cr("x22 = 0x%016lx", regs[22]);
 807       tty->print_cr("x23 = 0x%016lx", regs[23]);
 808       tty->print_cr("x24 = 0x%016lx", regs[24]);
 809       tty->print_cr("x25 = 0x%016lx", regs[25]);
 810       tty->print_cr("x26 = 0x%016lx", regs[26]);
 811       tty->print_cr("x27 = 0x%016lx", regs[27]);
 812       tty->print_cr("x28 = 0x%016lx", regs[28]);
 813       tty->print_cr("x30 = 0x%016lx", regs[30]);
 814       tty->print_cr("x31 = 0x%016lx", regs[31]);
 815       BREAKPOINT;
 816     }
 817   }
 818   fatal("DEBUG MESSAGE: %s", msg);
 819 }
 820 
 821 void MacroAssembler::resolve_jobject(Register value, Register tmp1, Register tmp2) {
 822   assert_different_registers(value, tmp1, tmp2);
 823   Label done, tagged, weak_tagged;
 824 
 825   beqz(value, done);           // Use null as-is.
 826   // Test for tag.
 827   andi(tmp1, value, JNIHandles::tag_mask);
 828   bnez(tmp1, tagged);
 829 
 830   // Resolve local handle
 831   access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp1, tmp2);
 832   verify_oop(value);
 833   j(done);
 834 
 835   bind(tagged);
 836   // Test for jweak tag.
 837   STATIC_ASSERT(JNIHandles::TypeTag::weak_global == 0b1);
 838   test_bit(tmp1, value, exact_log2(JNIHandles::TypeTag::weak_global));
 839   bnez(tmp1, weak_tagged);
 840 
 841   // Resolve global handle
 842   access_load_at(T_OBJECT, IN_NATIVE, value,
 843                  Address(value, -JNIHandles::TypeTag::global), tmp1, tmp2);
 844   verify_oop(value);
 845   j(done);
 846 
 847   bind(weak_tagged);
 848   // Resolve jweak.
 849   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, value,
 850                  Address(value, -JNIHandles::TypeTag::weak_global), tmp1, tmp2);
 851   verify_oop(value);
 852 
 853   bind(done);
 854 }
 855 
 856 void MacroAssembler::resolve_global_jobject(Register value, Register tmp1, Register tmp2) {
 857   assert_different_registers(value, tmp1, tmp2);
 858   Label done;
 859 
 860   beqz(value, done);           // Use null as-is.
 861 
 862 #ifdef ASSERT
 863   {
 864     STATIC_ASSERT(JNIHandles::TypeTag::global == 0b10);
 865     Label valid_global_tag;
 866     test_bit(tmp1, value, exact_log2(JNIHandles::TypeTag::global)); // Test for global tag.
 867     bnez(tmp1, valid_global_tag);
 868     stop("non global jobject using resolve_global_jobject");
 869     bind(valid_global_tag);
 870   }
 871 #endif
 872 
 873   // Resolve global handle
 874   access_load_at(T_OBJECT, IN_NATIVE, value,
 875                  Address(value, -JNIHandles::TypeTag::global), tmp1, tmp2);
 876   verify_oop(value);
 877 
 878   bind(done);
 879 }
 880 
 881 void MacroAssembler::stop(const char* msg) {
 882   BLOCK_COMMENT(msg);
 883   illegal_instruction(Assembler::csr::time);
 884   emit_int64((uintptr_t)msg);
 885 }
 886 
 887 void MacroAssembler::unimplemented(const char* what) {
 888   const char* buf = nullptr;
 889   {
 890     ResourceMark rm;
 891     stringStream ss;
 892     ss.print("unimplemented: %s", what);
 893     buf = code_string(ss.as_string());
 894   }
 895   stop(buf);
 896 }
 897 
 898 void MacroAssembler::emit_static_call_stub() {
 899   IncompressibleScope scope(this); // Fixed length: see CompiledDirectCall::to_interp_stub_size().
 900   // CompiledDirectCall::set_to_interpreted knows the
 901   // exact layout of this stub.
 902 
 903   mov_metadata(xmethod, (Metadata*)nullptr);
 904 
 905   // Jump to the entry point of the c2i stub.
 906   int32_t offset = 0;
 907   movptr2(t1, 0, offset, t0); // lui + lui + slli + add
 908   jr(t1, offset);
 909 }
 910 
 911 void MacroAssembler::call_VM_leaf_base(address entry_point,
 912                                        int number_of_arguments,
 913                                        Label *retaddr) {
 914   int32_t offset = 0;
 915   push_reg(RegSet::of(t1, xmethod), sp);   // push << t1 & xmethod >> to sp
 916   movptr(t1, entry_point, offset, t0);
 917   jalr(t1, offset);
 918   if (retaddr != nullptr) {
 919     bind(*retaddr);
 920   }
 921   pop_reg(RegSet::of(t1, xmethod), sp);   // pop << t1 & xmethod >> from sp
 922 }
 923 
 924 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
 925   call_VM_leaf_base(entry_point, number_of_arguments);
 926 }
 927 
 928 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
 929   pass_arg0(this, arg_0);
 930   call_VM_leaf_base(entry_point, 1);
 931 }
 932 
 933 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 934   assert_different_registers(arg_1, c_rarg0);
 935   pass_arg0(this, arg_0);
 936   pass_arg1(this, arg_1);
 937   call_VM_leaf_base(entry_point, 2);
 938 }
 939 
 940 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0,
 941                                   Register arg_1, Register arg_2) {
 942   assert_different_registers(arg_1, c_rarg0);
 943   assert_different_registers(arg_2, c_rarg0, c_rarg1);
 944   pass_arg0(this, arg_0);
 945   pass_arg1(this, arg_1);
 946   pass_arg2(this, arg_2);
 947   call_VM_leaf_base(entry_point, 3);
 948 }
 949 
 950 void MacroAssembler::super_call_VM_leaf(address entry_point) {
 951   MacroAssembler::call_VM_leaf_base(entry_point, 1);
 952 }
 953 
 954 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
 955   pass_arg0(this, arg_0);
 956   MacroAssembler::call_VM_leaf_base(entry_point, 1);
 957 }
 958 
 959 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 960 
 961   assert_different_registers(arg_0, c_rarg1);
 962   pass_arg1(this, arg_1);
 963   pass_arg0(this, arg_0);
 964   MacroAssembler::call_VM_leaf_base(entry_point, 2);
 965 }
 966 
 967 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 968   assert_different_registers(arg_0, c_rarg1, c_rarg2);
 969   assert_different_registers(arg_1, c_rarg2);
 970   pass_arg2(this, arg_2);
 971   pass_arg1(this, arg_1);
 972   pass_arg0(this, arg_0);
 973   MacroAssembler::call_VM_leaf_base(entry_point, 3);
 974 }
 975 
 976 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
 977   assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3);
 978   assert_different_registers(arg_1, c_rarg2, c_rarg3);
 979   assert_different_registers(arg_2, c_rarg3);
 980 
 981   pass_arg3(this, arg_3);
 982   pass_arg2(this, arg_2);
 983   pass_arg1(this, arg_1);
 984   pass_arg0(this, arg_0);
 985   MacroAssembler::call_VM_leaf_base(entry_point, 4);
 986 }
 987 
 988 void MacroAssembler::la(Register Rd, const address addr) {
 989   int32_t offset;
 990   la(Rd, addr, offset);
 991   addi(Rd, Rd, offset);
 992 }
 993 
 994 void MacroAssembler::la(Register Rd, const address addr, int32_t &offset) {
 995   int64_t distance = addr - pc();
 996   assert(is_valid_32bit_offset(distance), "Must be");
 997   auipc(Rd, (int32_t)distance + 0x800);
 998   offset = ((int32_t)distance << 20) >> 20;
 999 }
1000 
1001 // Materialize with auipc + addi sequence if adr is a literal
1002 // address inside code cache. Emit a movptr sequence otherwise.
1003 void MacroAssembler::la(Register Rd, const Address &adr) {
1004   switch (adr.getMode()) {
1005     case Address::literal: {
1006       relocInfo::relocType rtype = adr.rspec().reloc()->type();
1007       if (rtype == relocInfo::none) {
1008         mv(Rd, (intptr_t)(adr.target()));
1009       } else {
1010         if (CodeCache::contains(adr.target())) {
1011           relocate(adr.rspec(), [&] {
1012             la(Rd, adr.target());
1013           });
1014         } else {
1015           relocate(adr.rspec(), [&] {
1016             movptr(Rd, adr.target());
1017           });
1018         }
1019       }
1020       break;
1021     }
1022     case Address::base_plus_offset: {
1023       Address new_adr = legitimize_address(Rd, adr);
1024       if (!(new_adr.base() == Rd && new_adr.offset() == 0)) {
1025         addi(Rd, new_adr.base(), new_adr.offset());
1026       }
1027       break;
1028     }
1029     default:
1030       ShouldNotReachHere();
1031   }
1032 }
1033 
1034 void MacroAssembler::la(Register Rd, Label &label) {
1035   IncompressibleScope scope(this); // the label address may be patched back.
1036   wrap_label(Rd, label, &MacroAssembler::la);
1037 }
1038 
1039 void MacroAssembler::li16u(Register Rd, uint16_t imm) {
1040   lui(Rd, (uint32_t)imm << 12);
1041   srli(Rd, Rd, 12);
1042 }
1043 
1044 void MacroAssembler::li32(Register Rd, int32_t imm) {
1045   // int32_t is in range 0x8000 0000 ~ 0x7fff ffff, and imm[31] is the sign bit
1046   int64_t upper = imm, lower = imm;
1047   lower = (imm << 20) >> 20;
1048   upper -= lower;
1049   upper = (int32_t)upper;
1050   // lui Rd, imm[31:12] + imm[11]
1051   lui(Rd, upper);
1052   addiw(Rd, Rd, lower);
1053 }
1054 
1055 void MacroAssembler::li(Register Rd, int64_t imm) {
1056   // int64_t is in range 0x8000 0000 0000 0000 ~ 0x7fff ffff ffff ffff
1057   // li -> c.li
1058   if (do_compress() && (is_simm6(imm) && Rd != x0)) {
1059     c_li(Rd, imm);
1060     return;
1061   }
1062 
1063   int shift = 12;
1064   int64_t upper = imm, lower = imm;
1065   // Split imm to a lower 12-bit sign-extended part and the remainder,
1066   // because addi will sign-extend the lower imm.
1067   lower = ((int32_t)imm << 20) >> 20;
1068   upper -= lower;
1069 
1070   // Test whether imm is a 32-bit integer.
1071   if (!(((imm) & ~(int64_t)0x7fffffff) == 0 ||
1072         (((imm) & ~(int64_t)0x7fffffff) == ~(int64_t)0x7fffffff))) {
1073     while (((upper >> shift) & 1) == 0) { shift++; }
1074     upper >>= shift;
1075     li(Rd, upper);
1076     slli(Rd, Rd, shift);
1077     if (lower != 0) {
1078       addi(Rd, Rd, lower);
1079     }
1080   } else {
1081     // 32-bit integer
1082     Register hi_Rd = zr;
1083     if (upper != 0) {
1084       lui(Rd, (int32_t)upper);
1085       hi_Rd = Rd;
1086     }
1087     if (lower != 0 || hi_Rd == zr) {
1088       addiw(Rd, hi_Rd, lower);
1089     }
1090   }
1091 }
1092 
1093 void MacroAssembler::j(const address dest, Register temp) {
1094   assert(CodeCache::contains(dest), "Must be");
1095   assert_cond(dest != nullptr);
1096   int64_t distance = dest - pc();
1097 
1098   // We can't patch C, i.e. if Label wasn't bound we need to patch this jump.
1099   IncompressibleScope scope(this);
1100   if (is_simm21(distance) && ((distance % 2) == 0)) {
1101     Assembler::jal(x0, distance);
1102   } else {
1103     assert(temp != noreg && temp != x0, "Expecting a register");
1104     assert(temp != x1 && temp != x5, "temp register must not be x1/x5.");
1105     int32_t offset = 0;
1106     la(temp, dest, offset);
1107     jr(temp, offset);
1108   }
1109 }
1110 
1111 void MacroAssembler::j(const Address &dest, Register temp) {
1112   switch (dest.getMode()) {
1113     case Address::literal: {
1114       if (CodeCache::contains(dest.target())) {
1115         far_jump(dest, temp);
1116       } else {
1117         relocate(dest.rspec(), [&] {
1118           int32_t offset;
1119           movptr(temp, dest.target(), offset);
1120           jr(temp, offset);
1121         });
1122       }
1123       break;
1124     }
1125     case Address::base_plus_offset: {
1126       int32_t offset = ((int32_t)dest.offset() << 20) >> 20;
1127       la(temp, Address(dest.base(), dest.offset() - offset));
1128       jr(temp, offset);
1129       break;
1130     }
1131     default:
1132       ShouldNotReachHere();
1133   }
1134 }
1135 
1136 void MacroAssembler::j(Label &lab, Register temp) {
1137   assert_different_registers(x0, temp);
1138   if (lab.is_bound()) {
1139     MacroAssembler::j(target(lab), temp);
1140   } else {
1141     lab.add_patch_at(code(), locator());
1142     MacroAssembler::j(pc(), temp);
1143   }
1144 }
1145 
1146 void MacroAssembler::jr(Register Rd, int32_t offset) {
1147   assert(Rd != noreg, "expecting a register");
1148   assert(Rd != x1 && Rd != x5, "Rd register must not be x1/x5.");
1149   Assembler::jalr(x0, Rd, offset);
1150 }
1151 
1152 void MacroAssembler::call(const address dest, Register temp) {
1153   assert_cond(dest != nullptr);
1154   assert(temp != noreg, "expecting a register");
1155   assert(temp != x5, "temp register must not be x5.");
1156   int32_t offset = 0;
1157   la(temp, dest, offset);
1158   jalr(temp, offset);
1159 }
1160 
1161 void MacroAssembler::jalr(Register Rs, int32_t offset) {
1162   assert(Rs != noreg, "expecting a register");
1163   assert(Rs != x5, "Rs register must not be x5.");
1164   Assembler::jalr(x1, Rs, offset);
1165 }
1166 
1167 void MacroAssembler::rt_call(address dest, Register tmp) {
1168   assert(tmp != x5, "tmp register must not be x5.");
1169   RuntimeAddress target(dest);
1170   if (CodeCache::contains(dest)) {
1171     far_call(target, tmp);
1172   } else {
1173     relocate(target.rspec(), [&] {
1174       int32_t offset;
1175       movptr(tmp, target.target(), offset);
1176       jalr(tmp, offset);
1177     });
1178   }
1179 }
1180 
1181 void MacroAssembler::wrap_label(Register Rt, Label &L, jal_jalr_insn insn) {
1182   if (L.is_bound()) {
1183     (this->*insn)(Rt, target(L));
1184   } else {
1185     L.add_patch_at(code(), locator());
1186     (this->*insn)(Rt, pc());
1187   }
1188 }
1189 
1190 void MacroAssembler::wrap_label(Register r1, Register r2, Label &L,
1191                                 compare_and_branch_insn insn,
1192                                 compare_and_branch_label_insn neg_insn, bool is_far) {
1193   if (is_far) {
1194     Label done;
1195     (this->*neg_insn)(r1, r2, done, /* is_far */ false);
1196     j(L);
1197     bind(done);
1198   } else {
1199     if (L.is_bound()) {
1200       (this->*insn)(r1, r2, target(L));
1201     } else {
1202       L.add_patch_at(code(), locator());
1203       (this->*insn)(r1, r2, pc());
1204     }
1205   }
1206 }
1207 
1208 #define INSN(NAME, NEG_INSN)                                                              \
1209   void MacroAssembler::NAME(Register Rs1, Register Rs2, Label &L, bool is_far) {          \
1210     wrap_label(Rs1, Rs2, L, &MacroAssembler::NAME, &MacroAssembler::NEG_INSN, is_far);    \
1211   }
1212 
1213   INSN(beq,  bne);
1214   INSN(bne,  beq);
1215   INSN(blt,  bge);
1216   INSN(bge,  blt);
1217   INSN(bltu, bgeu);
1218   INSN(bgeu, bltu);
1219 
1220 #undef INSN
1221 
1222 #define INSN(NAME)                                                                \
1223   void MacroAssembler::NAME##z(Register Rs, const address dest) {                 \
1224     NAME(Rs, zr, dest);                                                           \
1225   }                                                                               \
1226   void MacroAssembler::NAME##z(Register Rs, Label &l, bool is_far) {              \
1227     NAME(Rs, zr, l, is_far);                                                      \
1228   }                                                                               \
1229 
1230   INSN(beq);
1231   INSN(bne);
1232   INSN(blt);
1233   INSN(ble);
1234   INSN(bge);
1235   INSN(bgt);
1236 
1237 #undef INSN
1238 
1239 #define INSN(NAME, NEG_INSN)                                                      \
1240   void MacroAssembler::NAME(Register Rs, Register Rt, const address dest) {       \
1241     NEG_INSN(Rt, Rs, dest);                                                       \
1242   }                                                                               \
1243   void MacroAssembler::NAME(Register Rs, Register Rt, Label &l, bool is_far) {    \
1244     NEG_INSN(Rt, Rs, l, is_far);                                                  \
1245   }
1246 
1247   INSN(bgt,  blt);
1248   INSN(ble,  bge);
1249   INSN(bgtu, bltu);
1250   INSN(bleu, bgeu);
1251 
1252 #undef INSN
1253 
1254 // cmov
1255 void MacroAssembler::cmov_eq(Register cmp1, Register cmp2, Register dst, Register src) {
1256   if (UseZicond) {
1257     xorr(t0, cmp1, cmp2);
1258     czero_eqz(dst, dst, t0);
1259     czero_nez(t0 , src, t0);
1260     orr(dst, dst, t0);
1261     return;
1262   }
1263   Label no_set;
1264   bne(cmp1, cmp2, no_set);
1265   mv(dst, src);
1266   bind(no_set);
1267 }
1268 
1269 void MacroAssembler::cmov_ne(Register cmp1, Register cmp2, Register dst, Register src) {
1270   if (UseZicond) {
1271     xorr(t0, cmp1, cmp2);
1272     czero_nez(dst, dst, t0);
1273     czero_eqz(t0 , src, t0);
1274     orr(dst, dst, t0);
1275     return;
1276   }
1277   Label no_set;
1278   beq(cmp1, cmp2, no_set);
1279   mv(dst, src);
1280   bind(no_set);
1281 }
1282 
1283 void MacroAssembler::cmov_le(Register cmp1, Register cmp2, Register dst, Register src) {
1284   if (UseZicond) {
1285     slt(t0, cmp2, cmp1);
1286     czero_eqz(dst, dst, t0);
1287     czero_nez(t0,  src, t0);
1288     orr(dst, dst, t0);
1289     return;
1290   }
1291   Label no_set;
1292   bgt(cmp1, cmp2, no_set);
1293   mv(dst, src);
1294   bind(no_set);
1295 }
1296 
1297 void MacroAssembler::cmov_leu(Register cmp1, Register cmp2, Register dst, Register src) {
1298   if (UseZicond) {
1299     sltu(t0, cmp2, cmp1);
1300     czero_eqz(dst, dst, t0);
1301     czero_nez(t0,  src, t0);
1302     orr(dst, dst, t0);
1303     return;
1304   }
1305   Label no_set;
1306   bgtu(cmp1, cmp2, no_set);
1307   mv(dst, src);
1308   bind(no_set);
1309 }
1310 
1311 void MacroAssembler::cmov_ge(Register cmp1, Register cmp2, Register dst, Register src) {
1312   if (UseZicond) {
1313     slt(t0, cmp1, cmp2);
1314     czero_eqz(dst, dst, t0);
1315     czero_nez(t0,  src, t0);
1316     orr(dst, dst, t0);
1317     return;
1318   }
1319   Label no_set;
1320   blt(cmp1, cmp2, no_set);
1321   mv(dst, src);
1322   bind(no_set);
1323 }
1324 
1325 void MacroAssembler::cmov_geu(Register cmp1, Register cmp2, Register dst, Register src) {
1326   if (UseZicond) {
1327     sltu(t0, cmp1, cmp2);
1328     czero_eqz(dst, dst, t0);
1329     czero_nez(t0,  src, t0);
1330     orr(dst, dst, t0);
1331     return;
1332   }
1333   Label no_set;
1334   bltu(cmp1, cmp2, no_set);
1335   mv(dst, src);
1336   bind(no_set);
1337 }
1338 
1339 void MacroAssembler::cmov_lt(Register cmp1, Register cmp2, Register dst, Register src) {
1340   if (UseZicond) {
1341     slt(t0, cmp1, cmp2);
1342     czero_nez(dst, dst, t0);
1343     czero_eqz(t0,  src, t0);
1344     orr(dst, dst, t0);
1345     return;
1346   }
1347   Label no_set;
1348   bge(cmp1, cmp2, no_set);
1349   mv(dst, src);
1350   bind(no_set);
1351 }
1352 
1353 void MacroAssembler::cmov_ltu(Register cmp1, Register cmp2, Register dst, Register src) {
1354   if (UseZicond) {
1355     sltu(t0, cmp1, cmp2);
1356     czero_nez(dst, dst, t0);
1357     czero_eqz(t0,  src, t0);
1358     orr(dst, dst, t0);
1359     return;
1360   }
1361   Label no_set;
1362   bgeu(cmp1, cmp2, no_set);
1363   mv(dst, src);
1364   bind(no_set);
1365 }
1366 
1367 void MacroAssembler::cmov_gt(Register cmp1, Register cmp2, Register dst, Register src) {
1368   if (UseZicond) {
1369     slt(t0, cmp2, cmp1);
1370     czero_nez(dst, dst, t0);
1371     czero_eqz(t0,  src, t0);
1372     orr(dst, dst, t0);
1373     return;
1374   }
1375   Label no_set;
1376   ble(cmp1, cmp2, no_set);
1377   mv(dst, src);
1378   bind(no_set);
1379 }
1380 
1381 void MacroAssembler::cmov_gtu(Register cmp1, Register cmp2, Register dst, Register src) {
1382   if (UseZicond) {
1383     sltu(t0, cmp2, cmp1);
1384     czero_nez(dst, dst, t0);
1385     czero_eqz(t0,  src, t0);
1386     orr(dst, dst, t0);
1387     return;
1388   }
1389   Label no_set;
1390   bleu(cmp1, cmp2, no_set);
1391   mv(dst, src);
1392   bind(no_set);
1393 }
1394 
1395 // ----------- cmove float/double -----------
1396 
1397 void MacroAssembler::cmov_fp_eq(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single) {
1398   Label no_set;
1399   bne(cmp1, cmp2, no_set);
1400   if (is_single) {
1401     fmv_s(dst, src);
1402   } else {
1403     fmv_d(dst, src);
1404   }
1405   bind(no_set);
1406 }
1407 
1408 void MacroAssembler::cmov_fp_ne(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single) {
1409   Label no_set;
1410   beq(cmp1, cmp2, no_set);
1411   if (is_single) {
1412     fmv_s(dst, src);
1413   } else {
1414     fmv_d(dst, src);
1415   }
1416   bind(no_set);
1417 }
1418 
1419 void MacroAssembler::cmov_fp_le(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single) {
1420   Label no_set;
1421   bgt(cmp1, cmp2, no_set);
1422   if (is_single) {
1423     fmv_s(dst, src);
1424   } else {
1425     fmv_d(dst, src);
1426   }
1427   bind(no_set);
1428 }
1429 
1430 void MacroAssembler::cmov_fp_leu(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single) {
1431   Label no_set;
1432   bgtu(cmp1, cmp2, no_set);
1433   if (is_single) {
1434     fmv_s(dst, src);
1435   } else {
1436     fmv_d(dst, src);
1437   }
1438   bind(no_set);
1439 }
1440 
1441 void MacroAssembler::cmov_fp_ge(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single) {
1442   Label no_set;
1443   blt(cmp1, cmp2, no_set);
1444   if (is_single) {
1445     fmv_s(dst, src);
1446   } else {
1447     fmv_d(dst, src);
1448   }
1449   bind(no_set);
1450 }
1451 
1452 void MacroAssembler::cmov_fp_geu(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single) {
1453   Label no_set;
1454   bltu(cmp1, cmp2, no_set);
1455   if (is_single) {
1456     fmv_s(dst, src);
1457   } else {
1458     fmv_d(dst, src);
1459   }
1460   bind(no_set);
1461 }
1462 
1463 void MacroAssembler::cmov_fp_lt(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single) {
1464   Label no_set;
1465   bge(cmp1, cmp2, no_set);
1466   if (is_single) {
1467     fmv_s(dst, src);
1468   } else {
1469     fmv_d(dst, src);
1470   }
1471   bind(no_set);
1472 }
1473 
1474 void MacroAssembler::cmov_fp_ltu(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single) {
1475   Label no_set;
1476   bgeu(cmp1, cmp2, no_set);
1477   if (is_single) {
1478     fmv_s(dst, src);
1479   } else {
1480     fmv_d(dst, src);
1481   }
1482   bind(no_set);
1483 }
1484 
1485 void MacroAssembler::cmov_fp_gt(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single) {
1486   Label no_set;
1487   ble(cmp1, cmp2, no_set);
1488   if (is_single) {
1489     fmv_s(dst, src);
1490   } else {
1491     fmv_d(dst, src);
1492   }
1493   bind(no_set);
1494 }
1495 
1496 void MacroAssembler::cmov_fp_gtu(Register cmp1, Register cmp2, FloatRegister dst, FloatRegister src, bool is_single) {
1497   Label no_set;
1498   bleu(cmp1, cmp2, no_set);
1499   if (is_single) {
1500     fmv_s(dst, src);
1501   } else {
1502     fmv_d(dst, src);
1503   }
1504   bind(no_set);
1505 }
1506 
1507 // ----------- cmove, compare float/double -----------
1508 //
1509 // For CmpF/D + CMoveI/L, ordered ones are quite straight and simple,
1510 // so, just list behaviour of unordered ones as follow.
1511 //
1512 // Set dst (CMoveI (Binary cop (CmpF/D op1 op2)) (Binary dst src))
1513 // (If one or both inputs to the compare are NaN, then)
1514 //    1. (op1 lt op2) => true  => CMove: dst = src
1515 //    2. (op1 le op2) => true  => CMove: dst = src
1516 //    3. (op1 gt op2) => false => CMove: dst = dst
1517 //    4. (op1 ge op2) => false => CMove: dst = dst
1518 //    5. (op1 eq op2) => false => CMove: dst = dst
1519 //    6. (op1 ne op2) => true  => CMove: dst = src
1520 
1521 void MacroAssembler::cmov_cmp_fp_eq(FloatRegister cmp1, FloatRegister cmp2, Register dst, Register src, bool is_single) {
1522   if (UseZicond) {
1523     if (is_single) {
1524       feq_s(t0, cmp1, cmp2);
1525     } else {
1526       feq_d(t0, cmp1, cmp2);
1527     }
1528     czero_nez(dst, dst, t0);
1529     czero_eqz(t0 , src, t0);
1530     orr(dst, dst, t0);
1531     return;
1532   }
1533   Label no_set;
1534   if (is_single) {
1535     // jump if cmp1 != cmp2, including the case of NaN
1536     // fallthrough (i.e. move src to dst) if cmp1 == cmp2
1537     float_bne(cmp1, cmp2, no_set);
1538   } else {
1539     double_bne(cmp1, cmp2, no_set);
1540   }
1541   mv(dst, src);
1542   bind(no_set);
1543 }
1544 
1545 void MacroAssembler::cmov_cmp_fp_ne(FloatRegister cmp1, FloatRegister cmp2, Register dst, Register src, bool is_single) {
1546   if (UseZicond) {
1547     if (is_single) {
1548       feq_s(t0, cmp1, cmp2);
1549     } else {
1550       feq_d(t0, cmp1, cmp2);
1551     }
1552     czero_eqz(dst, dst, t0);
1553     czero_nez(t0 , src, t0);
1554     orr(dst, dst, t0);
1555     return;
1556   }
1557   Label no_set;
1558   if (is_single) {
1559     // jump if cmp1 == cmp2
1560     // fallthrough (i.e. move src to dst) if cmp1 != cmp2, including the case of NaN
1561     float_beq(cmp1, cmp2, no_set);
1562   } else {
1563     double_beq(cmp1, cmp2, no_set);
1564   }
1565   mv(dst, src);
1566   bind(no_set);
1567 }
1568 
1569 void MacroAssembler::cmov_cmp_fp_le(FloatRegister cmp1, FloatRegister cmp2, Register dst, Register src, bool is_single) {
1570   if (UseZicond) {
1571     if (is_single) {
1572       flt_s(t0, cmp2, cmp1);
1573     } else {
1574       flt_d(t0, cmp2, cmp1);
1575     }
1576     czero_eqz(dst, dst, t0);
1577     czero_nez(t0 , src, t0);
1578     orr(dst, dst, t0);
1579     return;
1580   }
1581   Label no_set;
1582   if (is_single) {
1583     // jump if cmp1 > cmp2
1584     // fallthrough (i.e. move src to dst) if cmp1 <= cmp2 or either is NaN
1585     float_bgt(cmp1, cmp2, no_set);
1586   } else {
1587     double_bgt(cmp1, cmp2, no_set);
1588   }
1589   mv(dst, src);
1590   bind(no_set);
1591 }
1592 
1593 void MacroAssembler::cmov_cmp_fp_ge(FloatRegister cmp1, FloatRegister cmp2, Register dst, Register src, bool is_single) {
1594   if (UseZicond) {
1595     if (is_single) {
1596       fle_s(t0, cmp2, cmp1);
1597     } else {
1598       fle_d(t0, cmp2, cmp1);
1599     }
1600     czero_nez(dst, dst, t0);
1601     czero_eqz(t0 , src, t0);
1602     orr(dst, dst, t0);
1603     return;
1604   }
1605   Label no_set;
1606   if (is_single) {
1607     // jump if cmp1 < cmp2 or either is NaN
1608     // fallthrough (i.e. move src to dst) if cmp1 >= cmp2
1609     float_blt(cmp1, cmp2, no_set, false, true);
1610   } else {
1611     double_blt(cmp1, cmp2, no_set, false, true);
1612   }
1613   mv(dst, src);
1614   bind(no_set);
1615 }
1616 
1617 void MacroAssembler::cmov_cmp_fp_lt(FloatRegister cmp1, FloatRegister cmp2, Register dst, Register src, bool is_single) {
1618   if (UseZicond) {
1619     if (is_single) {
1620       fle_s(t0, cmp2, cmp1);
1621     } else {
1622       fle_d(t0, cmp2, cmp1);
1623     }
1624     czero_eqz(dst, dst, t0);
1625     czero_nez(t0 , src, t0);
1626     orr(dst, dst, t0);
1627     return;
1628   }
1629   Label no_set;
1630   if (is_single) {
1631     // jump if cmp1 >= cmp2
1632     // fallthrough (i.e. move src to dst) if cmp1 < cmp2 or either is NaN
1633     float_bge(cmp1, cmp2, no_set);
1634   } else {
1635     double_bge(cmp1, cmp2, no_set);
1636   }
1637   mv(dst, src);
1638   bind(no_set);
1639 }
1640 
1641 void MacroAssembler::cmov_cmp_fp_gt(FloatRegister cmp1, FloatRegister cmp2, Register dst, Register src, bool is_single) {
1642   if (UseZicond) {
1643     if (is_single) {
1644       flt_s(t0, cmp2, cmp1);
1645     } else {
1646       flt_d(t0, cmp2, cmp1);
1647     }
1648     czero_nez(dst, dst, t0);
1649     czero_eqz(t0 , src, t0);
1650     orr(dst, dst, t0);
1651     return;
1652   }
1653   Label no_set;
1654   if (is_single) {
1655     // jump if cmp1 <= cmp2 or either is NaN
1656     // fallthrough (i.e. move src to dst) if cmp1 > cmp2
1657     float_ble(cmp1, cmp2, no_set, false, true);
1658   } else {
1659     double_ble(cmp1, cmp2, no_set, false, true);
1660   }
1661   mv(dst, src);
1662   bind(no_set);
1663 }
1664 
1665 // ----------- cmove float/double, compare float/double -----------
1666 
1667 // Move src to dst only if cmp1 == cmp2,
1668 // otherwise leave dst unchanged, including the case where one of them is NaN.
1669 // Clarification:
1670 //   java code      :  cmp1 != cmp2 ? dst : src
1671 //   transformed to :  CMove dst, (cmp1 eq cmp2), dst, src
1672 void MacroAssembler::cmov_fp_cmp_fp_eq(FloatRegister cmp1, FloatRegister cmp2,
1673                                        FloatRegister dst, FloatRegister src,
1674                                        bool cmp_single, bool cmov_single) {
1675   Label no_set;
1676   if (cmp_single) {
1677     // jump if cmp1 != cmp2, including the case of NaN
1678     // not jump (i.e. move src to dst) if cmp1 == cmp2
1679     float_bne(cmp1, cmp2, no_set);
1680   } else {
1681     double_bne(cmp1, cmp2, no_set);
1682   }
1683   if (cmov_single) {
1684     fmv_s(dst, src);
1685   } else {
1686     fmv_d(dst, src);
1687   }
1688   bind(no_set);
1689 }
1690 
1691 // Keep dst unchanged only if cmp1 == cmp2,
1692 // otherwise move src to dst, including the case where one of them is NaN.
1693 // Clarification:
1694 //   java code      :  cmp1 == cmp2 ? dst : src
1695 //   transformed to :  CMove dst, (cmp1 ne cmp2), dst, src
1696 void MacroAssembler::cmov_fp_cmp_fp_ne(FloatRegister cmp1, FloatRegister cmp2,
1697                                        FloatRegister dst, FloatRegister src,
1698                                        bool cmp_single, bool cmov_single) {
1699   Label no_set;
1700   if (cmp_single) {
1701     // jump if cmp1 == cmp2
1702     // not jump (i.e. move src to dst) if cmp1 != cmp2, including the case of NaN
1703     float_beq(cmp1, cmp2, no_set);
1704   } else {
1705     double_beq(cmp1, cmp2, no_set);
1706   }
1707   if (cmov_single) {
1708     fmv_s(dst, src);
1709   } else {
1710     fmv_d(dst, src);
1711   }
1712   bind(no_set);
1713 }
1714 
1715 // When cmp1 <= cmp2 or any of them is NaN then dst = src, otherwise, dst = dst
1716 // Clarification
1717 //   scenario 1:
1718 //     java code      :  cmp2 < cmp1 ? dst : src
1719 //     transformed to :  CMove dst, (cmp1 le cmp2), dst, src
1720 //   scenario 2:
1721 //     java code      :  cmp1 > cmp2 ? dst : src
1722 //     transformed to :  CMove dst, (cmp1 le cmp2), dst, src
1723 void MacroAssembler::cmov_fp_cmp_fp_le(FloatRegister cmp1, FloatRegister cmp2,
1724                                        FloatRegister dst, FloatRegister src,
1725                                        bool cmp_single, bool cmov_single) {
1726   Label no_set;
1727   if (cmp_single) {
1728     // jump if cmp1 > cmp2
1729     // not jump (i.e. move src to dst) if cmp1 <= cmp2 or either is NaN
1730     float_bgt(cmp1, cmp2, no_set);
1731   } else {
1732     double_bgt(cmp1, cmp2, no_set);
1733   }
1734   if (cmov_single) {
1735     fmv_s(dst, src);
1736   } else {
1737     fmv_d(dst, src);
1738   }
1739   bind(no_set);
1740 }
1741 
1742 void MacroAssembler::cmov_fp_cmp_fp_ge(FloatRegister cmp1, FloatRegister cmp2,
1743                                        FloatRegister dst, FloatRegister src,
1744                                        bool cmp_single, bool cmov_single) {
1745   Label no_set;
1746   if (cmp_single) {
1747     // jump if cmp1 < cmp2 or either is NaN
1748     // not jump (i.e. move src to dst) if cmp1 >= cmp2
1749     float_blt(cmp1, cmp2, no_set, false, true);
1750   } else {
1751     double_blt(cmp1, cmp2, no_set, false, true);
1752   }
1753   if (cmov_single) {
1754     fmv_s(dst, src);
1755   } else {
1756     fmv_d(dst, src);
1757   }
1758   bind(no_set);
1759 }
1760 
1761 // When cmp1 < cmp2 or any of them is NaN then dst = src, otherwise, dst = dst
1762 // Clarification
1763 //   scenario 1:
1764 //     java code      :  cmp2 <= cmp1 ? dst : src
1765 //     transformed to :  CMove dst, (cmp1 lt cmp2), dst, src
1766 //   scenario 2:
1767 //     java code      :  cmp1 >= cmp2 ? dst : src
1768 //     transformed to :  CMove dst, (cmp1 lt cmp2), dst, src
1769 void MacroAssembler::cmov_fp_cmp_fp_lt(FloatRegister cmp1, FloatRegister cmp2,
1770                                        FloatRegister dst, FloatRegister src,
1771                                        bool cmp_single, bool cmov_single) {
1772   Label no_set;
1773   if (cmp_single) {
1774     // jump if cmp1 >= cmp2
1775     // not jump (i.e. move src to dst) if cmp1 < cmp2 or either is NaN
1776     float_bge(cmp1, cmp2, no_set);
1777   } else {
1778     double_bge(cmp1, cmp2, no_set);
1779   }
1780   if (cmov_single) {
1781     fmv_s(dst, src);
1782   } else {
1783     fmv_d(dst, src);
1784   }
1785   bind(no_set);
1786 }
1787 
1788 void MacroAssembler::cmov_fp_cmp_fp_gt(FloatRegister cmp1, FloatRegister cmp2,
1789                                        FloatRegister dst, FloatRegister src,
1790                                        bool cmp_single, bool cmov_single) {
1791   Label no_set;
1792   if (cmp_single) {
1793     // jump if cmp1 <= cmp2 or either is NaN
1794     // not jump (i.e. move src to dst) if cmp1 > cmp2
1795     float_ble(cmp1, cmp2, no_set, false, true);
1796   } else {
1797     double_ble(cmp1, cmp2, no_set, false, true);
1798   }
1799   if (cmov_single) {
1800     fmv_s(dst, src);
1801   } else {
1802     fmv_d(dst, src);
1803   }
1804   bind(no_set);
1805 }
1806 
1807 // Float compare branch instructions
1808 
1809 #define INSN(NAME, FLOATCMP, BRANCH)                                                                                    \
1810   void MacroAssembler::float_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far, bool is_unordered) {   \
1811     FLOATCMP##_s(t0, Rs1, Rs2);                                                                                         \
1812     BRANCH(t0, l, is_far);                                                                                              \
1813   }                                                                                                                     \
1814   void MacroAssembler::double_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far, bool is_unordered) {  \
1815     FLOATCMP##_d(t0, Rs1, Rs2);                                                                                         \
1816     BRANCH(t0, l, is_far);                                                                                              \
1817   }
1818 
1819   INSN(beq, feq, bnez);
1820   INSN(bne, feq, beqz);
1821 
1822 #undef INSN
1823 
1824 
1825 #define INSN(NAME, FLOATCMP1, FLOATCMP2)                                              \
1826   void MacroAssembler::float_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l,   \
1827                                     bool is_far, bool is_unordered) {                 \
1828     if (is_unordered) {                                                               \
1829       /* jump if either source is NaN or condition is expected */                     \
1830       FLOATCMP2##_s(t0, Rs2, Rs1);                                                    \
1831       beqz(t0, l, is_far);                                                            \
1832     } else {                                                                          \
1833       /* jump if no NaN in source and condition is expected */                        \
1834       FLOATCMP1##_s(t0, Rs1, Rs2);                                                    \
1835       bnez(t0, l, is_far);                                                            \
1836     }                                                                                 \
1837   }                                                                                   \
1838   void MacroAssembler::double_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l,  \
1839                                      bool is_far, bool is_unordered) {                \
1840     if (is_unordered) {                                                               \
1841       /* jump if either source is NaN or condition is expected */                     \
1842       FLOATCMP2##_d(t0, Rs2, Rs1);                                                    \
1843       beqz(t0, l, is_far);                                                            \
1844     } else {                                                                          \
1845       /* jump if no NaN in source and condition is expected */                        \
1846       FLOATCMP1##_d(t0, Rs1, Rs2);                                                    \
1847       bnez(t0, l, is_far);                                                            \
1848     }                                                                                 \
1849   }
1850 
1851   INSN(ble, fle, flt);
1852   INSN(blt, flt, fle);
1853 
1854 #undef INSN
1855 
1856 #define INSN(NAME, CMP)                                                              \
1857   void MacroAssembler::float_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l,  \
1858                                     bool is_far, bool is_unordered) {                \
1859     float_##CMP(Rs2, Rs1, l, is_far, is_unordered);                                  \
1860   }                                                                                  \
1861   void MacroAssembler::double_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, \
1862                                      bool is_far, bool is_unordered) {               \
1863     double_##CMP(Rs2, Rs1, l, is_far, is_unordered);                                 \
1864   }
1865 
1866   INSN(bgt, blt);
1867   INSN(bge, ble);
1868 
1869 #undef INSN
1870 
1871 void MacroAssembler::csrr(Register Rd, unsigned csr) {
1872   // These three are specified in zicntr and are unused.
1873   // Before adding use-cases add the appropriate hwprobe and flag.
1874   assert(csr != CSR_INSTRET && csr != CSR_CYCLE && csr != CSR_TIME,
1875          "Not intended for use without enabling zicntr.");
1876   csrrs(Rd, csr, x0);
1877 }
1878 
1879 #define INSN(NAME, OPFUN)                                      \
1880   void MacroAssembler::NAME(unsigned csr, Register Rs) {       \
1881     OPFUN(x0, csr, Rs);                                        \
1882   }
1883 
1884   INSN(csrw, csrrw);
1885   INSN(csrs, csrrs);
1886   INSN(csrc, csrrc);
1887 
1888 #undef INSN
1889 
1890 #define INSN(NAME, OPFUN)                                      \
1891   void MacroAssembler::NAME(unsigned csr, unsigned imm) {      \
1892     OPFUN(x0, csr, imm);                                       \
1893   }
1894 
1895   INSN(csrwi, csrrwi);
1896   INSN(csrsi, csrrsi);
1897   INSN(csrci, csrrci);
1898 
1899 #undef INSN
1900 
1901 #define INSN(NAME, CSR)                                      \
1902   void MacroAssembler::NAME(Register Rd, Register Rs) {      \
1903     csrrw(Rd, CSR, Rs);                                      \
1904   }
1905 
1906   INSN(fscsr,   CSR_FCSR);
1907   INSN(fsrm,    CSR_FRM);
1908   INSN(fsflags, CSR_FFLAGS);
1909 
1910 #undef INSN
1911 
1912 #define INSN(NAME)                              \
1913   void MacroAssembler::NAME(Register Rs) {      \
1914     NAME(x0, Rs);                               \
1915   }
1916 
1917   INSN(fscsr);
1918   INSN(fsrm);
1919   INSN(fsflags);
1920 
1921 #undef INSN
1922 
1923 void MacroAssembler::fsrmi(Register Rd, unsigned imm) {
1924   guarantee(imm < 5, "Rounding Mode is invalid in Rounding Mode register");
1925   csrrwi(Rd, CSR_FRM, imm);
1926 }
1927 
1928 void MacroAssembler::fsflagsi(Register Rd, unsigned imm) {
1929    csrrwi(Rd, CSR_FFLAGS, imm);
1930 }
1931 
1932 #define INSN(NAME)                             \
1933   void MacroAssembler::NAME(unsigned imm) {    \
1934     NAME(x0, imm);                             \
1935   }
1936 
1937   INSN(fsrmi);
1938   INSN(fsflagsi);
1939 
1940 #undef INSN
1941 
1942 void MacroAssembler::restore_cpu_control_state_after_jni(Register tmp) {
1943   if (RestoreMXCSROnJNICalls) {
1944     Label skip_fsrmi;
1945     frrm(tmp);
1946     // Set FRM to the state we need. We do want Round to Nearest.
1947     // We don't want non-IEEE rounding modes.
1948     guarantee(RoundingMode::rne == 0, "must be");
1949     beqz(tmp, skip_fsrmi);        // Only reset FRM if it's wrong
1950     fsrmi(RoundingMode::rne);
1951     bind(skip_fsrmi);
1952   }
1953 }
1954 
1955 void MacroAssembler::push_reg(Register Rs) {
1956   subi(esp, esp, wordSize);
1957   sd(Rs, Address(esp, 0));
1958 }
1959 
1960 void MacroAssembler::pop_reg(Register Rd) {
1961   ld(Rd, Address(esp, 0));
1962   addi(esp, esp, wordSize);
1963 }
1964 
1965 int MacroAssembler::bitset_to_regs(unsigned int bitset, unsigned char* regs) {
1966   int count = 0;
1967   // Scan bitset to accumulate register pairs
1968   for (int reg = 31; reg >= 0; reg--) {
1969     if ((1U << 31) & bitset) {
1970       regs[count++] = reg;
1971     }
1972     bitset <<= 1;
1973   }
1974   return count;
1975 }
1976 
1977 // Push integer registers in the bitset supplied. Don't push sp.
1978 // Return the number of words pushed
1979 int MacroAssembler::push_reg(RegSet regset, Register stack) {
1980   if (regset.bits() == 0) {
1981     return 0;
1982   }
1983   auto bitset = integer_cast<unsigned int>(regset.bits());
1984   DEBUG_ONLY(int words_pushed = 0;)
1985   unsigned char regs[32];
1986   int count = bitset_to_regs(bitset, regs);
1987   // reserve one slot to align for odd count
1988   int offset = is_even(count) ? 0 : wordSize;
1989 
1990   if (count) {
1991     sub(stack, stack, count * wordSize + offset);
1992   }
1993   for (int i = count - 1; i >= 0; i--) {
1994     sd(as_Register(regs[i]), Address(stack, (count - 1 - i) * wordSize + offset));
1995     DEBUG_ONLY(words_pushed++;)
1996   }
1997 
1998   assert(words_pushed == count, "oops, pushed != count");
1999 
2000   return count;
2001 }
2002 
2003 int MacroAssembler::pop_reg(RegSet regset, Register stack) {
2004   if (regset.bits() == 0) {
2005     return 0;
2006   }
2007   auto bitset = integer_cast<unsigned int>(regset.bits());
2008   DEBUG_ONLY(int words_popped = 0;)
2009   unsigned char regs[32];
2010   int count = bitset_to_regs(bitset, regs);
2011   // reserve one slot to align for odd count
2012   int offset = is_even(count) ? 0 : wordSize;
2013 
2014   for (int i = count - 1; i >= 0; i--) {
2015     ld(as_Register(regs[i]), Address(stack, (count - 1 - i) * wordSize + offset));
2016     DEBUG_ONLY(words_popped++;)
2017   }
2018 
2019   if (count) {
2020     add(stack, stack, count * wordSize + offset);
2021   }
2022   assert(words_popped == count, "oops, popped != count");
2023 
2024   return count;
2025 }
2026 
2027 // Push floating-point registers in the bitset supplied.
2028 // Return the number of words pushed
2029 int MacroAssembler::push_fp(FloatRegSet regset, Register stack) {
2030   if (regset.bits() == 0) {
2031     return 0;
2032   }
2033   auto bitset = integer_cast<unsigned int>(regset.bits());
2034   DEBUG_ONLY(int words_pushed = 0;)
2035   unsigned char regs[32];
2036   int count = bitset_to_regs(bitset, regs);
2037   int push_slots = count + (count & 1);
2038 
2039   if (count) {
2040     subi(stack, stack, push_slots * wordSize);
2041   }
2042 
2043   for (int i = count - 1; i >= 0; i--) {
2044     fsd(as_FloatRegister(regs[i]), Address(stack, (push_slots - 1 - i) * wordSize));
2045     DEBUG_ONLY(words_pushed++;)
2046   }
2047 
2048   assert(words_pushed == count, "oops, pushed(%d) != count(%d)", words_pushed, count);
2049 
2050   return count;
2051 }
2052 
2053 int MacroAssembler::pop_fp(FloatRegSet regset, Register stack) {
2054   if (regset.bits() == 0) {
2055     return 0;
2056   }
2057   auto bitset = integer_cast<unsigned int>(regset.bits());
2058   DEBUG_ONLY(int words_popped = 0;)
2059   unsigned char regs[32];
2060   int count = bitset_to_regs(bitset, regs);
2061   int pop_slots = count + (count & 1);
2062 
2063   for (int i = count - 1; i >= 0; i--) {
2064     fld(as_FloatRegister(regs[i]), Address(stack, (pop_slots - 1 - i) * wordSize));
2065     DEBUG_ONLY(words_popped++;)
2066   }
2067 
2068   if (count) {
2069     addi(stack, stack, pop_slots * wordSize);
2070   }
2071 
2072   assert(words_popped == count, "oops, popped(%d) != count(%d)", words_popped, count);
2073 
2074   return count;
2075 }
2076 
2077 /**
2078  * Emits code to update CRC-32 with a byte value according to constants in table
2079  *
2080  * @param [in,out]crc   Register containing the crc.
2081  * @param [in]val       Register containing the byte to fold into the CRC.
2082  * @param [in]table     Register containing the table of crc constants.
2083  *
2084  * uint32_t crc;
2085  * val = crc_table[(val ^ crc) & 0xFF];
2086  * crc = val ^ (crc >> 8);
2087  *
2088  */
2089 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
2090   assert_different_registers(crc, val, table);
2091 
2092   xorr(val, val, crc);
2093   zext(val, val, 8);
2094   shadd(val, val, table, val, 2);
2095   lwu(val, Address(val));
2096   srli(crc, crc, 8);
2097   xorr(crc, val, crc);
2098 }
2099 
2100 /**
2101  * Emits code to update CRC-32 with a 32-bit value according to tables 0 to 3
2102  *
2103  * @param [in,out]crc   Register containing the crc.
2104  * @param [in]v         Register containing the 32-bit to fold into the CRC.
2105  * @param [in]table0    Register containing table 0 of crc constants.
2106  * @param [in]table1    Register containing table 1 of crc constants.
2107  * @param [in]table2    Register containing table 2 of crc constants.
2108  * @param [in]table3    Register containing table 3 of crc constants.
2109  *
2110  * uint32_t crc;
2111  *   v = crc ^ v
2112  *   crc = table3[v&0xff]^table2[(v>>8)&0xff]^table1[(v>>16)&0xff]^table0[v>>24]
2113  *
2114  */
2115 void MacroAssembler::update_word_crc32(Register crc, Register v, Register tmp1, Register tmp2, Register tmp3,
2116         Register table0, Register table1, Register table2, Register table3, bool upper) {
2117   assert_different_registers(crc, v, tmp1, tmp2, tmp3, table0, table1, table2, table3);
2118 
2119   if (upper)
2120     srli(v, v, 32);
2121   xorr(v, v, crc);
2122 
2123   zext(tmp1, v, 8);
2124   shadd(tmp1, tmp1, table3, tmp2, 2);
2125   lwu(crc, Address(tmp1));
2126 
2127   slli(tmp1, v, 16);
2128   slli(tmp3, v, 8);
2129 
2130   srliw(tmp1, tmp1, 24);
2131   srliw(tmp3, tmp3, 24);
2132 
2133   shadd(tmp1, tmp1, table2, tmp1, 2);
2134   lwu(tmp2, Address(tmp1));
2135 
2136   shadd(tmp3, tmp3, table1, tmp3, 2);
2137   xorr(crc, crc, tmp2);
2138 
2139   lwu(tmp2, Address(tmp3));
2140   // It is more optimal to use 'srli' instead of 'srliw' for case when it is not necessary to clean upper bits
2141   if (upper)
2142     srli(tmp1, v, 24);
2143   else
2144     srliw(tmp1, v, 24);
2145 
2146   // no need to clear bits other than lowest two
2147   shadd(tmp1, tmp1, table0, tmp1, 2);
2148   xorr(crc, crc, tmp2);
2149   lwu(tmp2, Address(tmp1));
2150   xorr(crc, crc, tmp2);
2151 }
2152 
2153 
2154 #ifdef COMPILER2
2155 // This improvement (vectorization) is based on java.base/share/native/libzip/zlib/zcrc32.c.
2156 // To make it, following steps are taken:
2157 //  1. in zcrc32.c, modify N to 16 and related code,
2158 //  2. re-generate the tables needed, we use tables of (N == 16, W == 4)
2159 //  3. finally vectorize the code (original implementation in zcrc32.c is just scalar code).
2160 // New tables for vector version is after table3.
2161 void MacroAssembler::vector_update_crc32(Register crc, Register buf, Register len,
2162                                          Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5,
2163                                          Register table0, Register table3) {
2164     assert_different_registers(t1, crc, buf, len, tmp1, tmp2, tmp3, tmp4, tmp5, table0, table3);
2165     const int N = 16, W = 4;
2166     const int64_t single_table_size = 256;
2167     const Register blks = tmp2;
2168     const Register tmpTable = tmp3, tableN16 = tmp4;
2169     const VectorRegister vcrc = v4, vword = v8, vtmp = v12;
2170     Label VectorLoop;
2171     Label LastBlock;
2172 
2173     add(tableN16, table3, 1 * single_table_size * sizeof(juint), tmp1);
2174     mv(tmp5, 0xff);
2175 
2176     if (MaxVectorSize == 16) {
2177       vsetivli(zr, N, Assembler::e32, Assembler::m4, Assembler::ma, Assembler::ta);
2178     } else if (MaxVectorSize == 32) {
2179       vsetivli(zr, N, Assembler::e32, Assembler::m2, Assembler::ma, Assembler::ta);
2180     } else {
2181       assert(MaxVectorSize > 32, "sanity");
2182       vsetivli(zr, N, Assembler::e32, Assembler::m1, Assembler::ma, Assembler::ta);
2183     }
2184 
2185     vmv_v_x(vcrc, zr);
2186     vmv_s_x(vcrc, crc);
2187 
2188     // multiple of 64
2189     srli(blks, len, 6);
2190     slli(t1, blks, 6);
2191     sub(len, len, t1);
2192     subi(blks, blks, 1);
2193     blez(blks, LastBlock);
2194 
2195     bind(VectorLoop);
2196     {
2197       mv(tmpTable, tableN16);
2198 
2199       vle32_v(vword, buf);
2200       vxor_vv(vword, vword, vcrc);
2201 
2202       addi(buf, buf, N*4);
2203 
2204       vand_vx(vtmp, vword, tmp5);
2205       vsll_vi(vtmp, vtmp, 2);
2206       vluxei32_v(vcrc, tmpTable, vtmp);
2207 
2208       mv(tmp1, 1);
2209       for (int k = 1; k < W; k++) {
2210         addi(tmpTable, tmpTable, single_table_size*4);
2211 
2212         slli(t1, tmp1, 3);
2213         vsrl_vx(vtmp, vword, t1);
2214 
2215         vand_vx(vtmp, vtmp, tmp5);
2216         vsll_vi(vtmp, vtmp, 2);
2217         vluxei32_v(vtmp, tmpTable, vtmp);
2218 
2219         vxor_vv(vcrc, vcrc, vtmp);
2220 
2221         addi(tmp1, tmp1, 1);
2222       }
2223 
2224       subi(blks, blks, 1);
2225       bgtz(blks, VectorLoop);
2226     }
2227 
2228     bind(LastBlock);
2229     {
2230       vle32_v(vtmp, buf);
2231       vxor_vv(vcrc, vcrc, vtmp);
2232       mv(crc, zr);
2233       for (int i = 0; i < N; i++) {
2234         vmv_x_s(tmp2, vcrc);
2235         // in vmv_x_s, the value is sign-extended to SEW bits, but we need zero-extended here.
2236         zext(tmp2, tmp2, 32);
2237         vslidedown_vi(vcrc, vcrc, 1);
2238         xorr(crc, crc, tmp2);
2239         for (int j = 0; j < W; j++) {
2240           andr(t1, crc, tmp5);
2241           shadd(t1, t1, table0, tmp1, 2);
2242           lwu(t1, Address(t1, 0));
2243           srli(tmp2, crc, 8);
2244           xorr(crc, tmp2, t1);
2245         }
2246       }
2247       addi(buf, buf, N*4);
2248     }
2249 }
2250 
2251 void MacroAssembler::crc32_vclmul_fold_16_bytes_vectorsize_16(VectorRegister vx, VectorRegister vt,
2252                       VectorRegister vtmp1, VectorRegister vtmp2, VectorRegister vtmp3, VectorRegister vtmp4,
2253                       Register buf, Register tmp, const int STEP) {
2254   assert_different_registers(vx, vt, vtmp1, vtmp2, vtmp3, vtmp4);
2255   vclmul_vv(vtmp1, vx, vt);
2256   vclmulh_vv(vtmp2, vx, vt);
2257   vle64_v(vtmp4, buf); addi(buf, buf, STEP);
2258   // low parts
2259   vredxor_vs(vtmp3, vtmp1, vtmp4);
2260   // high parts
2261   vslidedown_vi(vx, vtmp4, 1);
2262   vredxor_vs(vtmp1, vtmp2, vx);
2263   // merge low and high back
2264   vslideup_vi(vx, vtmp1, 1);
2265   vmv_x_s(tmp, vtmp3);
2266   vmv_s_x(vx, tmp);
2267 }
2268 
2269 void MacroAssembler::crc32_vclmul_fold_16_bytes_vectorsize_16_2(VectorRegister vx, VectorRegister vy, VectorRegister vt,
2270                       VectorRegister vtmp1, VectorRegister vtmp2, VectorRegister vtmp3, VectorRegister vtmp4,
2271                       Register tmp) {
2272   assert_different_registers(vx, vy, vt, vtmp1, vtmp2, vtmp3, vtmp4);
2273   vclmul_vv(vtmp1, vx, vt);
2274   vclmulh_vv(vtmp2, vx, vt);
2275   // low parts
2276   vredxor_vs(vtmp3, vtmp1, vy);
2277   // high parts
2278   vslidedown_vi(vtmp4, vy, 1);
2279   vredxor_vs(vtmp1, vtmp2, vtmp4);
2280   // merge low and high back
2281   vslideup_vi(vx, vtmp1, 1);
2282   vmv_x_s(tmp, vtmp3);
2283   vmv_s_x(vx, tmp);
2284 }
2285 
2286 void MacroAssembler::crc32_vclmul_fold_16_bytes_vectorsize_16_3(VectorRegister vx, VectorRegister vy, VectorRegister vt,
2287                       VectorRegister vtmp1, VectorRegister vtmp2, VectorRegister vtmp3, VectorRegister vtmp4,
2288                       Register tmp) {
2289   assert_different_registers(vx, vy, vt, vtmp1, vtmp2, vtmp3, vtmp4);
2290   vclmul_vv(vtmp1, vx, vt);
2291   vclmulh_vv(vtmp2, vx, vt);
2292   // low parts
2293   vredxor_vs(vtmp3, vtmp1, vy);
2294   // high parts
2295   vslidedown_vi(vtmp4, vy, 1);
2296   vredxor_vs(vtmp1, vtmp2, vtmp4);
2297   // merge low and high back
2298   vslideup_vi(vy, vtmp1, 1);
2299   vmv_x_s(tmp, vtmp3);
2300   vmv_s_x(vy, tmp);
2301 }
2302 
2303 void MacroAssembler::kernel_crc32_vclmul_fold_vectorsize_16(Register crc, Register buf, Register len,
2304                                               Register vclmul_table, Register tmp1, Register tmp2) {
2305   assert_different_registers(crc, buf, len, vclmul_table, tmp1, tmp2, t1);
2306   assert(MaxVectorSize == 16, "sanity");
2307 
2308   const int TABLE_STEP = 16;
2309   const int STEP = 16;
2310   const int LOOP_STEP = 128;
2311   const int N = 2;
2312 
2313   Register loop_step = t1;
2314 
2315   // ======== preparation ========
2316 
2317   mv(loop_step, LOOP_STEP);
2318   sub(len, len, loop_step);
2319 
2320   vsetivli(zr, N, Assembler::e64, Assembler::m1, Assembler::mu, Assembler::tu);
2321   vle64_v(v0, buf); addi(buf, buf, STEP);
2322   vle64_v(v1, buf); addi(buf, buf, STEP);
2323   vle64_v(v2, buf); addi(buf, buf, STEP);
2324   vle64_v(v3, buf); addi(buf, buf, STEP);
2325   vle64_v(v4, buf); addi(buf, buf, STEP);
2326   vle64_v(v5, buf); addi(buf, buf, STEP);
2327   vle64_v(v6, buf); addi(buf, buf, STEP);
2328   vle64_v(v7, buf); addi(buf, buf, STEP);
2329 
2330   vmv_v_x(v31, zr);
2331   vsetivli(zr, 1, Assembler::e32, Assembler::m1, Assembler::mu, Assembler::tu);
2332   vmv_s_x(v31, crc);
2333   vsetivli(zr, N, Assembler::e64, Assembler::m1, Assembler::mu, Assembler::tu);
2334   vxor_vv(v0, v0, v31);
2335 
2336   // load table
2337   vle64_v(v31, vclmul_table);
2338 
2339   Label L_16_bytes_loop;
2340   j(L_16_bytes_loop);
2341 
2342 
2343   // ======== folding 128 bytes in data buffer per round ========
2344 
2345   align(OptoLoopAlignment);
2346   bind(L_16_bytes_loop);
2347   {
2348     crc32_vclmul_fold_16_bytes_vectorsize_16(v0, v31, v8, v9, v10, v11, buf, tmp2, STEP);
2349     crc32_vclmul_fold_16_bytes_vectorsize_16(v1, v31, v12, v13, v14, v15, buf, tmp2, STEP);
2350     crc32_vclmul_fold_16_bytes_vectorsize_16(v2, v31, v16, v17, v18, v19, buf, tmp2, STEP);
2351     crc32_vclmul_fold_16_bytes_vectorsize_16(v3, v31, v20, v21, v22, v23, buf, tmp2, STEP);
2352     crc32_vclmul_fold_16_bytes_vectorsize_16(v4, v31, v24, v25, v26, v27, buf, tmp2, STEP);
2353     crc32_vclmul_fold_16_bytes_vectorsize_16(v5, v31, v8, v9, v10, v11, buf, tmp2, STEP);
2354     crc32_vclmul_fold_16_bytes_vectorsize_16(v6, v31, v12, v13, v14, v15, buf, tmp2, STEP);
2355     crc32_vclmul_fold_16_bytes_vectorsize_16(v7, v31, v16, v17, v18, v19, buf, tmp2, STEP);
2356   }
2357   sub(len, len, loop_step);
2358   bge(len, loop_step, L_16_bytes_loop);
2359 
2360 
2361   // ======== folding into 64 bytes from 128 bytes in register ========
2362 
2363   // load table
2364   addi(vclmul_table, vclmul_table, TABLE_STEP);
2365   vle64_v(v31, vclmul_table);
2366 
2367   crc32_vclmul_fold_16_bytes_vectorsize_16_2(v0, v4, v31, v8, v9, v10, v11, tmp2);
2368   crc32_vclmul_fold_16_bytes_vectorsize_16_2(v1, v5, v31, v12, v13, v14, v15, tmp2);
2369   crc32_vclmul_fold_16_bytes_vectorsize_16_2(v2, v6, v31, v16, v17, v18, v19, tmp2);
2370   crc32_vclmul_fold_16_bytes_vectorsize_16_2(v3, v7, v31, v20, v21, v22, v23, tmp2);
2371 
2372 
2373   // ======== folding into 16 bytes from 64 bytes in register ========
2374 
2375   addi(vclmul_table, vclmul_table, TABLE_STEP);
2376   vle64_v(v31, vclmul_table);
2377   crc32_vclmul_fold_16_bytes_vectorsize_16_3(v0, v3, v31, v8, v9, v10, v11, tmp2);
2378 
2379   addi(vclmul_table, vclmul_table, TABLE_STEP);
2380   vle64_v(v31, vclmul_table);
2381   crc32_vclmul_fold_16_bytes_vectorsize_16_3(v1, v3, v31, v12, v13, v14, v15, tmp2);
2382 
2383   addi(vclmul_table, vclmul_table, TABLE_STEP);
2384   vle64_v(v31, vclmul_table);
2385   crc32_vclmul_fold_16_bytes_vectorsize_16_3(v2, v3, v31, v16, v17, v18, v19, tmp2);
2386 
2387   #undef FOLD_2_VCLMUL_3
2388 
2389 
2390   // ======== final: move result to scalar regsiters ========
2391 
2392   vmv_x_s(tmp1, v3);
2393   vslidedown_vi(v1, v3, 1);
2394   vmv_x_s(tmp2, v1);
2395 }
2396 
2397 void MacroAssembler::crc32_vclmul_fold_to_16_bytes_vectorsize_32(VectorRegister vx, VectorRegister vy, VectorRegister vt,
2398                             VectorRegister vtmp1, VectorRegister vtmp2, VectorRegister vtmp3, VectorRegister vtmp4) {
2399   assert_different_registers(vx, vy, vt, vtmp1, vtmp2, vtmp3, vtmp4);
2400   vclmul_vv(vtmp1, vx, vt);
2401   vclmulh_vv(vtmp2, vx, vt);
2402   // low parts
2403   vredxor_vs(vtmp3, vtmp1, vy);
2404   // high parts
2405   vslidedown_vi(vtmp4, vy, 1);
2406   vredxor_vs(vtmp1, vtmp2, vtmp4);
2407   // merge low and high back
2408   vslideup_vi(vy, vtmp1, 1);
2409   vmv_x_s(t1, vtmp3);
2410   vmv_s_x(vy, t1);
2411 }
2412 
2413 void MacroAssembler::kernel_crc32_vclmul_fold_vectorsize_32(Register crc, Register buf, Register len,
2414                                               Register vclmul_table, Register tmp1, Register tmp2) {
2415   assert_different_registers(crc, buf, len, vclmul_table, tmp1, tmp2, t1);
2416   assert(MaxVectorSize >= 32, "sanity");
2417 
2418   // utility: load table
2419   #define CRC32_VCLMUL_LOAD_TABLE(vt, rt, vtmp, rtmp) \
2420   vid_v(vtmp); \
2421   mv(rtmp, 2); \
2422   vremu_vx(vtmp, vtmp, rtmp); \
2423   vsll_vi(vtmp, vtmp, 3); \
2424   vluxei64_v(vt, rt, vtmp);
2425 
2426   const int TABLE_STEP = 16;
2427   const int STEP = 128;  // 128 bytes per round
2428   const int N = 2 * 8;   // 2: 128-bits/64-bits, 8: 8 pairs of double 64-bits
2429 
2430   Register step = tmp2;
2431 
2432 
2433   // ======== preparation ========
2434 
2435   mv(step, STEP);
2436   sub(len, len, step); // 2 rounds of folding with carry-less multiplication
2437 
2438   vsetivli(zr, N, Assembler::e64, Assembler::m4, Assembler::mu, Assembler::tu);
2439   // load data
2440   vle64_v(v4, buf);
2441   add(buf, buf, step);
2442 
2443   // load table
2444   CRC32_VCLMUL_LOAD_TABLE(v8, vclmul_table, v28, t1);
2445   // load mask,
2446   //    v28 should already contains: 0, 8, 0, 8, ...
2447   vmseq_vi(v2, v28, 0);
2448   //    now, v2 should contains: 101010...
2449   vmnand_mm(v1, v2, v2);
2450   //    now, v1 should contains: 010101...
2451 
2452   // initial crc
2453   vmv_v_x(v24, zr);
2454   vsetivli(zr, 1, Assembler::e32, Assembler::m4, Assembler::mu, Assembler::tu);
2455   vmv_s_x(v24, crc);
2456   vsetivli(zr, N, Assembler::e64, Assembler::m4, Assembler::mu, Assembler::tu);
2457   vxor_vv(v4, v4, v24);
2458 
2459   Label L_128_bytes_loop;
2460   j(L_128_bytes_loop);
2461 
2462 
2463   // ======== folding 128 bytes in data buffer per round ========
2464 
2465   align(OptoLoopAlignment);
2466   bind(L_128_bytes_loop);
2467   {
2468     // v4: data
2469     // v4: buf, reused
2470     // v8: table
2471     // v12: lows
2472     // v16: highs
2473     // v20: low_slides
2474     // v24: high_slides
2475     vclmul_vv(v12, v4, v8);
2476     vclmulh_vv(v16, v4, v8);
2477     vle64_v(v4, buf);
2478     add(buf, buf, step);
2479     // lows
2480     vslidedown_vi(v20, v12, 1);
2481     vmand_mm(v0, v2, v2);
2482     vxor_vv(v12, v12, v20, v0_t);
2483     // with buf data
2484     vxor_vv(v4, v4, v12, v0_t);
2485 
2486     // highs
2487     vslideup_vi(v24, v16, 1);
2488     vmand_mm(v0, v1, v1);
2489     vxor_vv(v16, v16, v24, v0_t);
2490     // with buf data
2491     vxor_vv(v4, v4, v16, v0_t);
2492   }
2493   sub(len, len, step);
2494   bge(len, step, L_128_bytes_loop);
2495 
2496 
2497   // ======== folding into 64 bytes from 128 bytes in register ========
2498 
2499   // load table
2500   addi(vclmul_table, vclmul_table, TABLE_STEP);
2501   CRC32_VCLMUL_LOAD_TABLE(v8, vclmul_table, v28, t1);
2502 
2503   // v4:  data, first (low) part, N/2 of 64-bits
2504   // v20: data, second (high) part, N/2 of 64-bits
2505   // v8:  table
2506   // v10: lows
2507   // v12: highs
2508   // v14: low_slides
2509   // v16: high_slides
2510 
2511   // high part
2512   vslidedown_vi(v20, v4, N/2);
2513 
2514   vsetivli(zr, N/2, Assembler::e64, Assembler::m2, Assembler::mu, Assembler::tu);
2515 
2516   vclmul_vv(v10, v4, v8);
2517   vclmulh_vv(v12, v4, v8);
2518 
2519   // lows
2520   vslidedown_vi(v14, v10, 1);
2521   vmand_mm(v0, v2, v2);
2522   vxor_vv(v10, v10, v14, v0_t);
2523   // with data part 2
2524   vxor_vv(v4, v20, v10, v0_t);
2525 
2526   // highs
2527   vslideup_vi(v16, v12, 1);
2528   vmand_mm(v0, v1, v1);
2529   vxor_vv(v12, v12, v16, v0_t);
2530   // with data part 2
2531   vxor_vv(v4, v20, v12, v0_t);
2532 
2533 
2534   // ======== folding into 16 bytes from 64 bytes in register ========
2535 
2536   // v4:  data, first part, 2 of 64-bits
2537   // v16: data, second part, 2 of 64-bits
2538   // v18: data, third part, 2 of 64-bits
2539   // v20: data, second part, 2 of 64-bits
2540   // v8:  table
2541 
2542   vslidedown_vi(v16, v4, 2);
2543   vslidedown_vi(v18, v4, 4);
2544   vslidedown_vi(v20, v4, 6);
2545 
2546   vsetivli(zr, 2, Assembler::e64, Assembler::m1, Assembler::mu, Assembler::tu);
2547 
2548   addi(vclmul_table, vclmul_table, TABLE_STEP);
2549   vle64_v(v8, vclmul_table);
2550   crc32_vclmul_fold_to_16_bytes_vectorsize_32(v4, v20, v8, v28, v29, v30, v31);
2551 
2552   addi(vclmul_table, vclmul_table, TABLE_STEP);
2553   vle64_v(v8, vclmul_table);
2554   crc32_vclmul_fold_to_16_bytes_vectorsize_32(v16, v20, v8, v28, v29, v30, v31);
2555 
2556   addi(vclmul_table, vclmul_table, TABLE_STEP);
2557   vle64_v(v8, vclmul_table);
2558   crc32_vclmul_fold_to_16_bytes_vectorsize_32(v18, v20, v8, v28, v29, v30, v31);
2559 
2560 
2561   // ======== final: move result to scalar regsiters ========
2562 
2563   vmv_x_s(tmp1, v20);
2564   vslidedown_vi(v4, v20, 1);
2565   vmv_x_s(tmp2, v4);
2566 
2567   #undef CRC32_VCLMUL_LOAD_TABLE
2568 }
2569 
2570 // For more details of the algorithm, please check the paper:
2571 //   "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction - Intel"
2572 //
2573 // Please also refer to the corresponding code in aarch64 or x86 ones.
2574 //
2575 // As the riscv carry-less multiplication is a bit different from the other platforms,
2576 // so the implementation itself is also a bit different from others.
2577 
2578 void MacroAssembler::kernel_crc32_vclmul_fold(Register crc, Register buf, Register len,
2579                         Register table0, Register table1, Register table2, Register table3,
2580                         Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) {
2581   const int64_t single_table_size = 256;
2582   const int64_t table_num = 8;   // 4 for scalar, 4 for plain vector
2583   const ExternalAddress table_addr = StubRoutines::crc_table_addr();
2584   Register vclmul_table = tmp3;
2585 
2586   la(vclmul_table, table_addr);
2587   add(vclmul_table, vclmul_table, table_num * single_table_size * sizeof(juint), tmp1);
2588   la(table0, table_addr);
2589 
2590   if (MaxVectorSize == 16) {
2591     kernel_crc32_vclmul_fold_vectorsize_16(crc, buf, len, vclmul_table, tmp1, tmp2);
2592   } else {
2593     kernel_crc32_vclmul_fold_vectorsize_32(crc, buf, len, vclmul_table, tmp1, tmp2);
2594   }
2595 
2596   mv(crc, zr);
2597   update_word_crc32(crc, tmp1, tmp3, tmp4, tmp5, table0, table1, table2, table3, false);
2598   update_word_crc32(crc, tmp1, tmp3, tmp4, tmp5, table0, table1, table2, table3, true);
2599   update_word_crc32(crc, tmp2, tmp3, tmp4, tmp5, table0, table1, table2, table3, false);
2600   update_word_crc32(crc, tmp2, tmp3, tmp4, tmp5, table0, table1, table2, table3, true);
2601 }
2602 
2603 #endif // COMPILER2
2604 
2605 /**
2606  * @param crc   register containing existing CRC (32-bit)
2607  * @param buf   register pointing to input byte buffer (byte*)
2608  * @param len   register containing number of bytes
2609  * @param table register that will contain address of CRC table
2610  * @param tmp   scratch registers
2611  */
2612 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len,
2613         Register table0, Register table1, Register table2, Register table3,
2614         Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register tmp6) {
2615   assert_different_registers(crc, buf, len, table0, table1, table2, table3, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6);
2616   Label L_vector_entry,
2617         L_unroll_loop,
2618         L_by4_loop_entry, L_by4_loop,
2619         L_by1_loop, L_exit, L_skip1, L_skip2;
2620 
2621   const int64_t single_table_size = 256;
2622   const int64_t unroll = 16;
2623   const int64_t unroll_words = unroll*wordSize;
2624 
2625   // tmp5 = 0xffffffff
2626   notr(tmp5, zr);
2627   srli(tmp5, tmp5, 32);
2628 
2629   andn(crc, tmp5, crc);
2630 
2631   const ExternalAddress table_addr = StubRoutines::crc_table_addr();
2632   la(table0, table_addr);
2633   add(table1, table0, 1 * single_table_size * sizeof(juint), tmp1);
2634   add(table2, table0, 2 * single_table_size * sizeof(juint), tmp1);
2635   add(table3, table2, 1 * single_table_size * sizeof(juint), tmp1);
2636 
2637   // Ensure basic 4-byte alignment of input byte buffer
2638   mv(tmp1, 4);
2639   blt(len, tmp1, L_by1_loop);
2640   test_bit(tmp1, buf, 0);
2641   beqz(tmp1, L_skip1);
2642     subiw(len, len, 1);
2643     lbu(tmp1, Address(buf));
2644     addi(buf, buf, 1);
2645     update_byte_crc32(crc, tmp1, table0);
2646   bind(L_skip1);
2647     test_bit(tmp1, buf, 1);
2648     beqz(tmp1, L_skip2);
2649     subiw(len, len, 2);
2650     lhu(tmp1, Address(buf));
2651     addi(buf, buf, 2);
2652     zext(tmp2, tmp1, 8);
2653     update_byte_crc32(crc, tmp2, table0);
2654     srli(tmp2, tmp1, 8);
2655     update_byte_crc32(crc, tmp2, table0);
2656   bind(L_skip2);
2657 
2658 #ifdef COMPILER2
2659   if (UseRVV) {
2660     const int64_t tmp_limit =
2661             UseZvbc ? 128 * 3 // 3 rounds of folding with carry-less multiplication
2662                     : MaxVectorSize >= 32 ? unroll_words*3 : unroll_words*5;
2663     mv(tmp1, tmp_limit);
2664     bge(len, tmp1, L_vector_entry);
2665   }
2666 #endif // COMPILER2
2667 
2668   mv(tmp1, unroll_words);
2669   blt(len, tmp1, L_by4_loop_entry);
2670 
2671   const Register loop_buf_end = tmp3;
2672 
2673   align(CodeEntryAlignment);
2674   // Entry for L_unroll_loop
2675     add(loop_buf_end, buf, len); // loop_buf_end will be used as endpoint for loop below
2676     andi(len, len, unroll_words - 1); // len = (len % unroll_words)
2677     sub(loop_buf_end, loop_buf_end, len);
2678   bind(L_unroll_loop);
2679     for (int i = 0; i < unroll; i++) {
2680       ld(tmp1, Address(buf, i*wordSize));
2681       update_word_crc32(crc, tmp1, tmp2, tmp4, tmp6, table0, table1, table2, table3, false);
2682       update_word_crc32(crc, tmp1, tmp2, tmp4, tmp6, table0, table1, table2, table3, true);
2683     }
2684 
2685     addi(buf, buf, unroll_words);
2686     blt(buf, loop_buf_end, L_unroll_loop);
2687 
2688   bind(L_by4_loop_entry);
2689     mv(tmp1, 4);
2690     blt(len, tmp1, L_by1_loop);
2691     add(loop_buf_end, buf, len); // loop_buf_end will be used as endpoint for loop below
2692     andi(len, len, 3);
2693     sub(loop_buf_end, loop_buf_end, len);
2694   bind(L_by4_loop);
2695     lwu(tmp1, Address(buf));
2696     update_word_crc32(crc, tmp1, tmp2, tmp4, tmp6, table0, table1, table2, table3, false);
2697     addi(buf, buf, 4);
2698     blt(buf, loop_buf_end, L_by4_loop);
2699 
2700   bind(L_by1_loop);
2701     beqz(len, L_exit);
2702 
2703     subiw(len, len, 1);
2704     lbu(tmp1, Address(buf));
2705     update_byte_crc32(crc, tmp1, table0);
2706     beqz(len, L_exit);
2707 
2708     subiw(len, len, 1);
2709     lbu(tmp1, Address(buf, 1));
2710     update_byte_crc32(crc, tmp1, table0);
2711     beqz(len, L_exit);
2712 
2713     subiw(len, len, 1);
2714     lbu(tmp1, Address(buf, 2));
2715     update_byte_crc32(crc, tmp1, table0);
2716 
2717 #ifdef COMPILER2
2718   // put vector code here, otherwise "offset is too large" error occurs.
2719   if (UseRVV) {
2720     // only need to jump exit when UseRVV == true, it's a jump from end of block `L_by1_loop`.
2721     j(L_exit);
2722 
2723     bind(L_vector_entry);
2724     if (UseZvbc) { // carry-less multiplication
2725       kernel_crc32_vclmul_fold(crc, buf, len,
2726                                table0, table1, table2, table3,
2727                                tmp1, tmp2, tmp3, tmp4, tmp6);
2728     } else { // plain vector instructions
2729       vector_update_crc32(crc, buf, len, tmp1, tmp2, tmp3, tmp4, tmp6, table0, table3);
2730     }
2731 
2732     bgtz(len, L_by4_loop_entry);
2733   }
2734 #endif // COMPILER2
2735 
2736   bind(L_exit);
2737     andn(crc, tmp5, crc);
2738 }
2739 
2740 #ifdef COMPILER2
2741 // Push vector registers in the bitset supplied.
2742 // Return the number of words pushed
2743 int MacroAssembler::push_v(VectorRegSet regset, Register stack) {
2744   if (regset.bits() == 0) {
2745     return 0;
2746   }
2747   auto bitset = integer_cast<unsigned int>(regset.bits());
2748   int vector_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2749 
2750   // Scan bitset to accumulate register pairs
2751   unsigned char regs[32];
2752   int count = bitset_to_regs(bitset, regs);
2753 
2754   for (int i = 0; i < count; i++) {
2755     sub(stack, stack, vector_size_in_bytes);
2756     vs1r_v(as_VectorRegister(regs[i]), stack);
2757   }
2758 
2759   return count * vector_size_in_bytes / wordSize;
2760 }
2761 
2762 int MacroAssembler::pop_v(VectorRegSet regset, Register stack) {
2763   if (regset.bits() == 0) {
2764     return 0;
2765   }
2766   auto bitset = integer_cast<unsigned int>(regset.bits());
2767   int vector_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2768 
2769   // Scan bitset to accumulate register pairs
2770   unsigned char regs[32];
2771   int count = bitset_to_regs(bitset, regs);
2772 
2773   for (int i = count - 1; i >= 0; i--) {
2774     vl1r_v(as_VectorRegister(regs[i]), stack);
2775     add(stack, stack, vector_size_in_bytes);
2776   }
2777 
2778   return count * vector_size_in_bytes / wordSize;
2779 }
2780 #endif // COMPILER2
2781 
2782 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude) {
2783   // Push integer registers x7, x10-x17, x28-x31.
2784   push_reg(RegSet::of(x7) + RegSet::range(x10, x17) + RegSet::range(x28, x31) - exclude, sp);
2785 
2786   // Push float registers f0-f7, f10-f17, f28-f31.
2787   subi(sp, sp, wordSize * 20);
2788   int offset = 0;
2789   for (int i = 0; i < 32; i++) {
2790     if (i <= f7->encoding() || i >= f28->encoding() || (i >= f10->encoding() && i <= f17->encoding())) {
2791       fsd(as_FloatRegister(i), Address(sp, wordSize * (offset++)));
2792     }
2793   }
2794 }
2795 
2796 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude) {
2797   int offset = 0;
2798   for (int i = 0; i < 32; i++) {
2799     if (i <= f7->encoding() || i >= f28->encoding() || (i >= f10->encoding() && i <= f17->encoding())) {
2800       fld(as_FloatRegister(i), Address(sp, wordSize * (offset++)));
2801     }
2802   }
2803   addi(sp, sp, wordSize * 20);
2804 
2805   pop_reg(RegSet::of(x7) + RegSet::range(x10, x17) + RegSet::range(x28, x31) - exclude, sp);
2806 }
2807 
2808 void MacroAssembler::push_CPU_state(bool save_vectors, int vector_size_in_bytes) {
2809   // integer registers, except zr(x0) & ra(x1) & sp(x2) & gp(x3) & tp(x4)
2810   push_reg(RegSet::range(x5, x31), sp);
2811 
2812   // float registers
2813   subi(sp, sp, 32 * wordSize);
2814   for (int i = 0; i < 32; i++) {
2815     fsd(as_FloatRegister(i), Address(sp, i * wordSize));
2816   }
2817 
2818   // vector registers
2819   if (save_vectors) {
2820     sub(sp, sp, vector_size_in_bytes * VectorRegister::number_of_registers);
2821     vsetvli(t0, x0, Assembler::e64, Assembler::m8);
2822     for (int i = 0; i < VectorRegister::number_of_registers; i += 8) {
2823       add(t0, sp, vector_size_in_bytes * i);
2824       vse64_v(as_VectorRegister(i), t0);
2825     }
2826   }
2827 }
2828 
2829 void MacroAssembler::pop_CPU_state(bool restore_vectors, int vector_size_in_bytes) {
2830   // vector registers
2831   if (restore_vectors) {
2832     vsetvli(t0, x0, Assembler::e64, Assembler::m8);
2833     for (int i = 0; i < VectorRegister::number_of_registers; i += 8) {
2834       vle64_v(as_VectorRegister(i), sp);
2835       add(sp, sp, vector_size_in_bytes * 8);
2836     }
2837   }
2838 
2839   // float registers
2840   for (int i = 0; i < 32; i++) {
2841     fld(as_FloatRegister(i), Address(sp, i * wordSize));
2842   }
2843   addi(sp, sp, 32 * wordSize);
2844 
2845   // integer registers, except zr(x0) & ra(x1) & sp(x2) & gp(x3) & tp(x4)
2846   pop_reg(RegSet::range(x5, x31), sp);
2847 }
2848 
2849 static int patch_offset_in_jal(address branch, int64_t offset) {
2850   assert(Assembler::is_simm21(offset) && ((offset % 2) == 0),
2851          "offset (%ld) is too large to be patched in one jal instruction!\n", offset);
2852   Assembler::patch(branch, 31, 31, (offset >> 20) & 0x1);                       // offset[20]    ==> branch[31]
2853   Assembler::patch(branch, 30, 21, (offset >> 1)  & 0x3ff);                     // offset[10:1]  ==> branch[30:21]
2854   Assembler::patch(branch, 20, 20, (offset >> 11) & 0x1);                       // offset[11]    ==> branch[20]
2855   Assembler::patch(branch, 19, 12, (offset >> 12) & 0xff);                      // offset[19:12] ==> branch[19:12]
2856   return MacroAssembler::instruction_size;                                   // only one instruction
2857 }
2858 
2859 static int patch_offset_in_conditional_branch(address branch, int64_t offset) {
2860   assert(Assembler::is_simm13(offset) && ((offset % 2) == 0),
2861          "offset (%ld) is too large to be patched in one beq/bge/bgeu/blt/bltu/bne instruction!\n", offset);
2862   Assembler::patch(branch, 31, 31, (offset >> 12) & 0x1);                       // offset[12]    ==> branch[31]
2863   Assembler::patch(branch, 30, 25, (offset >> 5)  & 0x3f);                      // offset[10:5]  ==> branch[30:25]
2864   Assembler::patch(branch, 7,  7,  (offset >> 11) & 0x1);                       // offset[11]    ==> branch[7]
2865   Assembler::patch(branch, 11, 8,  (offset >> 1)  & 0xf);                       // offset[4:1]   ==> branch[11:8]
2866   return MacroAssembler::instruction_size;                                   // only one instruction
2867 }
2868 
2869 static int patch_offset_in_pc_relative(address branch, int64_t offset) {
2870   const int PC_RELATIVE_INSTRUCTION_NUM = 2;                                    // auipc, addi/jalr/load
2871   Assembler::patch(branch, 31, 12, ((offset + 0x800) >> 12) & 0xfffff);         // Auipc.          offset[31:12]  ==> branch[31:12]
2872   Assembler::patch(branch + 4, 31, 20, offset & 0xfff);                         // Addi/Jalr/Load. offset[11:0]   ==> branch[31:20]
2873   return PC_RELATIVE_INSTRUCTION_NUM * MacroAssembler::instruction_size;
2874 }
2875 
2876 static int patch_addr_in_movptr1(address branch, address target) {
2877   int32_t lower = ((intptr_t)target << 35) >> 35;
2878   int64_t upper = ((intptr_t)target - lower) >> 29;
2879   Assembler::patch(branch + 0,  31, 12, upper & 0xfffff);                       // Lui.             target[48:29] + target[28] ==> branch[31:12]
2880   Assembler::patch(branch + 4,  31, 20, (lower >> 17) & 0xfff);                 // Addi.            target[28:17] ==> branch[31:20]
2881   Assembler::patch(branch + 12, 31, 20, (lower >> 6) & 0x7ff);                  // Addi.            target[16: 6] ==> branch[31:20]
2882   Assembler::patch(branch + 20, 31, 20, lower & 0x3f);                          // Addi/Jalr/Load.  target[ 5: 0] ==> branch[31:20]
2883   return MacroAssembler::movptr1_instruction_size;
2884 }
2885 
2886 static int patch_addr_in_movptr2(address instruction_address, address target) {
2887   uintptr_t addr = (uintptr_t)target;
2888 
2889   assert(addr < (1ull << 48), "48-bit overflow in address constant");
2890   unsigned int upper18 = (addr >> 30ull);
2891   int lower30 = (addr & 0x3fffffffu);
2892   int low12 = (lower30 << 20) >> 20;
2893   int mid18 = ((lower30 - low12) >> 12);
2894 
2895   Assembler::patch(instruction_address + (MacroAssembler::instruction_size * 0), 31, 12, (upper18 & 0xfffff)); // Lui
2896   Assembler::patch(instruction_address + (MacroAssembler::instruction_size * 1), 31, 12, (mid18   & 0xfffff)); // Lui
2897                                                                                                                   // Slli
2898                                                                                                                   // Add
2899   Assembler::patch(instruction_address + (MacroAssembler::instruction_size * 4), 31, 20, low12 & 0xfff);      // Addi/Jalr/Load
2900 
2901   assert(MacroAssembler::target_addr_for_insn(instruction_address) == target, "Must be");
2902 
2903   return MacroAssembler::movptr2_instruction_size;
2904 }
2905 
2906 static int patch_imm_in_li16u(address branch, uint16_t target) {
2907   Assembler::patch(branch, 31, 12, target); // patch lui only
2908   return MacroAssembler::instruction_size;
2909 }
2910 
2911 int MacroAssembler::patch_imm_in_li32(address branch, int32_t target) {
2912   const int LI32_INSTRUCTIONS_NUM = 2;                                          // lui + addiw
2913   int64_t upper = (intptr_t)target;
2914   int32_t lower = (((int32_t)target) << 20) >> 20;
2915   upper -= lower;
2916   upper = (int32_t)upper;
2917   Assembler::patch(branch + 0,  31, 12, (upper >> 12) & 0xfffff);               // Lui.
2918   Assembler::patch(branch + 4,  31, 20, lower & 0xfff);                         // Addiw.
2919   return LI32_INSTRUCTIONS_NUM * MacroAssembler::instruction_size;
2920 }
2921 
2922 static long get_offset_of_jal(address insn_addr) {
2923   assert_cond(insn_addr != nullptr);
2924   long offset = 0;
2925   unsigned insn = Assembler::ld_instr(insn_addr);
2926   long val = (long)Assembler::sextract(insn, 31, 12);
2927   offset |= ((val >> 19) & 0x1) << 20;
2928   offset |= (val & 0xff) << 12;
2929   offset |= ((val >> 8) & 0x1) << 11;
2930   offset |= ((val >> 9) & 0x3ff) << 1;
2931   offset = (offset << 43) >> 43;
2932   return offset;
2933 }
2934 
2935 static long get_offset_of_conditional_branch(address insn_addr) {
2936   long offset = 0;
2937   assert_cond(insn_addr != nullptr);
2938   unsigned insn = Assembler::ld_instr(insn_addr);
2939   offset = (long)Assembler::sextract(insn, 31, 31);
2940   offset = (offset << 12) | (((long)(Assembler::sextract(insn, 7, 7) & 0x1)) << 11);
2941   offset = offset | (((long)(Assembler::sextract(insn, 30, 25) & 0x3f)) << 5);
2942   offset = offset | (((long)(Assembler::sextract(insn, 11, 8) & 0xf)) << 1);
2943   offset = (offset << 41) >> 41;
2944   return offset;
2945 }
2946 
2947 static long get_offset_of_pc_relative(address insn_addr) {
2948   long offset = 0;
2949   assert_cond(insn_addr != nullptr);
2950   offset = ((long)(Assembler::sextract(Assembler::ld_instr(insn_addr), 31, 12))) << 12;                               // Auipc.
2951   offset += ((long)Assembler::sextract(Assembler::ld_instr(insn_addr + 4), 31, 20));                                  // Addi/Jalr/Load.
2952   offset = (offset << 32) >> 32;
2953   return offset;
2954 }
2955 
2956 static address get_target_of_movptr1(address insn_addr) {
2957   assert_cond(insn_addr != nullptr);
2958   intptr_t target_address = (((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr), 31, 12)) & 0xfffff) << 29; // Lui.
2959   target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 4), 31, 20)) << 17;                 // Addi.
2960   target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 12), 31, 20)) << 6;                 // Addi.
2961   target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 20), 31, 20));                      // Addi/Jalr/Load.
2962   return (address) target_address;
2963 }
2964 
2965 static address get_target_of_movptr2(address insn_addr) {
2966   assert_cond(insn_addr != nullptr);
2967   int32_t upper18 = ((Assembler::sextract(Assembler::ld_instr(insn_addr + MacroAssembler::instruction_size * 0), 31, 12)) & 0xfffff); // Lui
2968   int32_t mid18   = ((Assembler::sextract(Assembler::ld_instr(insn_addr + MacroAssembler::instruction_size * 1), 31, 12)) & 0xfffff); // Lui
2969                                                                                                                        // 2                              // Slli
2970                                                                                                                        // 3                              // Add
2971   int32_t low12  = ((Assembler::sextract(Assembler::ld_instr(insn_addr + MacroAssembler::instruction_size * 4), 31, 20))); // Addi/Jalr/Load.
2972   address ret = (address)(((intptr_t)upper18<<30ll) + ((intptr_t)mid18<<12ll) + low12);
2973   return ret;
2974 }
2975 
2976 address MacroAssembler::get_target_of_li32(address insn_addr) {
2977   assert_cond(insn_addr != nullptr);
2978   intptr_t target_address = (((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr), 31, 12)) & 0xfffff) << 12; // Lui.
2979   target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 4), 31, 20));                       // Addiw.
2980   return (address)target_address;
2981 }
2982 
2983 // Patch any kind of instruction; there may be several instructions.
2984 // Return the total length (in bytes) of the instructions.
2985 int MacroAssembler::pd_patch_instruction_size(address instruction_address, address target) {
2986   assert_cond(instruction_address != nullptr);
2987   int64_t offset = target - instruction_address;
2988   if (MacroAssembler::is_jal_at(instruction_address)) {                         // jal
2989     return patch_offset_in_jal(instruction_address, offset);
2990   } else if (MacroAssembler::is_branch_at(instruction_address)) {               // beq/bge/bgeu/blt/bltu/bne
2991     return patch_offset_in_conditional_branch(instruction_address, offset);
2992   } else if (MacroAssembler::is_pc_relative_at(instruction_address)) {          // auipc, addi/jalr/load
2993     return patch_offset_in_pc_relative(instruction_address, offset);
2994   } else if (MacroAssembler::is_movptr1_at(instruction_address)) {              // movptr1
2995     return patch_addr_in_movptr1(instruction_address, target);
2996   } else if (MacroAssembler::is_movptr2_at(instruction_address)) {              // movptr2
2997     return patch_addr_in_movptr2(instruction_address, target);
2998   } else if (MacroAssembler::is_li32_at(instruction_address)) {                 // li32
2999     int64_t imm = (intptr_t)target;
3000     return patch_imm_in_li32(instruction_address, (int32_t)imm);
3001   } else if (MacroAssembler::is_li16u_at(instruction_address)) {
3002     int64_t imm = (intptr_t)target;
3003     return patch_imm_in_li16u(instruction_address, (uint16_t)imm);
3004   } else {
3005 #ifdef ASSERT
3006     tty->print_cr("pd_patch_instruction_size: instruction 0x%x at " INTPTR_FORMAT " could not be patched!\n",
3007                   Assembler::ld_instr(instruction_address), p2i(instruction_address));
3008     Disassembler::decode(instruction_address - 16, instruction_address + 16);
3009 #endif
3010     ShouldNotReachHere();
3011     return -1;
3012   }
3013 }
3014 
3015 address MacroAssembler::target_addr_for_insn(address insn_addr) {
3016   long offset = 0;
3017   assert_cond(insn_addr != nullptr);
3018   if (MacroAssembler::is_jal_at(insn_addr)) {                     // jal
3019     offset = get_offset_of_jal(insn_addr);
3020   } else if (MacroAssembler::is_branch_at(insn_addr)) {           // beq/bge/bgeu/blt/bltu/bne
3021     offset = get_offset_of_conditional_branch(insn_addr);
3022   } else if (MacroAssembler::is_pc_relative_at(insn_addr)) {      // auipc, addi/jalr/load
3023     offset = get_offset_of_pc_relative(insn_addr);
3024   } else if (MacroAssembler::is_movptr1_at(insn_addr)) {          // movptr1
3025     return get_target_of_movptr1(insn_addr);
3026   } else if (MacroAssembler::is_movptr2_at(insn_addr)) {          // movptr2
3027     return get_target_of_movptr2(insn_addr);
3028   } else if (MacroAssembler::is_li32_at(insn_addr)) {             // li32
3029     return get_target_of_li32(insn_addr);
3030   } else {
3031     ShouldNotReachHere();
3032   }
3033   return address(((uintptr_t)insn_addr + offset));
3034 }
3035 
3036 int MacroAssembler::patch_oop(address insn_addr, address o) {
3037   // OOPs are either narrow (32 bits) or wide (48 bits).  We encode
3038   // narrow OOPs by setting the upper 16 bits in the first
3039   // instruction.
3040   if (MacroAssembler::is_li32_at(insn_addr)) {
3041     // Move narrow OOP
3042     uint32_t n = CompressedOops::narrow_oop_value(cast_to_oop(o));
3043     return patch_imm_in_li32(insn_addr, (int32_t)n);
3044   } else if (MacroAssembler::is_movptr1_at(insn_addr)) {
3045     // Move wide OOP
3046     return patch_addr_in_movptr1(insn_addr, o);
3047   } else if (MacroAssembler::is_movptr2_at(insn_addr)) {
3048     // Move wide OOP
3049     return patch_addr_in_movptr2(insn_addr, o);
3050   }
3051   ShouldNotReachHere();
3052   return -1;
3053 }
3054 
3055 void MacroAssembler::reinit_heapbase() {
3056   if (UseCompressedOops) {
3057     if (Universe::is_fully_initialized()) {
3058       mv(xheapbase, CompressedOops::base());
3059     } else {
3060       ld(xheapbase, ExternalAddress(CompressedOops::base_addr()));
3061     }
3062   }
3063 }
3064 
3065 void MacroAssembler::movptr(Register Rd, const Address &addr, Register temp) {
3066   assert(addr.getMode() == Address::literal, "must be applied to a literal address");
3067   relocate(addr.rspec(), [&] {
3068     movptr(Rd, addr.target(), temp);
3069   });
3070 }
3071 
3072 void MacroAssembler::movptr(Register Rd, address addr, Register temp) {
3073   int offset = 0;
3074   movptr(Rd, addr, offset, temp);
3075   addi(Rd, Rd, offset);
3076 }
3077 
3078 void MacroAssembler::movptr(Register Rd, address addr, int32_t &offset, Register temp) {
3079   uint64_t uimm64 = (uint64_t)addr;
3080 #ifndef PRODUCT
3081   {
3082     char buffer[64];
3083     os::snprintf_checked(buffer, sizeof(buffer), "0x%" PRIx64, uimm64);
3084     block_comment(buffer);
3085   }
3086 #endif
3087   assert(uimm64 < (1ull << 48), "48-bit overflow in address constant");
3088 
3089   if (temp == noreg) {
3090     movptr1(Rd, uimm64, offset);
3091   } else {
3092     movptr2(Rd, uimm64, offset, temp);
3093   }
3094 }
3095 
3096 void MacroAssembler::movptr1(Register Rd, uint64_t imm64, int32_t &offset) {
3097   // Load upper 31 bits
3098   //
3099   // In case of 11th bit of `lower` is 0, it's straightforward to understand.
3100   // In case of 11th bit of `lower` is 1, it's a bit tricky, to help understand,
3101   // imagine divide both `upper` and `lower` into 2 parts respectively, i.e.
3102   // [upper_20, upper_12], [lower_20, lower_12], they are the same just before
3103   // `lower = (lower << 52) >> 52;`.
3104   // After `upper -= lower;`,
3105   //    upper_20' = upper_20 - (-1) == upper_20 + 1
3106   //    upper_12 = 0x000
3107   // After `lui(Rd, upper);`, `Rd` = upper_20' << 12
3108   // Also divide `Rd` into 2 parts [Rd_20, Rd_12],
3109   //    Rd_20 == upper_20'
3110   //    Rd_12 == 0x000
3111   // After `addi(Rd, Rd, lower);`,
3112   //    Rd_20 = upper_20' + (-1) == upper_20 + 1 - 1 = upper_20
3113   //    Rd_12 = lower_12
3114   // So, finally Rd == [upper_20, lower_12]
3115   int64_t imm = imm64 >> 17;
3116   int64_t upper = imm, lower = imm;
3117   lower = (lower << 52) >> 52;
3118   upper -= lower;
3119   upper = (int32_t)upper;
3120   lui(Rd, upper);
3121   addi(Rd, Rd, lower);
3122 
3123   // Load the rest 17 bits.
3124   slli(Rd, Rd, 11);
3125   addi(Rd, Rd, (imm64 >> 6) & 0x7ff);
3126   slli(Rd, Rd, 6);
3127 
3128   // This offset will be used by following jalr/ld.
3129   offset = imm64 & 0x3f;
3130 }
3131 
3132 void MacroAssembler::movptr2(Register Rd, uint64_t addr, int32_t &offset, Register tmp) {
3133   assert_different_registers(Rd, tmp, noreg);
3134 
3135   // addr: [upper18, lower30[mid18, lower12]]
3136 
3137   int64_t upper18 = addr >> 18;
3138   lui(tmp, upper18);
3139 
3140   int64_t lower30 = addr & 0x3fffffff;
3141   int64_t mid18 = lower30, lower12 = lower30;
3142   lower12 = (lower12 << 52) >> 52;
3143   // For this tricky part (`mid18 -= lower12;` + `offset = lower12;`),
3144   // please refer to movptr1 above.
3145   mid18 -= (int32_t)lower12;
3146   lui(Rd, mid18);
3147 
3148   slli(tmp, tmp, 18);
3149   add(Rd, Rd, tmp);
3150 
3151   offset = lower12;
3152 }
3153 
3154 // floating point imm move
3155 bool MacroAssembler::can_hf_imm_load(short imm) {
3156   jshort h_bits = (jshort)imm;
3157   if (h_bits == 0) {
3158     return true;
3159   }
3160   return can_zfa_zli_half_float(imm);
3161 }
3162 
3163 bool MacroAssembler::can_fp_imm_load(float imm) {
3164   jint f_bits = jint_cast(imm);
3165   if (f_bits == 0) {
3166     return true;
3167   }
3168   return can_zfa_zli_float(imm);
3169 }
3170 
3171 bool MacroAssembler::can_dp_imm_load(double imm) {
3172   julong d_bits = julong_cast(imm);
3173   if (d_bits == 0) {
3174     return true;
3175   }
3176   return can_zfa_zli_double(imm);
3177 }
3178 
3179 void MacroAssembler::fli_h(FloatRegister Rd, short imm) {
3180   jshort h_bits = (jshort)imm;
3181   if (h_bits == 0) {
3182     fmv_h_x(Rd, zr);
3183     return;
3184   }
3185   int Rs = zfa_zli_lookup_half_float(h_bits);
3186   assert(Rs != -1, "Must be");
3187   _fli_h(Rd, Rs);
3188 }
3189 
3190 void MacroAssembler::fli_s(FloatRegister Rd, float imm) {
3191   jint f_bits = jint_cast(imm);
3192   if (f_bits == 0) {
3193     fmv_w_x(Rd, zr);
3194     return;
3195   }
3196   int Rs = zfa_zli_lookup_float(f_bits);
3197   assert(Rs != -1, "Must be");
3198   _fli_s(Rd, Rs);
3199 }
3200 
3201 void MacroAssembler::fli_d(FloatRegister Rd, double imm) {
3202   uint64_t d_bits = (uint64_t)julong_cast(imm);
3203   if (d_bits == 0) {
3204     fmv_d_x(Rd, zr);
3205     return;
3206   }
3207   int Rs = zfa_zli_lookup_double(d_bits);
3208   assert(Rs != -1, "Must be");
3209   _fli_d(Rd, Rs);
3210 }
3211 
3212 void MacroAssembler::add(Register Rd, Register Rn, int64_t increment, Register tmp) {
3213   if (is_simm12(increment)) {
3214     addi(Rd, Rn, increment);
3215   } else {
3216     assert_different_registers(Rn, tmp);
3217     mv(tmp, increment);
3218     add(Rd, Rn, tmp);
3219   }
3220 }
3221 
3222 void MacroAssembler::sub(Register Rd, Register Rn, int64_t decrement, Register tmp) {
3223   add(Rd, Rn, -decrement, tmp);
3224 }
3225 
3226 void MacroAssembler::addw(Register Rd, Register Rn, int64_t increment, Register tmp) {
3227   if (is_simm12(increment)) {
3228     addiw(Rd, Rn, increment);
3229   } else {
3230     assert_different_registers(Rn, tmp);
3231     mv(tmp, increment);
3232     addw(Rd, Rn, tmp);
3233   }
3234 }
3235 
3236 void MacroAssembler::subw(Register Rd, Register Rn, int64_t decrement, Register tmp) {
3237   addw(Rd, Rn, -decrement, tmp);
3238 }
3239 
3240 void MacroAssembler::andrw(Register Rd, Register Rs1, Register Rs2) {
3241   andr(Rd, Rs1, Rs2);
3242   sext(Rd, Rd, 32);
3243 }
3244 
3245 void MacroAssembler::orrw(Register Rd, Register Rs1, Register Rs2) {
3246   orr(Rd, Rs1, Rs2);
3247   sext(Rd, Rd, 32);
3248 }
3249 
3250 void MacroAssembler::xorrw(Register Rd, Register Rs1, Register Rs2) {
3251   xorr(Rd, Rs1, Rs2);
3252   sext(Rd, Rd, 32);
3253 }
3254 
3255 // Rd = Rs1 & (~Rd2)
3256 void MacroAssembler::andn(Register Rd, Register Rs1, Register Rs2) {
3257   if (UseZbb) {
3258     Assembler::andn(Rd, Rs1, Rs2);
3259     return;
3260   }
3261 
3262   notr(Rd, Rs2);
3263   andr(Rd, Rs1, Rd);
3264 }
3265 
3266 // Rd = Rs1 | (~Rd2)
3267 void MacroAssembler::orn(Register Rd, Register Rs1, Register Rs2) {
3268   if (UseZbb) {
3269     Assembler::orn(Rd, Rs1, Rs2);
3270     return;
3271   }
3272 
3273   notr(Rd, Rs2);
3274   orr(Rd, Rs1, Rd);
3275 }
3276 
3277 // Note: load_unsigned_short used to be called load_unsigned_word.
3278 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
3279   int off = offset();
3280   lhu(dst, src);
3281   return off;
3282 }
3283 
3284 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
3285   int off = offset();
3286   lbu(dst, src);
3287   return off;
3288 }
3289 
3290 int MacroAssembler::load_signed_short(Register dst, Address src) {
3291   int off = offset();
3292   lh(dst, src);
3293   return off;
3294 }
3295 
3296 int MacroAssembler::load_signed_byte(Register dst, Address src) {
3297   int off = offset();
3298   lb(dst, src);
3299   return off;
3300 }
3301 
3302 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed) {
3303   switch (size_in_bytes) {
3304     case  8:  ld(dst, src); break;
3305     case  4:  is_signed ? lw(dst, src) : lwu(dst, src); break;
3306     case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
3307     case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
3308     default:  ShouldNotReachHere();
3309   }
3310 }
3311 
3312 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes) {
3313   switch (size_in_bytes) {
3314     case  8:  sd(src, dst); break;
3315     case  4:  sw(src, dst); break;
3316     case  2:  sh(src, dst); break;
3317     case  1:  sb(src, dst); break;
3318     default:  ShouldNotReachHere();
3319   }
3320 }
3321 
3322 // granularity is 1 OR 2 bytes per load. dst and src.base() allowed to be the same register
3323 void MacroAssembler::load_short_misaligned(Register dst, Address src, Register tmp, bool is_signed, int granularity) {
3324   if (granularity != 1 && granularity != 2) {
3325     ShouldNotReachHere();
3326   }
3327   if (AvoidUnalignedAccesses && (granularity != 2)) {
3328     assert_different_registers(dst, tmp);
3329     assert_different_registers(tmp, src.base());
3330     is_signed ? lb(tmp, Address(src.base(), src.offset() + 1)) : lbu(tmp, Address(src.base(), src.offset() + 1));
3331     slli(tmp, tmp, 8);
3332     lbu(dst, src);
3333     add(dst, dst, tmp);
3334   } else {
3335     is_signed ? lh(dst, src) : lhu(dst, src);
3336   }
3337 }
3338 
3339 // granularity is 1, 2 OR 4 bytes per load, if granularity 2 or 4 then dst and src.base() allowed to be the same register
3340 void MacroAssembler::load_int_misaligned(Register dst, Address src, Register tmp, bool is_signed, int granularity) {
3341   if (AvoidUnalignedAccesses && (granularity != 4)) {
3342     switch(granularity) {
3343       case 1:
3344         assert_different_registers(dst, tmp, src.base());
3345         lbu(dst, src);
3346         lbu(tmp, Address(src.base(), src.offset() + 1));
3347         slli(tmp, tmp, 8);
3348         add(dst, dst, tmp);
3349         lbu(tmp, Address(src.base(), src.offset() + 2));
3350         slli(tmp, tmp, 16);
3351         add(dst, dst, tmp);
3352         is_signed ? lb(tmp, Address(src.base(), src.offset() + 3)) : lbu(tmp, Address(src.base(), src.offset() + 3));
3353         slli(tmp, tmp, 24);
3354         add(dst, dst, tmp);
3355         break;
3356       case 2:
3357         assert_different_registers(dst, tmp);
3358         assert_different_registers(tmp, src.base());
3359         is_signed ? lh(tmp, Address(src.base(), src.offset() + 2)) : lhu(tmp, Address(src.base(), src.offset() + 2));
3360         slli(tmp, tmp, 16);
3361         lhu(dst, src);
3362         add(dst, dst, tmp);
3363         break;
3364       default:
3365         ShouldNotReachHere();
3366     }
3367   } else {
3368     is_signed ? lw(dst, src) : lwu(dst, src);
3369   }
3370 }
3371 
3372 // granularity is 1, 2, 4 or 8 bytes per load, if granularity 4 or 8 then dst and src.base() allowed to be same register
3373 void MacroAssembler::load_long_misaligned(Register dst, Address src, Register tmp, int granularity) {
3374   if (AvoidUnalignedAccesses && (granularity != 8)) {
3375     switch(granularity){
3376       case 1:
3377         assert_different_registers(dst, tmp, src.base());
3378         lbu(dst, src);
3379         lbu(tmp, Address(src.base(), src.offset() + 1));
3380         slli(tmp, tmp, 8);
3381         add(dst, dst, tmp);
3382         lbu(tmp, Address(src.base(), src.offset() + 2));
3383         slli(tmp, tmp, 16);
3384         add(dst, dst, tmp);
3385         lbu(tmp, Address(src.base(), src.offset() + 3));
3386         slli(tmp, tmp, 24);
3387         add(dst, dst, tmp);
3388         lbu(tmp, Address(src.base(), src.offset() + 4));
3389         slli(tmp, tmp, 32);
3390         add(dst, dst, tmp);
3391         lbu(tmp, Address(src.base(), src.offset() + 5));
3392         slli(tmp, tmp, 40);
3393         add(dst, dst, tmp);
3394         lbu(tmp, Address(src.base(), src.offset() + 6));
3395         slli(tmp, tmp, 48);
3396         add(dst, dst, tmp);
3397         lbu(tmp, Address(src.base(), src.offset() + 7));
3398         slli(tmp, tmp, 56);
3399         add(dst, dst, tmp);
3400         break;
3401       case 2:
3402         assert_different_registers(dst, tmp, src.base());
3403         lhu(dst, src);
3404         lhu(tmp, Address(src.base(), src.offset() + 2));
3405         slli(tmp, tmp, 16);
3406         add(dst, dst, tmp);
3407         lhu(tmp, Address(src.base(), src.offset() + 4));
3408         slli(tmp, tmp, 32);
3409         add(dst, dst, tmp);
3410         lhu(tmp, Address(src.base(), src.offset() + 6));
3411         slli(tmp, tmp, 48);
3412         add(dst, dst, tmp);
3413         break;
3414       case 4:
3415         assert_different_registers(dst, tmp);
3416         assert_different_registers(tmp, src.base());
3417         lwu(tmp, Address(src.base(), src.offset() + 4));
3418         slli(tmp, tmp, 32);
3419         lwu(dst, src);
3420         add(dst, dst, tmp);
3421         break;
3422       default:
3423         ShouldNotReachHere();
3424     }
3425   } else {
3426     ld(dst, src);
3427   }
3428 }
3429 
3430 // reverse bytes in lower word, sign-extend
3431 // Rd[32:0] = Rs[7:0] Rs[15:8] Rs[23:16] Rs[31:24]
3432 void MacroAssembler::revbw(Register Rd, Register Rs, Register tmp1, Register tmp2) {
3433   if (UseZbb) {
3434     rev8(Rd, Rs);
3435     srai(Rd, Rd, 32);
3436     return;
3437   }
3438   assert_different_registers(Rs, tmp1, tmp2);
3439   assert_different_registers(Rd, tmp1, tmp2);
3440   zext(tmp1, Rs, 8);
3441   slli(tmp1, tmp1, 8);
3442   for (int step = 8; step < 24; step += 8) {
3443     srli(tmp2, Rs, step);
3444     zext(tmp2, tmp2, 8);
3445     orr(tmp1, tmp1, tmp2);
3446     slli(tmp1, tmp1, 8);
3447   }
3448   srli(Rd, Rs, 24);
3449   zext(Rd, Rd, 8);
3450   orr(Rd, tmp1, Rd);
3451   sext(Rd, Rd, 32);
3452 }
3453 
3454 // reverse bytes in doubleword
3455 // Rd[63:0] = Rs[7:0] Rs[15:8] Rs[23:16] Rs[31:24] Rs[39:32] Rs[47,40] Rs[55,48] Rs[63:56]
3456 void MacroAssembler::revb(Register Rd, Register Rs, Register tmp1, Register tmp2) {
3457   if (UseZbb) {
3458     rev8(Rd, Rs);
3459     return;
3460   }
3461   assert_different_registers(Rs, tmp1, tmp2);
3462   assert_different_registers(Rd, tmp1, tmp2);
3463   zext(tmp1, Rs, 8);
3464   slli(tmp1, tmp1, 8);
3465   for (int step = 8; step < 56; step += 8) {
3466     srli(tmp2, Rs, step);
3467     zext(tmp2, tmp2, 8);
3468     orr(tmp1, tmp1, tmp2);
3469     slli(tmp1, tmp1, 8);
3470   }
3471   srli(Rd, Rs, 56);
3472   orr(Rd, tmp1, Rd);
3473 }
3474 
3475 // rotate right with shift bits
3476 void MacroAssembler::ror(Register dst, Register src, Register shift, Register tmp)
3477 {
3478   if (UseZbb) {
3479     rorr(dst, src, shift);
3480     return;
3481   }
3482 
3483   assert_different_registers(dst, tmp);
3484   assert_different_registers(src, tmp);
3485 
3486   mv(tmp, 64);
3487   sub(tmp, tmp, shift);
3488   sll(tmp, src, tmp);
3489   srl(dst, src, shift);
3490   orr(dst, dst, tmp);
3491 }
3492 
3493 // rotate right with shift bits
3494 void MacroAssembler::ror(Register dst, Register src, uint32_t shift, Register tmp)
3495 {
3496   if (UseZbb) {
3497     rori(dst, src, shift);
3498     return;
3499   }
3500 
3501   assert_different_registers(dst, tmp);
3502   assert_different_registers(src, tmp);
3503   assert(shift < 64, "shift amount must be < 64");
3504   slli(tmp, src, 64 - shift);
3505   srli(dst, src, shift);
3506   orr(dst, dst, tmp);
3507 }
3508 
3509 // rotate left with shift bits, 32-bit version
3510 void MacroAssembler::rolw(Register dst, Register src, uint32_t shift, Register tmp) {
3511   if (UseZbb) {
3512     // no roliw available
3513     roriw(dst, src, 32 - shift);
3514     return;
3515   }
3516 
3517   assert_different_registers(dst, tmp);
3518   assert_different_registers(src, tmp);
3519   assert(shift < 32, "shift amount must be < 32");
3520   srliw(tmp, src, 32 - shift);
3521   slliw(dst, src, shift);
3522   orr(dst, dst, tmp);
3523 }
3524 
3525 void MacroAssembler::orptr(Address adr, RegisterOrConstant src, Register tmp1, Register tmp2) {
3526   ld(tmp1, adr);
3527   if (src.is_register()) {
3528     orr(tmp1, tmp1, src.as_register());
3529   } else {
3530     if (is_simm12(src.as_constant())) {
3531       ori(tmp1, tmp1, src.as_constant());
3532     } else {
3533       assert_different_registers(tmp1, tmp2);
3534       mv(tmp2, src.as_constant());
3535       orr(tmp1, tmp1, tmp2);
3536     }
3537   }
3538   sd(tmp1, adr);
3539 }
3540 
3541 void MacroAssembler::cmp_klass_beq(Register obj, Register klass,
3542                                    Register tmp1, Register tmp2,
3543                                    Label &L, bool is_far) {
3544   assert_different_registers(obj, klass, tmp1, tmp2);
3545   if (UseCompactObjectHeaders) {
3546     load_narrow_klass_compact(tmp1, obj);
3547   } else {
3548     lwu(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
3549   }
3550   decode_klass_not_null(tmp1, tmp2);
3551   beq(klass, tmp1, L, is_far);
3552 }
3553 
3554 void MacroAssembler::cmp_klass_bne(Register obj, Register klass,
3555                                    Register tmp1, Register tmp2,
3556                                    Label &L, bool is_far) {
3557   assert_different_registers(obj, klass, tmp1, tmp2);
3558   if (UseCompactObjectHeaders) {
3559     load_narrow_klass_compact(tmp1, obj);
3560   } else {
3561     lwu(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
3562   }
3563   decode_klass_not_null(tmp1, tmp2);
3564   bne(klass, tmp1, L, is_far);
3565 }
3566 
3567 // Move an oop into a register.
3568 void MacroAssembler::movoop(Register dst, jobject obj) {
3569   int oop_index;
3570   if (obj == nullptr) {
3571     oop_index = oop_recorder()->allocate_oop_index(obj);
3572   } else {
3573 #ifdef ASSERT
3574     {
3575       ThreadInVMfromUnknown tiv;
3576       assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "should be real oop");
3577     }
3578 #endif
3579     oop_index = oop_recorder()->find_index(obj);
3580   }
3581   RelocationHolder rspec = oop_Relocation::spec(oop_index);
3582 
3583   if (BarrierSet::barrier_set()->barrier_set_assembler()->supports_instruction_patching()) {
3584     movptr(dst, Address((address)obj, rspec));
3585   } else {
3586     address dummy = address(uintptr_t(pc()) & -wordSize); // A nearby aligned address
3587     ld(dst, Address(dummy, rspec));
3588   }
3589 }
3590 
3591 // Move a metadata address into a register.
3592 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
3593   assert((uintptr_t)obj < (1ull << 48), "48-bit overflow in metadata");
3594   int oop_index;
3595   if (obj == nullptr) {
3596     oop_index = oop_recorder()->allocate_metadata_index(obj);
3597   } else {
3598     oop_index = oop_recorder()->find_index(obj);
3599   }
3600   RelocationHolder rspec = metadata_Relocation::spec(oop_index);
3601   movptr(dst, Address((address)obj, rspec));
3602 }
3603 
3604 void MacroAssembler::inline_layout_info(Register holder_klass, Register index, Register layout_info) {
3605   assert_different_registers(holder_klass, index, layout_info);
3606   InlineLayoutInfo array[2];
3607   int size = (char*)&array[1] - (char*)&array[0]; // computing size of array elements
3608   if (is_power_of_2(size)) {
3609     slli(index, index, log2i_exact(size)); // Scale index by power of 2
3610   } else {
3611     mv(layout_info, size);
3612     mul(index, index, layout_info); // Scale the index to be the entry index * array_element_size
3613   }
3614   ld(layout_info, Address(holder_klass, InstanceKlass::inline_layout_info_array_offset()));
3615   add(layout_info, layout_info, Array<InlineLayoutInfo>::base_offset_in_bytes());
3616   add(layout_info, layout_info, index);
3617   la(layout_info, Address(layout_info));
3618 }
3619 
3620 void MacroAssembler::flat_field_copy(DecoratorSet decorators, Register src, Register dst,
3621                                      Register inline_layout_info) {
3622   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3623   bs->flat_field_copy(this, decorators, src, dst, inline_layout_info);
3624 }
3625 
3626 void MacroAssembler::payload_offset(Register inline_klass, Register offset) {
3627   ld(offset, Address(inline_klass, InlineKlass::adr_members_offset()));
3628   lwu(offset, Address(offset, InlineKlass::payload_offset_offset()));
3629 }
3630 
3631 void MacroAssembler::payload_address(Register oop, Register data, Register inline_klass) {
3632   assert_different_registers(data, t0);
3633   // ((address) (void*) o) + vk->payload_offset();
3634   Register offset = (data == oop) ? t0 : data;
3635   payload_offset(inline_klass, offset);
3636   if (data == oop) {
3637     add(data, data, offset);
3638   } else {
3639     add(data, oop, offset);
3640     la(data, Address(data));
3641   }
3642 }
3643 
3644 // Writes to stack successive pages until offset reached to check for
3645 // stack overflow + shadow pages.  This clobbers tmp.
3646 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
3647   assert_different_registers(tmp, size, t0);
3648   // Bang stack for total size given plus shadow page size.
3649   // Bang one page at a time because large size can bang beyond yellow and
3650   // red zones.
3651   mv(t0, (int)os::vm_page_size());
3652   Label loop;
3653   bind(loop);
3654   sub(tmp, sp, t0);
3655   subw(size, size, t0);
3656   sd(size, Address(tmp));
3657   bgtz(size, loop);
3658 
3659   // Bang down shadow pages too.
3660   // At this point, (tmp-0) is the last address touched, so don't
3661   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
3662   // was post-decremented.)  Skip this address by starting at i=1, and
3663   // touch a few more pages below.  N.B.  It is important to touch all
3664   // the way down to and including i=StackShadowPages.
3665   for (int i = 0; i < (int)(StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()) - 1; i++) {
3666     // this could be any sized move but this is can be a debugging crumb
3667     // so the bigger the better.
3668     sub(tmp, tmp, (int)os::vm_page_size());
3669     sd(size, Address(tmp, 0));
3670   }
3671 }
3672 
3673 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp1, Register tmp2) {
3674   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
3675   ld(dst, Address(xmethod, Method::const_offset()));
3676   ld(dst, Address(dst, ConstMethod::constants_offset()));
3677   ld(dst, Address(dst, ConstantPool::pool_holder_offset()));
3678   ld(dst, Address(dst, mirror_offset));
3679   resolve_oop_handle(dst, tmp1, tmp2);
3680 }
3681 
3682 void MacroAssembler::resolve_oop_handle(Register result, Register tmp1, Register tmp2) {
3683   // OopHandle::resolve is an indirection.
3684   assert_different_registers(result, tmp1, tmp2);
3685   access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp1, tmp2);
3686 }
3687 
3688 // ((WeakHandle)result).resolve()
3689 void MacroAssembler::resolve_weak_handle(Register result, Register tmp1, Register tmp2) {
3690   assert_different_registers(result, tmp1, tmp2);
3691   Label resolved;
3692 
3693   // A null weak handle resolves to null.
3694   beqz(result, resolved);
3695 
3696   // Only 64 bit platforms support GCs that require a tmp register
3697   // Only IN_HEAP loads require a thread_tmp register
3698   // WeakHandle::resolve is an indirection like jweak.
3699   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
3700                  result, Address(result), tmp1, tmp2);
3701   bind(resolved);
3702 }
3703 
3704 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators,
3705                                     Register dst, Address src,
3706                                     Register tmp1, Register tmp2) {
3707   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
3708   decorators = AccessInternal::decorator_fixup(decorators, type);
3709   bool as_raw = (decorators & AS_RAW) != 0;
3710   if (as_raw) {
3711     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, tmp2);
3712   } else {
3713     bs->load_at(this, decorators, type, dst, src, tmp1, tmp2);
3714   }
3715 }
3716 
3717 void MacroAssembler::null_check(Register reg, int offset) {
3718   if (needs_explicit_null_check(offset)) {
3719     // provoke OS null exception if reg is null by
3720     // accessing M[reg] w/o changing any registers
3721     // NOTE: this is plenty to provoke a segv
3722     ld(zr, Address(reg, 0));
3723   } else {
3724     // nothing to do, (later) access of M[reg + offset]
3725     // will provoke OS null exception if reg is null
3726   }
3727 }
3728 
3729 void MacroAssembler::test_field_is_null_free_inline_type(Register flags, Register temp_reg, Label& is_null_free_inline_type) {
3730   test_bit(temp_reg, flags, ResolvedFieldEntry::is_null_free_inline_type_shift);
3731   bnez(temp_reg, is_null_free_inline_type);
3732 }
3733 
3734 void MacroAssembler::test_field_is_not_null_free_inline_type(Register flags, Register temp_reg, Label& not_null_free_inline_type) {
3735   test_bit(temp_reg, flags, ResolvedFieldEntry::is_null_free_inline_type_shift);
3736   beqz(temp_reg, not_null_free_inline_type);
3737 }
3738 
3739 void MacroAssembler::test_field_is_flat(Register flags, Register temp_reg, Label& is_flat) {
3740   test_bit(temp_reg, flags, ResolvedFieldEntry::is_flat_shift);
3741   bnez(temp_reg, is_flat);
3742 }
3743 
3744 void MacroAssembler::test_markword_is_inline_type(Register markword, Label& is_inline_type) {
3745   assert_different_registers(markword, t1);
3746   mv(t1, markWord::inline_type_pattern_mask);
3747   andr(markword, markword, t1);
3748   mv(t1, markWord::inline_type_pattern);
3749   beq(markword, t1, is_inline_type);
3750 }
3751 
3752 void MacroAssembler::test_oop_is_not_inline_type(Register object, Register tmp, Label& not_inline_type, bool can_be_null) {
3753   assert_different_registers(tmp, t0);
3754   if (can_be_null) {
3755     beqz(object, not_inline_type);
3756   }
3757   const int is_inline_type_mask = markWord::inline_type_pattern;
3758   ld(tmp, Address(object, oopDesc::mark_offset_in_bytes()));
3759   mv(t0, is_inline_type_mask);
3760   andr(tmp, tmp, t0);
3761   bne(tmp, t0, not_inline_type);
3762 }
3763 
3764 void MacroAssembler::test_oop_prototype_bit(Register oop, Register temp_reg, int32_t tst_bit, bool jmp_set, Label& jmp_label) {
3765   assert_different_registers(temp_reg, t0);
3766   // load mark word
3767   ld(temp_reg, Address(oop, oopDesc::mark_offset_in_bytes()));
3768   if (!UseObjectMonitorTable) {
3769     Label test_mark_word;
3770     // check displaced
3771     test_bit(t0, temp_reg, exact_log2(markWord::unlocked_value));
3772     bnez(t0, test_mark_word);
3773     // slow path use klass prototype
3774     load_prototype_header(temp_reg, oop);
3775 
3776     bind(test_mark_word);
3777   }
3778   andi(temp_reg, temp_reg, tst_bit);
3779   if (jmp_set) {
3780     bnez(temp_reg, jmp_label, /* is_far */ true);
3781   } else {
3782     beqz(temp_reg, jmp_label, /* is_far */ true);
3783   }
3784 }
3785 
3786 void MacroAssembler::test_flat_array_oop(Register oop, Register temp_reg, Label& is_flat_array) {
3787   test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, true, is_flat_array);
3788 }
3789 
3790 void MacroAssembler::test_null_free_array_oop(Register oop, Register temp_reg, Label& is_null_free_array) {
3791   test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, true, is_null_free_array);
3792 }
3793 
3794 void MacroAssembler::test_non_flat_array_oop(Register oop, Register temp_reg, Label&is_non_flat_array) {
3795   test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, false, is_non_flat_array);
3796 }
3797 
3798 void MacroAssembler::test_non_null_free_array_oop(Register oop, Register temp_reg, Label&is_non_null_free_array) {
3799   test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, false, is_non_null_free_array);
3800 }
3801 
3802 void MacroAssembler::test_flat_array_layout(Register lh, Label& is_flat_array) {
3803   test_bit(t0, lh, exact_log2(Klass::_lh_array_tag_flat_value_bit_inplace));
3804   bnez(t0, is_flat_array);
3805 }
3806 
3807 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
3808                                      Address dst, Register val,
3809                                      Register tmp1, Register tmp2, Register tmp3) {
3810   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
3811   decorators = AccessInternal::decorator_fixup(decorators, type);
3812   bool as_raw = (decorators & AS_RAW) != 0;
3813   if (as_raw) {
3814     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
3815   } else {
3816     bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
3817   }
3818 }
3819 
3820 // Algorithm must match CompressedOops::encode.
3821 void MacroAssembler::encode_heap_oop(Register d, Register s) {
3822   verify_oop_msg(s, "broken oop in encode_heap_oop");
3823   if (CompressedOops::base() == nullptr) {
3824     if (CompressedOops::shift() != 0) {
3825       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3826       srli(d, s, LogMinObjAlignmentInBytes);
3827     } else {
3828       mv(d, s);
3829     }
3830   } else {
3831     Label notNull;
3832     sub(d, s, xheapbase);
3833     bgez(d, notNull);
3834     mv(d, zr);
3835     bind(notNull);
3836     if (CompressedOops::shift() != 0) {
3837       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3838       srli(d, d, CompressedOops::shift());
3839     }
3840   }
3841 }
3842 
3843 void MacroAssembler::encode_heap_oop_not_null(Register r) {
3844 #ifdef ASSERT
3845   if (CheckCompressedOops) {
3846     Label ok;
3847     bnez(r, ok);
3848     stop("null oop passed to encode_heap_oop_not_null");
3849     bind(ok);
3850   }
3851 #endif
3852   verify_oop_msg(r, "broken oop in encode_heap_oop_not_null");
3853   if (CompressedOops::base() != nullptr) {
3854     sub(r, r, xheapbase);
3855   }
3856   if (CompressedOops::shift() != 0) {
3857     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3858     srli(r, r, LogMinObjAlignmentInBytes);
3859   }
3860 }
3861 
3862 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
3863 #ifdef ASSERT
3864   if (CheckCompressedOops) {
3865     Label ok;
3866     bnez(src, ok);
3867     stop("null oop passed to encode_heap_oop_not_null2");
3868     bind(ok);
3869   }
3870 #endif
3871   verify_oop_msg(src, "broken oop in encode_heap_oop_not_null2");
3872 
3873   Register data = src;
3874   if (CompressedOops::base() != nullptr) {
3875     sub(dst, src, xheapbase);
3876     data = dst;
3877   }
3878   if (CompressedOops::shift() != 0) {
3879     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3880     srli(dst, data, LogMinObjAlignmentInBytes);
3881     data = dst;
3882   }
3883   if (data == src) {
3884     mv(dst, src);
3885   }
3886 }
3887 
3888 void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
3889   assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders");
3890   ld(dst, Address(src, oopDesc::mark_offset_in_bytes()));
3891   srli(dst, dst, markWord::klass_shift);
3892 }
3893 
3894 void MacroAssembler::load_narrow_klass(Register dst, Register src) {
3895   if (UseCompactObjectHeaders) {
3896     load_narrow_klass_compact(dst, src);
3897   } else {
3898     lwu(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3899   }
3900 }
3901 
3902 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
3903   assert_different_registers(dst, tmp);
3904   assert_different_registers(src, tmp);
3905   load_narrow_klass(dst, src);
3906   decode_klass_not_null(dst, tmp);
3907 }
3908 
3909 void MacroAssembler::load_prototype_header(Register dst, Register src, Register tmp) {
3910   load_klass(dst, src, tmp);
3911   ld(dst, Address(dst, Klass::prototype_header_offset()));
3912 }
3913 
3914 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
3915   // FIXME: Should this be a store release? concurrent gcs assumes
3916   // klass length is valid if klass field is not null.
3917   assert(!UseCompactObjectHeaders, "not with compact headers");
3918   encode_klass_not_null(src, tmp);
3919   sw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3920 
3921 }
3922 
3923 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3924   assert(!UseCompactObjectHeaders, "not with compact headers");
3925   // Store to klass gap in destination
3926   sw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3927 }
3928 
3929 void MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
3930   assert_different_registers(r, tmp);
3931   decode_klass_not_null(r, r, tmp);
3932 }
3933 
3934 void MacroAssembler::decode_klass_not_null(Register dst, Register src, Register tmp) {
3935   assert_different_registers(dst, tmp);
3936   assert_different_registers(src, tmp);
3937 
3938   if (CompressedKlassPointers::base() == nullptr) {
3939     if (CompressedKlassPointers::shift() != 0) {
3940       slli(dst, src, CompressedKlassPointers::shift());
3941     } else {
3942       mv(dst, src);
3943     }
3944     return;
3945   }
3946 
3947   Register xbase = tmp;
3948 
3949   mv(xbase, (uintptr_t)CompressedKlassPointers::base());
3950 
3951   if (CompressedKlassPointers::shift() != 0) {
3952     // dst = (src << shift) + xbase
3953     shadd(dst, src, xbase, dst /* temporary, dst != xbase */, CompressedKlassPointers::shift());
3954   } else {
3955     add(dst, xbase, src);
3956   }
3957 }
3958 
3959 void MacroAssembler::encode_klass_not_null(Register r, Register tmp) {
3960   assert_different_registers(r, tmp);
3961   encode_klass_not_null(r, r, tmp);
3962 }
3963 
3964 void MacroAssembler::encode_klass_not_null(Register dst, Register src, Register tmp) {
3965   if (CompressedKlassPointers::base() == nullptr) {
3966     if (CompressedKlassPointers::shift() != 0) {
3967       srli(dst, src, CompressedKlassPointers::shift());
3968     } else {
3969       mv(dst, src);
3970     }
3971     return;
3972   }
3973 
3974   if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0 &&
3975       CompressedKlassPointers::shift() == 0) {
3976     zext(dst, src, 32);
3977     return;
3978   }
3979 
3980   Register xbase = dst;
3981   if (dst == src) {
3982     xbase = tmp;
3983   }
3984 
3985   assert_different_registers(src, xbase);
3986   mv(xbase, (uintptr_t)CompressedKlassPointers::base());
3987   sub(dst, src, xbase);
3988   if (CompressedKlassPointers::shift() != 0) {
3989     srli(dst, dst, CompressedKlassPointers::shift());
3990   }
3991 }
3992 
3993 void MacroAssembler::decode_heap_oop_not_null(Register r) {
3994   decode_heap_oop_not_null(r, r);
3995 }
3996 
3997 void MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
3998   assert(UseCompressedOops, "should only be used for compressed headers");
3999   assert(Universe::heap() != nullptr, "java heap should be initialized");
4000   // Cannot assert, unverified entry point counts instructions (see .ad file)
4001   // vtableStubs also counts instructions in pd_code_size_limit.
4002   // Also do not verify_oop as this is called by verify_oop.
4003   if (CompressedOops::shift() != 0) {
4004     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
4005     slli(dst, src, LogMinObjAlignmentInBytes);
4006     if (CompressedOops::base() != nullptr) {
4007       add(dst, xheapbase, dst);
4008     }
4009   } else {
4010     assert(CompressedOops::base() == nullptr, "sanity");
4011     mv(dst, src);
4012   }
4013 }
4014 
4015 void  MacroAssembler::decode_heap_oop(Register d, Register s) {
4016   if (CompressedOops::base() == nullptr) {
4017     if (CompressedOops::shift() != 0 || d != s) {
4018       slli(d, s, CompressedOops::shift());
4019     }
4020   } else {
4021     Label done;
4022     mv(d, s);
4023     beqz(s, done);
4024     shadd(d, s, xheapbase, d, LogMinObjAlignmentInBytes);
4025     bind(done);
4026   }
4027   verify_oop_msg(d, "broken oop in decode_heap_oop");
4028 }
4029 
4030 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1,
4031                                     Register tmp2, Register tmp3, DecoratorSet decorators) {
4032   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3);
4033 }
4034 
4035 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
4036                                    Register tmp2, DecoratorSet decorators) {
4037   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2);
4038 }
4039 
4040 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
4041                                             Register tmp2, DecoratorSet decorators) {
4042   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL, dst, src, tmp1, tmp2);
4043 }
4044 
4045 // Used for storing nulls.
4046 void MacroAssembler::store_heap_oop_null(Address dst) {
4047   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
4048 }
4049 
4050 // Look up the method for a megamorphic invokeinterface call.
4051 // The target method is determined by <intf_klass, itable_index>.
4052 // The receiver klass is in recv_klass.
4053 // On success, the result will be in method_result, and execution falls through.
4054 // On failure, execution transfers to the given label.
4055 void MacroAssembler::lookup_interface_method(Register recv_klass,
4056                                              Register intf_klass,
4057                                              RegisterOrConstant itable_index,
4058                                              Register method_result,
4059                                              Register scan_tmp,
4060                                              Label& L_no_such_interface,
4061                                              bool return_method) {
4062   assert_different_registers(recv_klass, intf_klass, scan_tmp);
4063   assert_different_registers(method_result, intf_klass, scan_tmp);
4064   assert(recv_klass != method_result || !return_method,
4065          "recv_klass can be destroyed when method isn't needed");
4066   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
4067          "caller must use same register for non-constant itable index as for method");
4068 
4069   // Compute start of first itableOffsetEntry (which is at the end of the vtable).
4070   int vtable_base = in_bytes(Klass::vtable_start_offset());
4071   int itentry_off = in_bytes(itableMethodEntry::method_offset());
4072   int scan_step   = itableOffsetEntry::size() * wordSize;
4073   int vte_size    = vtableEntry::size_in_bytes();
4074   assert(vte_size == wordSize, "else adjust times_vte_scale");
4075 
4076   lwu(scan_tmp, Address(recv_klass, Klass::vtable_length_offset()));
4077 
4078   // Could store the aligned, prescaled offset in the klass.
4079   shadd(scan_tmp, scan_tmp, recv_klass, scan_tmp, 3);
4080   add(scan_tmp, scan_tmp, vtable_base);
4081 
4082   if (return_method) {
4083     // Adjust recv_klass by scaled itable_index, so we can free itable_index.
4084     assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
4085     if (itable_index.is_register()) {
4086       slli(t0, itable_index.as_register(), 3);
4087     } else {
4088       mv(t0, itable_index.as_constant() << 3);
4089     }
4090     add(recv_klass, recv_klass, t0);
4091     if (itentry_off) {
4092       add(recv_klass, recv_klass, itentry_off);
4093     }
4094   }
4095 
4096   Label search, found_method;
4097 
4098   ld(method_result, Address(scan_tmp, itableOffsetEntry::interface_offset()));
4099   beq(intf_klass, method_result, found_method);
4100   bind(search);
4101   // Check that the previous entry is non-null. A null entry means that
4102   // the receiver class doesn't implement the interface, and wasn't the
4103   // same as when the caller was compiled.
4104   beqz(method_result, L_no_such_interface, /* is_far */ true);
4105   addi(scan_tmp, scan_tmp, scan_step);
4106   ld(method_result, Address(scan_tmp, itableOffsetEntry::interface_offset()));
4107   bne(intf_klass, method_result, search);
4108 
4109   bind(found_method);
4110 
4111   // Got a hit.
4112   if (return_method) {
4113     lwu(scan_tmp, Address(scan_tmp, itableOffsetEntry::offset_offset()));
4114     add(method_result, recv_klass, scan_tmp);
4115     ld(method_result, Address(method_result));
4116   }
4117 }
4118 
4119 // Look up the method for a megamorphic invokeinterface call in a single pass over itable:
4120 // - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData
4121 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index
4122 // The target method is determined by <holder_klass, itable_index>.
4123 // The receiver klass is in recv_klass.
4124 // On success, the result will be in method_result, and execution falls through.
4125 // On failure, execution transfers to the given label.
4126 void MacroAssembler::lookup_interface_method_stub(Register recv_klass,
4127                                                   Register holder_klass,
4128                                                   Register resolved_klass,
4129                                                   Register method_result,
4130                                                   Register temp_itbl_klass,
4131                                                   Register scan_temp,
4132                                                   int itable_index,
4133                                                   Label& L_no_such_interface) {
4134   // 'method_result' is only used as output register at the very end of this method.
4135   // Until then we can reuse it as 'holder_offset'.
4136   Register holder_offset = method_result;
4137   assert_different_registers(resolved_klass, recv_klass, holder_klass, temp_itbl_klass, scan_temp, holder_offset);
4138 
4139   int vtable_start_offset_bytes = in_bytes(Klass::vtable_start_offset());
4140   int scan_step = itableOffsetEntry::size() * wordSize;
4141   int ioffset_bytes = in_bytes(itableOffsetEntry::interface_offset());
4142   int ooffset_bytes = in_bytes(itableOffsetEntry::offset_offset());
4143   int itmentry_off_bytes = in_bytes(itableMethodEntry::method_offset());
4144   const int vte_scale = exact_log2(vtableEntry::size_in_bytes());
4145 
4146   Label L_loop_search_resolved_entry, L_resolved_found, L_holder_found;
4147 
4148   lwu(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
4149   add(recv_klass, recv_klass, vtable_start_offset_bytes + ioffset_bytes);
4150   // itableOffsetEntry[] itable = recv_klass + Klass::vtable_start_offset()
4151   //                            + sizeof(vtableEntry) * (recv_klass->_vtable_len);
4152   // scan_temp = &(itable[0]._interface)
4153   // temp_itbl_klass = itable[0]._interface;
4154   shadd(scan_temp, scan_temp, recv_klass, scan_temp, vte_scale);
4155   ld(temp_itbl_klass, Address(scan_temp));
4156   mv(holder_offset, zr);
4157 
4158   // Initial checks:
4159   //   - if (holder_klass != resolved_klass), go to "scan for resolved"
4160   //   - if (itable[0] == holder_klass), shortcut to "holder found"
4161   //   - if (itable[0] == 0), no such interface
4162   bne(resolved_klass, holder_klass, L_loop_search_resolved_entry);
4163   beq(holder_klass, temp_itbl_klass, L_holder_found);
4164   beqz(temp_itbl_klass, L_no_such_interface);
4165 
4166   // Loop: Look for holder_klass record in itable
4167   //   do {
4168   //     temp_itbl_klass = *(scan_temp += scan_step);
4169   //     if (temp_itbl_klass == holder_klass) {
4170   //       goto L_holder_found; // Found!
4171   //     }
4172   //   } while (temp_itbl_klass != 0);
4173   //   goto L_no_such_interface // Not found.
4174   Label L_search_holder;
4175   bind(L_search_holder);
4176     add(scan_temp, scan_temp, scan_step);
4177     ld(temp_itbl_klass, Address(scan_temp));
4178     beq(holder_klass, temp_itbl_klass, L_holder_found);
4179     bnez(temp_itbl_klass, L_search_holder);
4180 
4181   j(L_no_such_interface);
4182 
4183   // Loop: Look for resolved_class record in itable
4184   //   while (true) {
4185   //     temp_itbl_klass = *(scan_temp += scan_step);
4186   //     if (temp_itbl_klass == 0) {
4187   //       goto L_no_such_interface;
4188   //     }
4189   //     if (temp_itbl_klass == resolved_klass) {
4190   //        goto L_resolved_found;  // Found!
4191   //     }
4192   //     if (temp_itbl_klass == holder_klass) {
4193   //        holder_offset = scan_temp;
4194   //     }
4195   //   }
4196   //
4197   Label L_loop_search_resolved;
4198   bind(L_loop_search_resolved);
4199     add(scan_temp, scan_temp, scan_step);
4200     ld(temp_itbl_klass, Address(scan_temp));
4201   bind(L_loop_search_resolved_entry);
4202     beqz(temp_itbl_klass, L_no_such_interface);
4203     beq(resolved_klass, temp_itbl_klass, L_resolved_found);
4204     bne(holder_klass, temp_itbl_klass, L_loop_search_resolved);
4205     mv(holder_offset, scan_temp);
4206     j(L_loop_search_resolved);
4207 
4208   // See if we already have a holder klass. If not, go and scan for it.
4209   bind(L_resolved_found);
4210   beqz(holder_offset, L_search_holder);
4211   mv(scan_temp, holder_offset);
4212 
4213   // Finally, scan_temp contains holder_klass vtable offset
4214   bind(L_holder_found);
4215   lwu(method_result, Address(scan_temp, ooffset_bytes - ioffset_bytes));
4216   add(recv_klass, recv_klass, itable_index * wordSize + itmentry_off_bytes
4217                               - vtable_start_offset_bytes - ioffset_bytes); // substract offsets to restore the original value of recv_klass
4218   add(method_result, recv_klass, method_result);
4219   ld(method_result, Address(method_result));
4220 }
4221 
4222 // virtual method calling
4223 void MacroAssembler::lookup_virtual_method(Register recv_klass,
4224                                            RegisterOrConstant vtable_index,
4225                                            Register method_result) {
4226   const ByteSize base = Klass::vtable_start_offset();
4227   assert(vtableEntry::size() * wordSize == 8,
4228          "adjust the scaling in the code below");
4229   int vtable_offset_in_bytes = in_bytes(base + vtableEntry::method_offset());
4230 
4231   if (vtable_index.is_register()) {
4232     shadd(method_result, vtable_index.as_register(), recv_klass, method_result, LogBytesPerWord);
4233     ld(method_result, Address(method_result, vtable_offset_in_bytes));
4234   } else {
4235     vtable_offset_in_bytes += vtable_index.as_constant() * wordSize;
4236     ld(method_result, form_address(method_result, recv_klass, vtable_offset_in_bytes));
4237   }
4238 }
4239 
4240 void MacroAssembler::membar(uint32_t order_constraint) {
4241   if (UseZtso && ((order_constraint & StoreLoad) != StoreLoad)) {
4242     // TSO allows for stores to be reordered after loads. When the compiler
4243     // generates a fence to disallow that, we are required to generate the
4244     // fence for correctness.
4245     BLOCK_COMMENT("elided tso membar");
4246     return;
4247   }
4248 
4249   address prev = pc() - MacroAssembler::instruction_size;
4250   address last = code()->last_merge_candidate();
4251 
4252   if (last != nullptr && is_membar(last) && prev == last) {
4253     // We are merging two memory barrier instructions.  On RISCV we
4254     // can do this simply by ORing them together.
4255     set_membar_kind(prev, get_membar_kind(prev) | order_constraint);
4256     BLOCK_COMMENT("merged membar");
4257     return;
4258   }
4259 
4260   code()->set_last_merge_candidate(pc());
4261   uint32_t predecessor = 0;
4262   uint32_t successor = 0;
4263   membar_mask_to_pred_succ(order_constraint, predecessor, successor);
4264   fence(predecessor, successor);
4265 }
4266 
4267 void MacroAssembler::cmodx_fence() {
4268   BLOCK_COMMENT("cmodx fence");
4269   if (VM_Version::supports_fencei_barrier()) {
4270     Assembler::fencei();
4271   }
4272 }
4273 
4274 // Form an address from base + offset in Rd. Rd my or may not
4275 // actually be used: you must use the Address that is returned. It
4276 // is up to you to ensure that the shift provided matches the size
4277 // of your data.
4278 Address MacroAssembler::form_address(Register Rd, Register base, int64_t byte_offset) {
4279   if (is_simm12(byte_offset)) { // 12: imm in range 2^12
4280     return Address(base, byte_offset);
4281   }
4282 
4283   assert_different_registers(Rd, base, noreg);
4284 
4285   // Do it the hard way
4286   mv(Rd, byte_offset);
4287   add(Rd, base, Rd);
4288   return Address(Rd);
4289 }
4290 
4291 void MacroAssembler::check_klass_subtype(Register sub_klass,
4292                                          Register super_klass,
4293                                          Register tmp_reg,
4294                                          Label& L_success) {
4295   Label L_failure;
4296   check_klass_subtype_fast_path(sub_klass, super_klass, tmp_reg, &L_success, &L_failure, nullptr);
4297   check_klass_subtype_slow_path(sub_klass, super_klass, tmp_reg, noreg, &L_success, nullptr);
4298   bind(L_failure);
4299 }
4300 
4301 void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool in_nmethod, Register tmp_reg) {
4302   ld(tmp_reg, Address(xthread, JavaThread::polling_word_offset()));
4303   if (at_return) {
4304     bgtu(in_nmethod ? sp : fp, tmp_reg, slow_path, /* is_far */ true);
4305   } else {
4306     test_bit(tmp_reg, tmp_reg, exact_log2(SafepointMechanism::poll_bit()));
4307     bnez(tmp_reg, slow_path, /* is_far */ true);
4308   }
4309 }
4310 
4311 void MacroAssembler::load_reserved(Register dst,
4312                                    Register addr,
4313                                    Assembler::operand_size size,
4314                                    Assembler::Aqrl acquire) {
4315   switch (size) {
4316     case int64:
4317       lr_d(dst, addr, acquire);
4318       break;
4319     case int32:
4320       lr_w(dst, addr, acquire);
4321       break;
4322     case uint32:
4323       lr_w(dst, addr, acquire);
4324       zext(dst, dst, 32);
4325       break;
4326     default:
4327       ShouldNotReachHere();
4328   }
4329 }
4330 
4331 void MacroAssembler::store_conditional(Register dst,
4332                                        Register new_val,
4333                                        Register addr,
4334                                        Assembler::operand_size size,
4335                                        Assembler::Aqrl release) {
4336   switch (size) {
4337     case int64:
4338       sc_d(dst, addr, new_val, release);
4339       break;
4340     case int32:
4341     case uint32:
4342       sc_w(dst, addr, new_val, release);
4343       break;
4344     default:
4345       ShouldNotReachHere();
4346   }
4347 }
4348 
4349 
4350 void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expected, Register new_val,
4351                                                  Assembler::operand_size size,
4352                                                  Register shift, Register mask, Register aligned_addr) {
4353   assert(size == int8 || size == int16, "unsupported operand size");
4354 
4355   andi(shift, addr, 3);
4356   slli(shift, shift, 3);
4357 
4358   andi(aligned_addr, addr, ~3);
4359 
4360   if (size == int8) {
4361     mv(mask, 0xff);
4362   } else {
4363     // size == int16 case
4364     mv(mask, -1);
4365     zext(mask, mask, 16);
4366   }
4367   sll(mask, mask, shift);
4368 
4369   sll(expected, expected, shift);
4370   andr(expected, expected, mask);
4371 
4372   sll(new_val, new_val, shift);
4373   andr(new_val, new_val, mask);
4374 }
4375 
4376 // cmpxchg_narrow_value will kill t0, t1, expected, new_val and tmps.
4377 // It's designed to implement compare and swap byte/boolean/char/short by lr.w/sc.w or amocas.w,
4378 // which are forced to work with 4-byte aligned address.
4379 void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected,
4380                                           Register new_val,
4381                                           Assembler::operand_size size,
4382                                           Assembler::Aqrl acquire, Assembler::Aqrl release,
4383                                           Register result, bool result_as_bool,
4384                                           Register tmp1, Register tmp2, Register tmp3) {
4385   assert(!(UseZacas && UseZabha), "Use amocas");
4386   assert_different_registers(addr, expected, new_val, result, tmp1, tmp2, tmp3, t0, t1);
4387 
4388   Register scratch0 = t0, aligned_addr = t1;
4389   Register shift = tmp1, mask = tmp2, scratch1 = tmp3;
4390 
4391   cmpxchg_narrow_value_helper(addr, expected, new_val, size, shift, mask, aligned_addr);
4392 
4393   Label retry, fail, done;
4394 
4395   if (UseZacas) {
4396     lw(result, aligned_addr);
4397 
4398     bind(retry); // amocas loads the current value into result
4399     notr(scratch1, mask);
4400 
4401     andr(scratch0, result, scratch1);  // scratch0 = word - cas bits
4402     orr(scratch1, expected, scratch0); // scratch1 = non-cas bits + cas bits
4403     bne(result, scratch1, fail);       // cas bits differ, cas failed
4404 
4405     // result is the same as expected, use as expected value.
4406 
4407     // scratch0 is still = word - cas bits
4408     // Or in the new value to create complete new value.
4409     orr(scratch0, scratch0, new_val);
4410 
4411     mv(scratch1, result); // save our expected value
4412     atomic_cas(result, scratch0, aligned_addr, operand_size::int32, acquire, release);
4413     bne(scratch1, result, retry);
4414   } else {
4415     notr(scratch1, mask);
4416     bind(retry);
4417 
4418     load_reserved(result, aligned_addr, operand_size::int32, acquire);
4419     andr(scratch0, result, mask);
4420     bne(scratch0, expected, fail);
4421 
4422     andr(scratch0, result, scratch1); // scratch1 is ~mask
4423     orr(scratch0, scratch0, new_val);
4424     store_conditional(scratch0, scratch0, aligned_addr, operand_size::int32, release);
4425     bnez(scratch0, retry);
4426   }
4427 
4428   if (result_as_bool) {
4429     mv(result, 1);
4430     j(done);
4431 
4432     bind(fail);
4433     mv(result, zr);
4434 
4435     bind(done);
4436   } else {
4437     bind(fail);
4438 
4439     andr(scratch0, result, mask);
4440     srl(result, scratch0, shift);
4441 
4442     if (size == int8) {
4443       sext(result, result, 8);
4444     } else {
4445       // size == int16 case
4446       sext(result, result, 16);
4447     }
4448   }
4449 }
4450 
4451 // weak_cmpxchg_narrow_value is a weak version of cmpxchg_narrow_value, to implement
4452 // the weak CAS stuff. The major difference is that it just failed when store conditional
4453 // failed.
4454 void MacroAssembler::weak_cmpxchg_narrow_value(Register addr, Register expected,
4455                                                Register new_val,
4456                                                Assembler::operand_size size,
4457                                                Assembler::Aqrl acquire, Assembler::Aqrl release,
4458                                                Register result,
4459                                                Register tmp1, Register tmp2, Register tmp3) {
4460   assert(!(UseZacas && UseZabha), "Use amocas");
4461   assert_different_registers(addr, expected, new_val, result, tmp1, tmp2, tmp3, t0, t1);
4462 
4463   Register scratch0 = t0, aligned_addr = t1;
4464   Register shift = tmp1, mask = tmp2, scratch1 = tmp3;
4465 
4466   cmpxchg_narrow_value_helper(addr, expected, new_val, size, shift, mask, aligned_addr);
4467 
4468   Label fail, done;
4469 
4470   if (UseZacas) {
4471     lw(result, aligned_addr);
4472 
4473     notr(scratch1, mask);
4474 
4475     andr(scratch0, result, scratch1);  // scratch0 = word - cas bits
4476     orr(scratch1, expected, scratch0); // scratch1 = non-cas bits + cas bits
4477     bne(result, scratch1, fail);       // cas bits differ, cas failed
4478 
4479     // result is the same as expected, use as expected value.
4480 
4481     // scratch0 is still = word - cas bits
4482     // Or in the new value to create complete new value.
4483     orr(scratch0, scratch0, new_val);
4484 
4485     mv(scratch1, result); // save our expected value
4486     atomic_cas(result, scratch0, aligned_addr, operand_size::int32, acquire, release);
4487     bne(scratch1, result, fail); // This weak, so just bail-out.
4488   } else {
4489     notr(scratch1, mask);
4490 
4491     load_reserved(result, aligned_addr, operand_size::int32, acquire);
4492     andr(scratch0, result, mask);
4493     bne(scratch0, expected, fail);
4494 
4495     andr(scratch0, result, scratch1); // scratch1 is ~mask
4496     orr(scratch0, scratch0, new_val);
4497     store_conditional(scratch0, scratch0, aligned_addr, operand_size::int32, release);
4498     bnez(scratch0, fail);
4499   }
4500 
4501   // Success
4502   mv(result, 1);
4503   j(done);
4504 
4505   // Fail
4506   bind(fail);
4507   mv(result, zr);
4508 
4509   bind(done);
4510 }
4511 
4512 void MacroAssembler::cmpxchg(Register addr, Register expected,
4513                              Register new_val,
4514                              Assembler::operand_size size,
4515                              Assembler::Aqrl acquire, Assembler::Aqrl release,
4516                              Register result, bool result_as_bool) {
4517   assert((UseZacas && UseZabha) || (size != int8 && size != int16), "unsupported operand size");
4518   assert_different_registers(addr, t0);
4519   assert_different_registers(expected, t0);
4520   assert_different_registers(new_val, t0);
4521 
4522   // NOTE:
4523   // Register _result_ may be the same register as _new_val_ or _expected_.
4524   // Hence do NOT use _result_ until after 'cas'.
4525   //
4526   // Register _expected_ may be the same register as _new_val_ and is assumed to be preserved.
4527   // Hence do NOT change _expected_ or _new_val_.
4528   //
4529   // Having _expected_ and _new_val_ being the same register is a very puzzling cas.
4530   //
4531   // TODO: Address these issues.
4532 
4533   if (UseZacas) {
4534     if (result_as_bool) {
4535       mv(t0, expected);
4536       atomic_cas(t0, new_val, addr, size, acquire, release);
4537       xorr(t0, t0, expected);
4538       seqz(result, t0);
4539     } else {
4540       mv(t0, expected);
4541       atomic_cas(t0, new_val, addr, size, acquire, release);
4542       mv(result, t0);
4543     }
4544     return;
4545   }
4546 
4547   Label retry_load, done, ne_done;
4548   bind(retry_load);
4549   load_reserved(t0, addr, size, acquire);
4550   bne(t0, expected, ne_done);
4551   store_conditional(t0, new_val, addr, size, release);
4552   bnez(t0, retry_load);
4553 
4554   // equal, succeed
4555   if (result_as_bool) {
4556     mv(result, 1);
4557   } else {
4558     mv(result, expected);
4559   }
4560   j(done);
4561 
4562   // not equal, failed
4563   bind(ne_done);
4564   if (result_as_bool) {
4565     mv(result, zr);
4566   } else {
4567     mv(result, t0);
4568   }
4569 
4570   bind(done);
4571 }
4572 
4573 void MacroAssembler::weak_cmpxchg(Register addr, Register expected,
4574                                   Register new_val,
4575                                   Assembler::operand_size size,
4576                                   Assembler::Aqrl acquire, Assembler::Aqrl release,
4577                                   Register result) {
4578   assert((UseZacas && UseZabha) || (size != int8 && size != int16), "unsupported operand size");
4579   assert_different_registers(addr, t0);
4580   assert_different_registers(expected, t0);
4581   assert_different_registers(new_val, t0);
4582 
4583   if (UseZacas) {
4584     cmpxchg(addr, expected, new_val, size, acquire, release, result, true);
4585     return;
4586   }
4587 
4588   Label fail, done;
4589   load_reserved(t0, addr, size, acquire);
4590   bne(t0, expected, fail);
4591   store_conditional(t0, new_val, addr, size, release);
4592   bnez(t0, fail);
4593 
4594   // Success
4595   mv(result, 1);
4596   j(done);
4597 
4598   // Fail
4599   bind(fail);
4600   mv(result, zr);
4601 
4602   bind(done);
4603 }
4604 
4605 #define ATOMIC_OP(NAME, AOP, ACQUIRE, RELEASE)                                              \
4606 void MacroAssembler::atomic_##NAME(Register prev, RegisterOrConstant incr, Register addr) { \
4607   prev = prev->is_valid() ? prev : zr;                                                      \
4608   if (incr.is_register()) {                                                                 \
4609     AOP(prev, addr, incr.as_register(), (Assembler::Aqrl)(ACQUIRE | RELEASE));              \
4610   } else {                                                                                  \
4611     mv(t0, incr.as_constant());                                                             \
4612     AOP(prev, addr, t0, (Assembler::Aqrl)(ACQUIRE | RELEASE));                              \
4613   }                                                                                         \
4614   return;                                                                                   \
4615 }
4616 
4617 ATOMIC_OP(add, amoadd_d, Assembler::relaxed, Assembler::relaxed)
4618 ATOMIC_OP(addw, amoadd_w, Assembler::relaxed, Assembler::relaxed)
4619 ATOMIC_OP(addal, amoadd_d, Assembler::aq, Assembler::rl)
4620 ATOMIC_OP(addalw, amoadd_w, Assembler::aq, Assembler::rl)
4621 
4622 #undef ATOMIC_OP
4623 
4624 #define ATOMIC_XCHG(OP, AOP, ACQUIRE, RELEASE)                                       \
4625 void MacroAssembler::atomic_##OP(Register prev, Register newv, Register addr) {      \
4626   prev = prev->is_valid() ? prev : zr;                                               \
4627   AOP(prev, addr, newv, (Assembler::Aqrl)(ACQUIRE | RELEASE));                       \
4628   return;                                                                            \
4629 }
4630 
4631 ATOMIC_XCHG(xchg, amoswap_d, Assembler::relaxed, Assembler::relaxed)
4632 ATOMIC_XCHG(xchgw, amoswap_w, Assembler::relaxed, Assembler::relaxed)
4633 ATOMIC_XCHG(xchgal, amoswap_d, Assembler::aq, Assembler::rl)
4634 ATOMIC_XCHG(xchgalw, amoswap_w, Assembler::aq, Assembler::rl)
4635 
4636 #undef ATOMIC_XCHG
4637 
4638 #define ATOMIC_XCHGU(OP1, OP2)                                                       \
4639 void MacroAssembler::atomic_##OP1(Register prev, Register newv, Register addr) {     \
4640   atomic_##OP2(prev, newv, addr);                                                    \
4641   zext(prev, prev, 32);                                                       \
4642   return;                                                                            \
4643 }
4644 
4645 ATOMIC_XCHGU(xchgwu, xchgw)
4646 ATOMIC_XCHGU(xchgalwu, xchgalw)
4647 
4648 #undef ATOMIC_XCHGU
4649 
4650 void MacroAssembler::atomic_cas(Register prev, Register newv, Register addr,
4651                                 Assembler::operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release) {
4652   switch (size) {
4653     case int64:
4654       amocas_d(prev, addr, newv, (Assembler::Aqrl)(acquire | release));
4655       break;
4656     case int32:
4657       amocas_w(prev, addr, newv, (Assembler::Aqrl)(acquire | release));
4658       break;
4659     case uint32:
4660       amocas_w(prev, addr, newv, (Assembler::Aqrl)(acquire | release));
4661       zext(prev, prev, 32);
4662       break;
4663     case int16:
4664       amocas_h(prev, addr, newv, (Assembler::Aqrl)(acquire | release));
4665       break;
4666     case int8:
4667       amocas_b(prev, addr, newv, (Assembler::Aqrl)(acquire | release));
4668       break;
4669     default:
4670       ShouldNotReachHere();
4671   }
4672 }
4673 
4674 void MacroAssembler::far_jump(const Address &entry, Register tmp) {
4675   assert(CodeCache::contains(entry.target()),
4676          "destination of far jump not found in code cache");
4677   assert(entry.rspec().type() == relocInfo::external_word_type
4678         || entry.rspec().type() == relocInfo::runtime_call_type
4679         || entry.rspec().type() == relocInfo::none, "wrong entry relocInfo type");
4680   // Fixed length: see MacroAssembler::far_branch_size()
4681   // We can use auipc + jr here because we know that the total size of
4682   // the code cache cannot exceed 2Gb.
4683   relocate(entry.rspec(), [&] {
4684     int64_t distance = entry.target() - pc();
4685     int32_t offset = ((int32_t)distance << 20) >> 20;
4686     assert(is_valid_32bit_offset(distance), "Far jump using wrong instructions.");
4687     auipc(tmp, (int32_t)distance + 0x800);
4688     jr(tmp, offset);
4689   });
4690 }
4691 
4692 void MacroAssembler::far_call(const Address &entry, Register tmp) {
4693   assert(tmp != x5, "tmp register must not be x5.");
4694   assert(CodeCache::contains(entry.target()),
4695          "destination of far call not found in code cache");
4696   assert(entry.rspec().type() == relocInfo::external_word_type
4697         || entry.rspec().type() == relocInfo::runtime_call_type
4698         || entry.rspec().type() == relocInfo::none, "wrong entry relocInfo type");
4699   // Fixed length: see MacroAssembler::far_branch_size()
4700   // We can use auipc + jalr here because we know that the total size of
4701   // the code cache cannot exceed 2Gb.
4702   relocate(entry.rspec(), [&] {
4703     int64_t distance = entry.target() - pc();
4704     int32_t offset = ((int32_t)distance << 20) >> 20;
4705     assert(is_valid_32bit_offset(distance), "Far call using wrong instructions.");
4706     auipc(tmp, (int32_t)distance + 0x800);
4707     jalr(tmp, offset);
4708   });
4709 }
4710 
4711 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
4712                                                    Register super_klass,
4713                                                    Register tmp_reg,
4714                                                    Label* L_success,
4715                                                    Label* L_failure,
4716                                                    Label* L_slow_path,
4717                                                    Register super_check_offset) {
4718   assert_different_registers(sub_klass, super_klass, tmp_reg, super_check_offset);
4719   bool must_load_sco = !super_check_offset->is_valid();
4720   if (must_load_sco) {
4721     assert(tmp_reg != noreg, "supply either a temp or a register offset");
4722   }
4723 
4724   Label L_fallthrough;
4725   int label_nulls = 0;
4726   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
4727   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
4728   if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; }
4729   assert(label_nulls <= 1, "at most one null in batch");
4730 
4731   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
4732   int sco_offset = in_bytes(Klass::super_check_offset_offset());
4733   Address super_check_offset_addr(super_klass, sco_offset);
4734 
4735   // Hacked jmp, which may only be used just before L_fallthrough.
4736 #define final_jmp(label)                                                \
4737   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
4738   else                            j(label)             /*omit semi*/
4739 
4740   // If the pointers are equal, we are done (e.g., String[] elements).
4741   // This self-check enables sharing of secondary supertype arrays among
4742   // non-primary types such as array-of-interface. Otherwise, each such
4743   // type would need its own customized SSA.
4744   // We move this check to the front of the fast path because many
4745   // type checks are in fact trivially successful in this manner,
4746   // so we get a nicely predicted branch right at the start of the check.
4747   beq(sub_klass, super_klass, *L_success);
4748 
4749   // Check the supertype display:
4750   if (must_load_sco) {
4751     lwu(tmp_reg, super_check_offset_addr);
4752     super_check_offset = tmp_reg;
4753   }
4754   add(t0, sub_klass, super_check_offset);
4755   Address super_check_addr(t0);
4756   ld(t0, super_check_addr); // load displayed supertype
4757   beq(super_klass, t0, *L_success);
4758 
4759   // This check has worked decisively for primary supers.
4760   // Secondary supers are sought in the super_cache ('super_cache_addr').
4761   // (Secondary supers are interfaces and very deeply nested subtypes.)
4762   // This works in the same check above because of a tricky aliasing
4763   // between the super_Cache and the primary super display elements.
4764   // (The 'super_check_addr' can address either, as the case requires.)
4765   // Note that the cache is updated below if it does not help us find
4766   // what we need immediately.
4767   // So if it was a primary super, we can just fail immediately.
4768   // Otherwise, it's the slow path for us (no success at this point).
4769 
4770   mv(t1, sc_offset);
4771   if (L_failure == &L_fallthrough) {
4772     beq(super_check_offset, t1, *L_slow_path);
4773   } else {
4774     bne(super_check_offset, t1, *L_failure, /* is_far */ true);
4775     final_jmp(*L_slow_path);
4776   }
4777 
4778   bind(L_fallthrough);
4779 
4780 #undef final_jmp
4781 }
4782 
4783 // Scans count pointer sized words at [addr] for occurrence of value,
4784 // generic
4785 void MacroAssembler::repne_scan(Register addr, Register value, Register count,
4786                                 Register tmp) {
4787   Label Lloop, Lexit;
4788   beqz(count, Lexit);
4789   bind(Lloop);
4790   ld(tmp, addr);
4791   beq(value, tmp, Lexit);
4792   addi(addr, addr, wordSize);
4793   subi(count, count, 1);
4794   bnez(count, Lloop);
4795   bind(Lexit);
4796 }
4797 
4798 void MacroAssembler::check_klass_subtype_slow_path_linear(Register sub_klass,
4799                                                           Register super_klass,
4800                                                           Register tmp1_reg,
4801                                                           Register tmp2_reg,
4802                                                           Label* L_success,
4803                                                           Label* L_failure,
4804                                                           bool set_cond_codes) {
4805   assert_different_registers(sub_klass, super_klass, tmp1_reg);
4806   if (tmp2_reg != noreg) {
4807     assert_different_registers(sub_klass, super_klass, tmp1_reg, tmp2_reg, t0);
4808   }
4809 #define IS_A_TEMP(reg) ((reg) == tmp1_reg || (reg) == tmp2_reg)
4810 
4811   Label L_fallthrough;
4812   int label_nulls = 0;
4813   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
4814   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
4815 
4816   assert(label_nulls <= 1, "at most one null in the batch");
4817 
4818   // A couple of useful fields in sub_klass:
4819   int ss_offset = in_bytes(Klass::secondary_supers_offset());
4820   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
4821   Address secondary_supers_addr(sub_klass, ss_offset);
4822   Address super_cache_addr(     sub_klass, sc_offset);
4823 
4824   BLOCK_COMMENT("check_klass_subtype_slow_path");
4825 
4826   // Do a linear scan of the secondary super-klass chain.
4827   // This code is rarely used, so simplicity is a virtue here.
4828   // The repne_scan instruction uses fixed registers, which we must spill.
4829   // Don't worry too much about pre-existing connections with the input regs.
4830 
4831   assert(sub_klass != x10, "killed reg"); // killed by mv(x10, super)
4832   assert(sub_klass != x12, "killed reg"); // killed by la(x12, &pst_counter)
4833 
4834   RegSet pushed_registers;
4835   if (!IS_A_TEMP(x12)) {
4836     pushed_registers += x12;
4837   }
4838   if (!IS_A_TEMP(x15)) {
4839     pushed_registers += x15;
4840   }
4841 
4842   if (super_klass != x10) {
4843     if (!IS_A_TEMP(x10)) {
4844       pushed_registers += x10;
4845     }
4846   }
4847 
4848   push_reg(pushed_registers, sp);
4849 
4850   // Get super_klass value into x10 (even if it was in x15 or x12)
4851   mv(x10, super_klass);
4852 
4853 #ifndef PRODUCT
4854   incrementw(ExternalAddress((address)&SharedRuntime::_partial_subtype_ctr));
4855 #endif // PRODUCT
4856 
4857   // We will consult the secondary-super array.
4858   ld(x15, secondary_supers_addr);
4859   // Load the array length.
4860   lwu(x12, Address(x15, Array<Klass*>::length_offset_in_bytes()));
4861   // Skip to start of data.
4862   addi(x15, x15, Array<Klass*>::base_offset_in_bytes());
4863 
4864   // Set t0 to an obvious invalid value, falling through by default
4865   mv(t0, -1);
4866   // Scan X12 words at [X15] for an occurrence of X10.
4867   repne_scan(x15, x10, x12, t0);
4868 
4869   // pop will restore x10, so we should use a temp register to keep its value
4870   mv(t1, x10);
4871 
4872   // Unspill the temp registers:
4873   pop_reg(pushed_registers, sp);
4874 
4875   bne(t1, t0, *L_failure);
4876 
4877   // Success. Cache the super we found an proceed in triumph.
4878   if (UseSecondarySupersCache) {
4879     sd(super_klass, super_cache_addr);
4880   }
4881 
4882   if (L_success != &L_fallthrough) {
4883     j(*L_success);
4884   }
4885 
4886 #undef IS_A_TEMP
4887 
4888   bind(L_fallthrough);
4889 }
4890 
4891 // population_count variant for running without the CPOP
4892 // instruction, which was introduced with Zbb extension.
4893 void MacroAssembler::population_count(Register dst, Register src,
4894                                       Register tmp1, Register tmp2) {
4895   if (UsePopCountInstruction) {
4896     cpop(dst, src);
4897   } else {
4898     assert_different_registers(src, tmp1, tmp2);
4899     assert_different_registers(dst, tmp1, tmp2);
4900     Label loop, done;
4901 
4902     mv(tmp1, src);
4903     // dst = 0;
4904     // while(tmp1 != 0) {
4905     //   dst++;
4906     //   tmp1 &= (tmp1 - 1);
4907     // }
4908     mv(dst, zr);
4909     beqz(tmp1, done);
4910     {
4911       bind(loop);
4912       addi(dst, dst, 1);
4913       subi(tmp2, tmp1, 1);
4914       andr(tmp1, tmp1, tmp2);
4915       bnez(tmp1, loop);
4916     }
4917     bind(done);
4918   }
4919 }
4920 
4921 // If Register r is invalid, remove a new register from
4922 // available_regs, and add new register to regs_to_push.
4923 Register MacroAssembler::allocate_if_noreg(Register r,
4924                                   RegSetIterator<Register> &available_regs,
4925                                   RegSet &regs_to_push) {
4926   if (!r->is_valid()) {
4927     r = *available_regs++;
4928     regs_to_push += r;
4929   }
4930   return r;
4931 }
4932 
4933 // check_klass_subtype_slow_path_table() looks for super_klass in the
4934 // hash table belonging to super_klass, branching to L_success or
4935 // L_failure as appropriate. This is essentially a shim which
4936 // allocates registers as necessary then calls
4937 // lookup_secondary_supers_table() to do the work. Any of the tmp
4938 // regs may be noreg, in which case this logic will chooses some
4939 // registers push and pop them from the stack.
4940 void MacroAssembler::check_klass_subtype_slow_path_table(Register sub_klass,
4941                                                          Register super_klass,
4942                                                          Register tmp1_reg,
4943                                                          Register tmp2_reg,
4944                                                          Label* L_success,
4945                                                          Label* L_failure,
4946                                                          bool set_cond_codes) {
4947   RegSet tmps = RegSet::of(tmp1_reg, tmp2_reg);
4948 
4949   assert_different_registers(sub_klass, super_klass, tmp1_reg, tmp2_reg);
4950 
4951   Label L_fallthrough;
4952   int label_nulls = 0;
4953   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
4954   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
4955   assert(label_nulls <= 1, "at most one null in the batch");
4956 
4957   BLOCK_COMMENT("check_klass_subtype_slow_path");
4958 
4959   RegSet caller_save_regs = RegSet::of(x7) + RegSet::range(x10, x17) + RegSet::range(x28, x31);
4960   RegSetIterator<Register> available_regs = (caller_save_regs - tmps - sub_klass - super_klass).begin();
4961 
4962   RegSet pushed_regs;
4963 
4964   tmp1_reg = allocate_if_noreg(tmp1_reg, available_regs, pushed_regs);
4965   tmp2_reg = allocate_if_noreg(tmp2_reg, available_regs, pushed_regs);
4966 
4967   Register tmp3_reg = noreg, tmp4_reg = noreg, result_reg = noreg;
4968 
4969   tmp3_reg = allocate_if_noreg(tmp3_reg, available_regs, pushed_regs);
4970   tmp4_reg = allocate_if_noreg(tmp4_reg, available_regs, pushed_regs);
4971   result_reg = allocate_if_noreg(result_reg, available_regs, pushed_regs);
4972 
4973   push_reg(pushed_regs, sp);
4974 
4975   lookup_secondary_supers_table_var(sub_klass,
4976                                     super_klass,
4977                                     result_reg,
4978                                     tmp1_reg, tmp2_reg, tmp3_reg,
4979                                     tmp4_reg, nullptr);
4980 
4981   // Move the result to t1 as we are about to unspill the tmp registers.
4982   mv(t1, result_reg);
4983 
4984   // Unspill the tmp. registers:
4985   pop_reg(pushed_regs, sp);
4986 
4987   // NB! Callers may assume that, when set_cond_codes is true, this
4988   // code sets tmp2_reg to a nonzero value.
4989   if (set_cond_codes) {
4990     mv(tmp2_reg, 1);
4991   }
4992 
4993   bnez(t1, *L_failure);
4994 
4995   if (L_success != &L_fallthrough) {
4996     j(*L_success);
4997   }
4998 
4999   bind(L_fallthrough);
5000 }
5001 
5002 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
5003                                                    Register super_klass,
5004                                                    Register tmp1_reg,
5005                                                    Register tmp2_reg,
5006                                                    Label* L_success,
5007                                                    Label* L_failure,
5008                                                    bool set_cond_codes) {
5009   if (UseSecondarySupersTable) {
5010     check_klass_subtype_slow_path_table
5011       (sub_klass, super_klass, tmp1_reg, tmp2_reg, L_success, L_failure, set_cond_codes);
5012   } else {
5013     check_klass_subtype_slow_path_linear
5014       (sub_klass, super_klass, tmp1_reg, tmp2_reg, L_success, L_failure, set_cond_codes);
5015   }
5016 }
5017 
5018 // Ensure that the inline code and the stub are using the same registers
5019 // as we need to call the stub from inline code when there is a collision
5020 // in the hashed lookup in the secondary supers array.
5021 #define LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS(r_super_klass, r_array_base, r_array_length,  \
5022                                                 r_array_index, r_sub_klass, result, r_bitmap) \
5023 do {                                                                                          \
5024   assert(r_super_klass  == x10                             &&                                 \
5025          r_array_base   == x11                             &&                                 \
5026          r_array_length == x12                             &&                                 \
5027          (r_array_index == x13  || r_array_index == noreg) &&                                 \
5028          (r_sub_klass   == x14  || r_sub_klass   == noreg) &&                                 \
5029          (result        == x15  || result        == noreg) &&                                 \
5030          (r_bitmap      == x16  || r_bitmap      == noreg), "registers must match riscv.ad"); \
5031 } while(0)
5032 
5033 bool MacroAssembler::lookup_secondary_supers_table_const(Register r_sub_klass,
5034                                                          Register r_super_klass,
5035                                                          Register result,
5036                                                          Register tmp1,
5037                                                          Register tmp2,
5038                                                          Register tmp3,
5039                                                          Register tmp4,
5040                                                          u1 super_klass_slot,
5041                                                          bool stub_is_near) {
5042   assert_different_registers(r_sub_klass, r_super_klass, result, tmp1, tmp2, tmp3, tmp4, t0, t1);
5043 
5044   Label L_fallthrough;
5045 
5046   BLOCK_COMMENT("lookup_secondary_supers_table {");
5047 
5048   const Register
5049     r_array_base   = tmp1, // x11
5050     r_array_length = tmp2, // x12
5051     r_array_index  = tmp3, // x13
5052     r_bitmap       = tmp4; // x16
5053 
5054   LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS(r_super_klass, r_array_base, r_array_length,
5055                                           r_array_index, r_sub_klass, result, r_bitmap);
5056 
5057   u1 bit = super_klass_slot;
5058 
5059   // Initialize result value to 1 which means mismatch.
5060   mv(result, 1);
5061 
5062   ld(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset()));
5063 
5064   // First check the bitmap to see if super_klass might be present. If
5065   // the bit is zero, we are certain that super_klass is not one of
5066   // the secondary supers.
5067   test_bit(t0, r_bitmap, bit);
5068   beqz(t0, L_fallthrough);
5069 
5070   // Get the first array index that can contain super_klass into r_array_index.
5071   if (bit != 0) {
5072     slli(r_array_index, r_bitmap, (Klass::SECONDARY_SUPERS_TABLE_MASK - bit));
5073     population_count(r_array_index, r_array_index, tmp1, tmp2);
5074   } else {
5075     mv(r_array_index, (u1)1);
5076   }
5077 
5078   // We will consult the secondary-super array.
5079   ld(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
5080 
5081   // The value i in r_array_index is >= 1, so even though r_array_base
5082   // points to the length, we don't need to adjust it to point to the data.
5083   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code");
5084   assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code");
5085 
5086   shadd(result, r_array_index, r_array_base, result, LogBytesPerWord);
5087   ld(result, Address(result));
5088   xorr(result, result, r_super_klass);
5089   beqz(result, L_fallthrough); // Found a match
5090 
5091   // Is there another entry to check? Consult the bitmap.
5092   test_bit(t0, r_bitmap, (bit + 1) & Klass::SECONDARY_SUPERS_TABLE_MASK);
5093   beqz(t0, L_fallthrough);
5094 
5095   // Linear probe.
5096   if (bit != 0) {
5097     ror(r_bitmap, r_bitmap, bit);
5098   }
5099 
5100   // The slot we just inspected is at secondary_supers[r_array_index - 1].
5101   // The next slot to be inspected, by the stub we're about to call,
5102   // is secondary_supers[r_array_index]. Bits 0 and 1 in the bitmap
5103   // have been checked.
5104   rt_call(StubRoutines::lookup_secondary_supers_table_slow_path_stub());
5105 
5106   BLOCK_COMMENT("} lookup_secondary_supers_table");
5107 
5108   bind(L_fallthrough);
5109 
5110   if (VerifySecondarySupers) {
5111     verify_secondary_supers_table(r_sub_klass, r_super_klass, // x14, x10
5112                                   result, tmp1, tmp2, tmp3);  // x15, x11, x12, x13
5113   }
5114   return true;
5115 }
5116 
5117 // At runtime, return 0 in result if r_super_klass is a superclass of
5118 // r_sub_klass, otherwise return nonzero. Use this version of
5119 // lookup_secondary_supers_table() if you don't know ahead of time
5120 // which superclass will be searched for. Used by interpreter and
5121 // runtime stubs. It is larger and has somewhat greater latency than
5122 // the version above, which takes a constant super_klass_slot.
5123 void MacroAssembler::lookup_secondary_supers_table_var(Register r_sub_klass,
5124                                                        Register r_super_klass,
5125                                                        Register result,
5126                                                        Register tmp1,
5127                                                        Register tmp2,
5128                                                        Register tmp3,
5129                                                        Register tmp4,
5130                                                        Label *L_success) {
5131   assert_different_registers(r_sub_klass, r_super_klass, result, tmp1, tmp2, tmp3, tmp4, t0, t1);
5132 
5133   Label L_fallthrough;
5134 
5135   BLOCK_COMMENT("lookup_secondary_supers_table {");
5136 
5137   const Register
5138     r_array_index = tmp3,
5139     r_bitmap      = tmp4,
5140     slot          = t1;
5141 
5142   lbu(slot, Address(r_super_klass, Klass::hash_slot_offset()));
5143 
5144   // Make sure that result is nonzero if the test below misses.
5145   mv(result, 1);
5146 
5147   ld(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset()));
5148 
5149   // First check the bitmap to see if super_klass might be present. If
5150   // the bit is zero, we are certain that super_klass is not one of
5151   // the secondary supers.
5152 
5153   // This next instruction is equivalent to:
5154   // mv(tmp_reg, (u1)(Klass::SECONDARY_SUPERS_TABLE_SIZE - 1));
5155   // sub(r_array_index, slot, tmp_reg);
5156   xori(r_array_index, slot, (u1)(Klass::SECONDARY_SUPERS_TABLE_SIZE - 1));
5157   sll(r_array_index, r_bitmap, r_array_index);
5158   test_bit(t0, r_array_index, Klass::SECONDARY_SUPERS_TABLE_SIZE - 1);
5159   beqz(t0, L_fallthrough);
5160 
5161   // Get the first array index that can contain super_klass into r_array_index.
5162   population_count(r_array_index, r_array_index, tmp1, tmp2);
5163 
5164   // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word.
5165 
5166   const Register
5167     r_array_base   = tmp1,
5168     r_array_length = tmp2;
5169 
5170   // The value i in r_array_index is >= 1, so even though r_array_base
5171   // points to the length, we don't need to adjust it to point to the data.
5172   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code");
5173   assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code");
5174 
5175   // We will consult the secondary-super array.
5176   ld(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
5177 
5178   shadd(result, r_array_index, r_array_base, result, LogBytesPerWord);
5179   ld(result, Address(result));
5180   xorr(result, result, r_super_klass);
5181   beqz(result, L_success ? *L_success : L_fallthrough); // Found a match
5182 
5183   // Is there another entry to check? Consult the bitmap.
5184   ror(r_bitmap, r_bitmap, slot);
5185   test_bit(t0, r_bitmap, 1);
5186   beqz(t0, L_fallthrough);
5187 
5188   // The slot we just inspected is at secondary_supers[r_array_index - 1].
5189   // The next slot to be inspected, by the logic we're about to call,
5190   // is secondary_supers[r_array_index]. Bits 0 and 1 in the bitmap
5191   // have been checked.
5192   lookup_secondary_supers_table_slow_path(r_super_klass, r_array_base, r_array_index,
5193                                           r_bitmap, result, r_array_length, false /*is_stub*/);
5194 
5195   BLOCK_COMMENT("} lookup_secondary_supers_table");
5196 
5197   bind(L_fallthrough);
5198 
5199   if (VerifySecondarySupers) {
5200     verify_secondary_supers_table(r_sub_klass, r_super_klass,
5201                                   result, tmp1, tmp2, tmp3);
5202   }
5203 
5204   if (L_success) {
5205     beqz(result, *L_success);
5206   }
5207 }
5208 
5209 // Called by code generated by check_klass_subtype_slow_path
5210 // above. This is called when there is a collision in the hashed
5211 // lookup in the secondary supers array.
5212 void MacroAssembler::lookup_secondary_supers_table_slow_path(Register r_super_klass,
5213                                                              Register r_array_base,
5214                                                              Register r_array_index,
5215                                                              Register r_bitmap,
5216                                                              Register result,
5217                                                              Register tmp,
5218                                                              bool is_stub) {
5219   assert_different_registers(r_super_klass, r_array_base, r_array_index, r_bitmap, tmp, result, t0);
5220 
5221   const Register
5222     r_array_length = tmp,
5223     r_sub_klass    = noreg; // unused
5224 
5225   if (is_stub) {
5226     LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS(r_super_klass, r_array_base, r_array_length,
5227                                             r_array_index, r_sub_klass, result, r_bitmap);
5228   }
5229 
5230   Label L_matched, L_fallthrough, L_bitmap_full;
5231 
5232   // Initialize result value to 1 which means mismatch.
5233   mv(result, 1);
5234 
5235   // Load the array length.
5236   lwu(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes()));
5237   // And adjust the array base to point to the data.
5238   // NB! Effectively increments current slot index by 1.
5239   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "");
5240   addi(r_array_base, r_array_base, Array<Klass*>::base_offset_in_bytes());
5241 
5242   // Check if bitmap is SECONDARY_SUPERS_BITMAP_FULL
5243   assert(Klass::SECONDARY_SUPERS_BITMAP_FULL == ~uintx(0), "Adjust this code");
5244   subw(t0, r_array_length, Klass::SECONDARY_SUPERS_TABLE_SIZE - 2);
5245   bgtz(t0, L_bitmap_full);
5246 
5247   // NB! Our caller has checked bits 0 and 1 in the bitmap. The
5248   // current slot (at secondary_supers[r_array_index]) has not yet
5249   // been inspected, and r_array_index may be out of bounds if we
5250   // wrapped around the end of the array.
5251 
5252   { // This is conventional linear probing, but instead of terminating
5253     // when a null entry is found in the table, we maintain a bitmap
5254     // in which a 0 indicates missing entries.
5255     // As long as the bitmap is not completely full,
5256     // array_length == popcount(bitmap). The array_length check above
5257     // guarantees there are 0s in the bitmap, so the loop eventually
5258     // terminates.
5259     Label L_loop;
5260     bind(L_loop);
5261 
5262     // Check for wraparound.
5263     Label skip;
5264     blt(r_array_index, r_array_length, skip);
5265     mv(r_array_index, zr);
5266     bind(skip);
5267 
5268     shadd(t0, r_array_index, r_array_base, t0, LogBytesPerWord);
5269     ld(t0, Address(t0));
5270     beq(t0, r_super_klass, L_matched);
5271 
5272     test_bit(t0, r_bitmap, 2);  // look-ahead check (Bit 2); result is non-zero
5273     beqz(t0, L_fallthrough);
5274 
5275     ror(r_bitmap, r_bitmap, 1);
5276     addi(r_array_index, r_array_index, 1);
5277     j(L_loop);
5278   }
5279 
5280   { // Degenerate case: more than 64 secondary supers.
5281     // FIXME: We could do something smarter here, maybe a vectorized
5282     // comparison or a binary search, but is that worth any added
5283     // complexity?
5284     bind(L_bitmap_full);
5285     repne_scan(r_array_base, r_super_klass, r_array_length, t0);
5286     bne(r_super_klass, t0, L_fallthrough);
5287   }
5288 
5289   bind(L_matched);
5290   mv(result, zr);
5291 
5292   bind(L_fallthrough);
5293 }
5294 
5295 // Make sure that the hashed lookup and a linear scan agree.
5296 void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass,
5297                                                    Register r_super_klass,
5298                                                    Register result,
5299                                                    Register tmp1,
5300                                                    Register tmp2,
5301                                                    Register tmp3) {
5302   assert_different_registers(r_sub_klass, r_super_klass, tmp1, tmp2, tmp3, result, t0, t1);
5303 
5304   const Register
5305     r_array_base   = tmp1,  // X11
5306     r_array_length = tmp2,  // X12
5307     r_array_index  = noreg, // unused
5308     r_bitmap       = noreg; // unused
5309 
5310   BLOCK_COMMENT("verify_secondary_supers_table {");
5311 
5312   // We will consult the secondary-super array.
5313   ld(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
5314 
5315   // Load the array length.
5316   lwu(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes()));
5317   // And adjust the array base to point to the data.
5318   addi(r_array_base, r_array_base, Array<Klass*>::base_offset_in_bytes());
5319 
5320   repne_scan(r_array_base, r_super_klass, r_array_length, t0);
5321   Label failed;
5322   mv(tmp3, 1);
5323   bne(r_super_klass, t0, failed);
5324   mv(tmp3, zr);
5325   bind(failed);
5326 
5327   snez(result, result); // normalize result to 0/1 for comparison
5328 
5329   Label passed;
5330   beq(tmp3, result, passed);
5331   {
5332     mv(x10, r_super_klass);
5333     mv(x11, r_sub_klass);
5334     mv(x12, tmp3);
5335     mv(x13, result);
5336     mv(x14, (address)("mismatch"));
5337     rt_call(CAST_FROM_FN_PTR(address, Klass::on_secondary_supers_verification_failure));
5338     should_not_reach_here();
5339   }
5340   bind(passed);
5341 
5342   BLOCK_COMMENT("} verify_secondary_supers_table");
5343 }
5344 
5345 // Defines obj, preserves var_size_in_bytes, okay for tmp2 == var_size_in_bytes.
5346 void MacroAssembler::tlab_allocate(Register obj,
5347                                    Register var_size_in_bytes,
5348                                    int con_size_in_bytes,
5349                                    Register tmp1,
5350                                    Register tmp2,
5351                                    Label& slow_case,
5352                                    bool is_far) {
5353   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
5354   bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, tmp1, tmp2, slow_case, is_far);
5355 }
5356 
5357 // get_thread() can be called anywhere inside generated code so we
5358 // need to save whatever non-callee save context might get clobbered
5359 // by the call to Thread::current() or, indeed, the call setup code.
5360 void MacroAssembler::get_thread(Register thread) {
5361   // save all call-clobbered regs except thread
5362   RegSet saved_regs = RegSet::range(x5, x7) + RegSet::range(x10, x17) +
5363                       RegSet::range(x28, x31) + ra - thread;
5364   push_reg(saved_regs, sp);
5365 
5366   mv(t1, CAST_FROM_FN_PTR(address, Thread::current));
5367   jalr(t1);
5368   if (thread != c_rarg0) {
5369     mv(thread, c_rarg0);
5370   }
5371 
5372   // restore pushed registers
5373   pop_reg(saved_regs, sp);
5374 }
5375 
5376 void MacroAssembler::load_byte_map_base(Register reg) {
5377   CardTableBarrierSet* ctbs = CardTableBarrierSet::barrier_set();
5378   // Strictly speaking the card table base isn't an address at all, and it might
5379   // even be negative. It is thus materialised as a constant.
5380   mv(reg, (uint64_t)ctbs->card_table_base_const());
5381 }
5382 
5383 void MacroAssembler::build_frame(int framesize) {
5384   assert(framesize >= 2, "framesize must include space for FP/RA");
5385   assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment");
5386   sub(sp, sp, framesize);
5387   sd(fp, Address(sp, framesize - 2 * wordSize));
5388   sd(ra, Address(sp, framesize - wordSize));
5389   if (PreserveFramePointer) { add(fp, sp, framesize); }
5390 }
5391 
5392 void MacroAssembler::remove_frame(int framesize) {
5393   assert(framesize >= 2, "framesize must include space for FP/RA");
5394   assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment");
5395   ld(fp, Address(sp, framesize - 2 * wordSize));
5396   ld(ra, Address(sp, framesize - wordSize));
5397   add(sp, sp, framesize);
5398 }
5399 
5400 void MacroAssembler::remove_frame(int initial_framesize, bool needs_stack_repair) {
5401   assert(!needs_stack_repair, "unimplemented");
5402   remove_frame(initial_framesize);
5403 }
5404 
5405 #ifdef COMPILER2
5406 // C2 compiled method's prolog code
5407 // Moved here from riscv.ad to support Valhalla code belows
5408 void MacroAssembler::verified_entry(Compile* C, int sp_inc) {
5409   if (C->clinit_barrier_on_entry()) {
5410     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
5411 
5412     Label L_skip_barrier;
5413 
5414     mov_metadata(t1, C->method()->holder()->constant_encoding());
5415     clinit_barrier(t1, t0, &L_skip_barrier);
5416     far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
5417     bind(L_skip_barrier);
5418   }
5419 
5420   int bangsize = C->output()->bang_size_in_bytes();
5421   if (C->output()->need_stack_bang(bangsize)) {
5422     generate_stack_overflow_check(bangsize);
5423   }
5424 
5425   // n.b. frame size includes space for return pc and fp
5426   const long framesize = C->output()->frame_size_in_bytes();
5427   build_frame(framesize);
5428 
5429   assert(!C->needs_stack_repair(), "unimplemented");
5430 }
5431 #endif // COMPILER2
5432 
5433 // Move a value between registers/stack slots and update the reg_state
5434 bool MacroAssembler::move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]) {
5435   Unimplemented();
5436   return false;
5437 }
5438 
5439 // Read all fields from an inline type oop and store the values in registers/stack slots
5440 bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index,
5441                                           VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index,
5442                                           RegState reg_state[]) {
5443 
5444   Unimplemented();
5445   return false;
5446 }
5447 
5448 // Pack fields back into an inline type oop
5449 bool MacroAssembler::pack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, int vtarg_index,
5450                                         VMRegPair* from, int from_count, int& from_index, VMReg to,
5451                                         RegState reg_state[], Register val_array) {
5452   Unimplemented();
5453   return false;
5454 }
5455 
5456 // Calculate the extra stack space required for packing or unpacking inline
5457 // args and adjust the stack pointer
5458 int MacroAssembler::extend_stack_for_inline_args(int args_on_stack) {
5459   Unimplemented();
5460   return false;
5461 }
5462 
5463 VMReg MacroAssembler::spill_reg_for(VMReg reg) {
5464   Unimplemented();
5465   return reg;
5466 }
5467 
5468 void MacroAssembler::reserved_stack_check() {
5469   // testing if reserved zone needs to be enabled
5470   Label no_reserved_zone_enabling;
5471 
5472   ld(t0, Address(xthread, JavaThread::reserved_stack_activation_offset()));
5473   bltu(sp, t0, no_reserved_zone_enabling);
5474 
5475   enter();   // RA and FP are live.
5476   mv(c_rarg0, xthread);
5477   rt_call(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone));
5478   leave();
5479 
5480   // We have already removed our own frame.
5481   // throw_delayed_StackOverflowError will think that it's been
5482   // called by our caller.
5483   j(RuntimeAddress(SharedRuntime::throw_delayed_StackOverflowError_entry()));
5484   should_not_reach_here();
5485 
5486   bind(no_reserved_zone_enabling);
5487 }
5488 
5489 // Move the address of the polling page into dest.
5490 void MacroAssembler::get_polling_page(Register dest, relocInfo::relocType rtype) {
5491   ld(dest, Address(xthread, JavaThread::polling_page_offset()));
5492 }
5493 
5494 // Read the polling page.  The address of the polling page must
5495 // already be in r.
5496 void MacroAssembler::read_polling_page(Register r, int32_t offset, relocInfo::relocType rtype) {
5497   relocate(rtype, [&] {
5498     lwu(zr, Address(r, offset));
5499   });
5500 }
5501 
5502 void MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
5503 #ifdef ASSERT
5504   {
5505     ThreadInVMfromUnknown tiv;
5506     assert (UseCompressedOops, "should only be used for compressed oops");
5507     assert (Universe::heap() != nullptr, "java heap should be initialized");
5508     assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
5509     assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "should be real oop");
5510   }
5511 #endif
5512   int oop_index = oop_recorder()->find_index(obj);
5513   relocate(oop_Relocation::spec(oop_index), [&] {
5514     li32(dst, 0xDEADBEEF);
5515   });
5516   zext(dst, dst, 32);
5517 }
5518 
5519 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
5520   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
5521   int index = oop_recorder()->find_index(k);
5522 
5523   narrowKlass nk = CompressedKlassPointers::encode(k);
5524   relocate(metadata_Relocation::spec(index), [&] {
5525     li32(dst, nk);
5526   });
5527   zext(dst, dst, 32);
5528 }
5529 
5530 address MacroAssembler::reloc_call(Address entry, Register tmp) {
5531   assert(entry.rspec().type() == relocInfo::runtime_call_type ||
5532          entry.rspec().type() == relocInfo::opt_virtual_call_type ||
5533          entry.rspec().type() == relocInfo::static_call_type ||
5534          entry.rspec().type() == relocInfo::virtual_call_type, "wrong reloc type");
5535 
5536   address target = entry.target();
5537 
5538   if (!in_scratch_emit_size()) {
5539     address stub = emit_reloc_call_address_stub(offset(), target);
5540     if (stub == nullptr) {
5541       postcond(pc() == badAddress);
5542       return nullptr; // CodeCache is full
5543     }
5544   }
5545 
5546   address call_pc = pc();
5547 #ifdef ASSERT
5548   if (entry.rspec().type() != relocInfo::runtime_call_type) {
5549     assert_alignment(call_pc);
5550   }
5551 #endif
5552 
5553   // The relocation created while emitting the stub will ensure this
5554   // call instruction is subsequently patched to call the stub.
5555   relocate(entry.rspec(), [&] {
5556     auipc(tmp, 0);
5557     ld(tmp, Address(tmp, 0));
5558     jalr(tmp);
5559   });
5560 
5561   postcond(pc() != badAddress);
5562   return call_pc;
5563 }
5564 
5565 address MacroAssembler::ic_call(address entry, jint method_index) {
5566   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
5567   assert(!in_compressible_scope(), "Must be");
5568   movptr(t0, (address)Universe::non_oop_word(), t1);
5569   assert_cond(entry != nullptr);
5570   return reloc_call(Address(entry, rh));
5571 }
5572 
5573 int MacroAssembler::ic_check_size() {
5574   // No compressed
5575   return (MacroAssembler::instruction_size * (2 /* 2 loads */ + 1 /* branch */)) +
5576           far_branch_size() + (UseCompactObjectHeaders ? MacroAssembler::instruction_size * 1 : 0);
5577 }
5578 
5579 int MacroAssembler::ic_check(int end_alignment) {
5580   IncompressibleScope scope(this);
5581   Register receiver = j_rarg0;
5582   Register data = t0;
5583 
5584   Register tmp1 = t1; // scratch
5585   // t2 is saved on call, thus should have been saved before this check.
5586   // Hence we can clobber it.
5587   Register tmp2 = t2;
5588 
5589   // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
5590   // before the inline cache check, so we don't have to execute any nop instructions when dispatching
5591   // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
5592   // before the inline cache check here, and not after
5593   align(end_alignment, ic_check_size());
5594   int uep_offset = offset();
5595 
5596   if (UseCompactObjectHeaders) {
5597     load_narrow_klass_compact(tmp1, receiver);
5598     lwu(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
5599   } else {
5600     lwu(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
5601     lwu(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
5602   }
5603 
5604   Label ic_hit;
5605   beq(tmp1, tmp2, ic_hit);
5606   // Note, far_jump is not fixed size.
5607   // Is this ever generates a movptr alignment/size will be off.
5608   far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
5609   bind(ic_hit);
5610 
5611   assert((offset() % end_alignment) == 0, "Misaligned verified entry point.");
5612   return uep_offset;
5613 }
5614 
5615 // Emit an address stub for a call to a target which is too far away.
5616 // Note that we only put the target address of the call in the stub.
5617 //
5618 // code sequences:
5619 //
5620 // call-site:
5621 //   load target address from stub
5622 //   jump-and-link target address
5623 //
5624 // Related address stub for this call site in the stub section:
5625 //   alignment nop
5626 //   target address
5627 
5628 address MacroAssembler::emit_reloc_call_address_stub(int insts_call_instruction_offset, address dest) {
5629   address stub = start_a_stub(max_reloc_call_address_stub_size());
5630   if (stub == nullptr) {
5631     return nullptr;  // CodeBuffer::expand failed
5632   }
5633 
5634   // We are always 4-byte aligned here.
5635   assert_alignment(pc());
5636 
5637   // Make sure the address of destination 8-byte aligned.
5638   align(wordSize, 0);
5639 
5640   RelocationHolder rh = trampoline_stub_Relocation::spec(code()->insts()->start() +
5641                                                          insts_call_instruction_offset);
5642   const int stub_start_offset = offset();
5643   relocate(rh, [&] {
5644     assert(offset() - stub_start_offset == 0,
5645            "%ld - %ld == %ld : should be", (long)offset(), (long)stub_start_offset, (long)0);
5646     assert(offset() % wordSize == 0, "bad alignment");
5647     emit_int64((int64_t)dest);
5648   });
5649 
5650   const address stub_start_addr = addr_at(stub_start_offset);
5651   end_a_stub();
5652 
5653   return stub_start_addr;
5654 }
5655 
5656 int MacroAssembler::max_reloc_call_address_stub_size() {
5657   // Max stub size: alignment nop, target address.
5658   return 1 * MacroAssembler::instruction_size + wordSize;
5659 }
5660 
5661 int MacroAssembler::static_call_stub_size() {
5662   // (lui, addi, slli, addi, slli, addi) + (lui + lui + slli + add) + jalr
5663   return 11 * MacroAssembler::instruction_size;
5664 }
5665 
5666 Address MacroAssembler::add_memory_helper(const Address dst, Register tmp) {
5667   switch (dst.getMode()) {
5668     case Address::base_plus_offset:
5669       // This is the expected mode, although we allow all the other
5670       // forms below.
5671       return form_address(tmp, dst.base(), dst.offset());
5672     default:
5673       la(tmp, dst);
5674       return Address(tmp);
5675   }
5676 }
5677 
5678 void MacroAssembler::increment(const Address dst, int64_t value, Register tmp1, Register tmp2) {
5679   assert(((dst.getMode() == Address::base_plus_offset &&
5680            is_simm12(dst.offset())) || is_simm12(value)),
5681           "invalid value and address mode combination");
5682   Address adr = add_memory_helper(dst, tmp2);
5683   assert(!adr.uses(tmp1), "invalid dst for address increment");
5684   ld(tmp1, adr);
5685   add(tmp1, tmp1, value, tmp2);
5686   sd(tmp1, adr);
5687 }
5688 
5689 void MacroAssembler::incrementw(const Address dst, int32_t value, Register tmp1, Register tmp2) {
5690   assert(((dst.getMode() == Address::base_plus_offset &&
5691            is_simm12(dst.offset())) || is_simm12(value)),
5692           "invalid value and address mode combination");
5693   Address adr = add_memory_helper(dst, tmp2);
5694   assert(!adr.uses(tmp1), "invalid dst for address increment");
5695   lwu(tmp1, adr);
5696   addw(tmp1, tmp1, value, tmp2);
5697   sw(tmp1, adr);
5698 }
5699 
5700 void MacroAssembler::decrement(const Address dst, int64_t value, Register tmp1, Register tmp2) {
5701   assert(((dst.getMode() == Address::base_plus_offset &&
5702            is_simm12(dst.offset())) || is_simm12(value)),
5703           "invalid value and address mode combination");
5704   Address adr = add_memory_helper(dst, tmp2);
5705   assert(!adr.uses(tmp1), "invalid dst for address decrement");
5706   ld(tmp1, adr);
5707   sub(tmp1, tmp1, value, tmp2);
5708   sd(tmp1, adr);
5709 }
5710 
5711 void MacroAssembler::decrementw(const Address dst, int32_t value, Register tmp1, Register tmp2) {
5712   assert(((dst.getMode() == Address::base_plus_offset &&
5713            is_simm12(dst.offset())) || is_simm12(value)),
5714           "invalid value and address mode combination");
5715   Address adr = add_memory_helper(dst, tmp2);
5716   assert(!adr.uses(tmp1), "invalid dst for address decrement");
5717   lwu(tmp1, adr);
5718   subw(tmp1, tmp1, value, tmp2);
5719   sw(tmp1, adr);
5720 }
5721 
5722 void MacroAssembler::load_method_holder_cld(Register result, Register method) {
5723   load_method_holder(result, method);
5724   ld(result, Address(result, InstanceKlass::class_loader_data_offset()));
5725 }
5726 
5727 void MacroAssembler::load_method_holder(Register holder, Register method) {
5728   ld(holder, Address(method, Method::const_offset()));                      // ConstMethod*
5729   ld(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
5730   ld(holder, Address(holder, ConstantPool::pool_holder_offset()));          // InstanceKlass*
5731 }
5732 
5733 void MacroAssembler::load_metadata(Register dst, Register src) {
5734   if (UseCompactObjectHeaders) {
5735     load_narrow_klass_compact(dst, src);
5736   } else {
5737     lwu(dst, Address(src, oopDesc::klass_offset_in_bytes()));
5738   }
5739 }
5740 
5741 // string indexof
5742 // compute index by trailing zeros
5743 void MacroAssembler::compute_index(Register haystack, Register trailing_zeros,
5744                                    Register match_mask, Register result,
5745                                    Register ch2, Register tmp,
5746                                    bool haystack_isL) {
5747   int haystack_chr_shift = haystack_isL ? 0 : 1;
5748   srl(match_mask, match_mask, trailing_zeros);
5749   srli(match_mask, match_mask, 1);
5750   srli(tmp, trailing_zeros, LogBitsPerByte);
5751   if (!haystack_isL) andi(tmp, tmp, 0xE);
5752   add(haystack, haystack, tmp);
5753   ld(ch2, Address(haystack));
5754   if (!haystack_isL) srli(tmp, tmp, haystack_chr_shift);
5755   add(result, result, tmp);
5756 }
5757 
5758 // string indexof
5759 // Find pattern element in src, compute match mask,
5760 // only the first occurrence of 0x80/0x8000 at low bits is the valid match index
5761 // match mask patterns and corresponding indices would be like:
5762 // - 0x8080808080808080 (Latin1)
5763 // -   7 6 5 4 3 2 1 0  (match index)
5764 // - 0x8000800080008000 (UTF16)
5765 // -   3   2   1   0    (match index)
5766 void MacroAssembler::compute_match_mask(Register src, Register pattern, Register match_mask,
5767                                         Register mask1, Register mask2) {
5768   xorr(src, pattern, src);
5769   sub(match_mask, src, mask1);
5770   orr(src, src, mask2);
5771   notr(src, src);
5772   andr(match_mask, match_mask, src);
5773 }
5774 
5775 #ifdef COMPILER2
5776 // Code for BigInteger::mulAdd intrinsic
5777 // out     = x10
5778 // in      = x11
5779 // offset  = x12  (already out.length-offset)
5780 // len     = x13
5781 // k       = x14
5782 // tmp     = x28
5783 //
5784 // pseudo code from java implementation:
5785 // long kLong = k & LONG_MASK;
5786 // carry = 0;
5787 // offset = out.length-offset - 1;
5788 // for (int j = len - 1; j >= 0; j--) {
5789 //     product = (in[j] & LONG_MASK) * kLong + (out[offset] & LONG_MASK) + carry;
5790 //     out[offset--] = (int)product;
5791 //     carry = product >>> 32;
5792 // }
5793 // return (int)carry;
5794 void MacroAssembler::mul_add(Register out, Register in, Register offset,
5795                              Register len, Register k, Register tmp) {
5796   Label L_tail_loop, L_unroll, L_end;
5797   mv(tmp, out);
5798   mv(out, zr);
5799   blez(len, L_end);
5800   zext(k, k, 32);
5801   slliw(t0, offset, LogBytesPerInt);
5802   add(offset, tmp, t0);
5803   slliw(t0, len, LogBytesPerInt);
5804   add(in, in, t0);
5805 
5806   const int unroll = 8;
5807   mv(tmp, unroll);
5808   blt(len, tmp, L_tail_loop);
5809   bind(L_unroll);
5810   for (int i = 0; i < unroll; i++) {
5811     subi(in, in, BytesPerInt);
5812     lwu(t0, Address(in, 0));
5813     mul(t1, t0, k);
5814     add(t0, t1, out);
5815     subi(offset, offset, BytesPerInt);
5816     lwu(t1, Address(offset, 0));
5817     add(t0, t0, t1);
5818     sw(t0, Address(offset, 0));
5819     srli(out, t0, 32);
5820   }
5821   subw(len, len, tmp);
5822   bge(len, tmp, L_unroll);
5823 
5824   bind(L_tail_loop);
5825   blez(len, L_end);
5826   subi(in, in, BytesPerInt);
5827   lwu(t0, Address(in, 0));
5828   mul(t1, t0, k);
5829   add(t0, t1, out);
5830   subi(offset, offset, BytesPerInt);
5831   lwu(t1, Address(offset, 0));
5832   add(t0, t0, t1);
5833   sw(t0, Address(offset, 0));
5834   srli(out, t0, 32);
5835   subiw(len, len, 1);
5836   j(L_tail_loop);
5837 
5838   bind(L_end);
5839 }
5840 
5841 // Multiply and multiply-accumulate unsigned 64-bit registers.
5842 void MacroAssembler::wide_mul(Register prod_lo, Register prod_hi, Register n, Register m) {
5843   assert_different_registers(prod_lo, prod_hi);
5844 
5845   mul(prod_lo, n, m);
5846   mulhu(prod_hi, n, m);
5847 }
5848 
5849 void MacroAssembler::wide_madd(Register sum_lo, Register sum_hi, Register n,
5850                                Register m, Register tmp1, Register tmp2) {
5851   assert_different_registers(sum_lo, sum_hi);
5852   assert_different_registers(sum_hi, tmp2);
5853 
5854   wide_mul(tmp1, tmp2, n, m);
5855   cad(sum_lo, sum_lo, tmp1, tmp1);  // Add tmp1 to sum_lo with carry output to tmp1
5856   adc(sum_hi, sum_hi, tmp2, tmp1);  // Add tmp2 with carry to sum_hi
5857 }
5858 
5859 // add two unsigned input and output carry
5860 void MacroAssembler::cad(Register dst, Register src1, Register src2, Register carry)
5861 {
5862   assert_different_registers(dst, carry);
5863   assert_different_registers(dst, src2);
5864   add(dst, src1, src2);
5865   sltu(carry, dst, src2);
5866 }
5867 
5868 // add two input with carry
5869 void MacroAssembler::adc(Register dst, Register src1, Register src2, Register carry) {
5870   assert_different_registers(dst, carry);
5871   add(dst, src1, src2);
5872   add(dst, dst, carry);
5873 }
5874 
5875 // add two unsigned input with carry and output carry
5876 void MacroAssembler::cadc(Register dst, Register src1, Register src2, Register carry) {
5877   assert_different_registers(dst, src2);
5878   adc(dst, src1, src2, carry);
5879   sltu(carry, dst, src2);
5880 }
5881 
5882 void MacroAssembler::add2_with_carry(Register final_dest_hi, Register dest_hi, Register dest_lo,
5883                                      Register src1, Register src2, Register carry) {
5884   cad(dest_lo, dest_lo, src1, carry);
5885   add(dest_hi, dest_hi, carry);
5886   cad(dest_lo, dest_lo, src2, carry);
5887   add(final_dest_hi, dest_hi, carry);
5888 }
5889 
5890 /**
5891  * Multiply 64 bit by 64 bit first loop.
5892  */
5893 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
5894                                            Register y, Register y_idx, Register z,
5895                                            Register carry, Register product,
5896                                            Register idx, Register kdx) {
5897   //
5898   //  jlong carry, x[], y[], z[];
5899   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx--, kdx--) {
5900   //    huge_128 product = y[idx] * x[xstart] + carry;
5901   //    z[kdx] = (jlong)product;
5902   //    carry  = (jlong)(product >>> 64);
5903   //  }
5904   //  z[xstart] = carry;
5905   //
5906 
5907   Label L_first_loop, L_first_loop_exit;
5908   Label L_one_x, L_one_y, L_multiply;
5909 
5910   subiw(xstart, xstart, 1);
5911   bltz(xstart, L_one_x);
5912 
5913   shadd(t0, xstart, x, t0, LogBytesPerInt);
5914   ld(x_xstart, Address(t0, 0));
5915   ror(x_xstart, x_xstart, 32); // convert big-endian to little-endian
5916 
5917   bind(L_first_loop);
5918   subiw(idx, idx, 1);
5919   bltz(idx, L_first_loop_exit);
5920   subiw(idx, idx, 1);
5921   bltz(idx, L_one_y);
5922 
5923   shadd(t0, idx, y, t0, LogBytesPerInt);
5924   ld(y_idx, Address(t0, 0));
5925   ror(y_idx, y_idx, 32); // convert big-endian to little-endian
5926   bind(L_multiply);
5927 
5928   mulhu(t0, x_xstart, y_idx);
5929   mul(product, x_xstart, y_idx);
5930   cad(product, product, carry, t1);
5931   adc(carry, t0, zr, t1);
5932 
5933   subiw(kdx, kdx, 2);
5934   ror(product, product, 32); // back to big-endian
5935   shadd(t0, kdx, z, t0, LogBytesPerInt);
5936   sd(product, Address(t0, 0));
5937 
5938   j(L_first_loop);
5939 
5940   bind(L_one_y);
5941   lwu(y_idx, Address(y, 0));
5942   j(L_multiply);
5943 
5944   bind(L_one_x);
5945   lwu(x_xstart, Address(x, 0));
5946   j(L_first_loop);
5947 
5948   bind(L_first_loop_exit);
5949 }
5950 
5951 /**
5952  * Multiply 128 bit by 128 bit. Unrolled inner loop.
5953  *
5954  */
5955 void MacroAssembler::multiply_128_x_128_loop(Register y, Register z,
5956                                              Register carry, Register carry2,
5957                                              Register idx, Register jdx,
5958                                              Register yz_idx1, Register yz_idx2,
5959                                              Register tmp, Register tmp3, Register tmp4,
5960                                              Register tmp6, Register product_hi) {
5961   //   jlong carry, x[], y[], z[];
5962   //   int kdx = xstart+1;
5963   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
5964   //     huge_128 tmp3 = (y[idx+1] * product_hi) + z[kdx+idx+1] + carry;
5965   //     jlong carry2  = (jlong)(tmp3 >>> 64);
5966   //     huge_128 tmp4 = (y[idx]   * product_hi) + z[kdx+idx] + carry2;
5967   //     carry  = (jlong)(tmp4 >>> 64);
5968   //     z[kdx+idx+1] = (jlong)tmp3;
5969   //     z[kdx+idx] = (jlong)tmp4;
5970   //   }
5971   //   idx += 2;
5972   //   if (idx > 0) {
5973   //     yz_idx1 = (y[idx] * product_hi) + z[kdx+idx] + carry;
5974   //     z[kdx+idx] = (jlong)yz_idx1;
5975   //     carry  = (jlong)(yz_idx1 >>> 64);
5976   //   }
5977   //
5978 
5979   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
5980 
5981   srliw(jdx, idx, 2);
5982 
5983   bind(L_third_loop);
5984 
5985   subw(jdx, jdx, 1);
5986   bltz(jdx, L_third_loop_exit);
5987   subw(idx, idx, 4);
5988 
5989   shadd(t0, idx, y, t0, LogBytesPerInt);
5990   ld(yz_idx2, Address(t0, 0));
5991   ld(yz_idx1, Address(t0, wordSize));
5992 
5993   shadd(tmp6, idx, z, t0, LogBytesPerInt);
5994 
5995   ror(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
5996   ror(yz_idx2, yz_idx2, 32);
5997 
5998   ld(t1, Address(tmp6, 0));
5999   ld(t0, Address(tmp6, wordSize));
6000 
6001   mul(tmp3, product_hi, yz_idx1); //  yz_idx1 * product_hi -> tmp4:tmp3
6002   mulhu(tmp4, product_hi, yz_idx1);
6003 
6004   ror(t0, t0, 32, tmp); // convert big-endian to little-endian
6005   ror(t1, t1, 32, tmp);
6006 
6007   mul(tmp, product_hi, yz_idx2); //  yz_idx2 * product_hi -> carry2:tmp
6008   mulhu(carry2, product_hi, yz_idx2);
6009 
6010   cad(tmp3, tmp3, carry, carry);
6011   adc(tmp4, tmp4, zr, carry);
6012   cad(tmp3, tmp3, t0, t0);
6013   cadc(tmp4, tmp4, tmp, t0);
6014   adc(carry, carry2, zr, t0);
6015   cad(tmp4, tmp4, t1, carry2);
6016   adc(carry, carry, zr, carry2);
6017 
6018   ror(tmp3, tmp3, 32); // convert little-endian to big-endian
6019   ror(tmp4, tmp4, 32);
6020   sd(tmp4, Address(tmp6, 0));
6021   sd(tmp3, Address(tmp6, wordSize));
6022 
6023   j(L_third_loop);
6024 
6025   bind(L_third_loop_exit);
6026 
6027   andi(idx, idx, 0x3);
6028   beqz(idx, L_post_third_loop_done);
6029 
6030   Label L_check_1;
6031   subiw(idx, idx, 2);
6032   bltz(idx, L_check_1);
6033 
6034   shadd(t0, idx, y, t0, LogBytesPerInt);
6035   ld(yz_idx1, Address(t0, 0));
6036   ror(yz_idx1, yz_idx1, 32);
6037 
6038   mul(tmp3, product_hi, yz_idx1); //  yz_idx1 * product_hi -> tmp4:tmp3
6039   mulhu(tmp4, product_hi, yz_idx1);
6040 
6041   shadd(t0, idx, z, t0, LogBytesPerInt);
6042   ld(yz_idx2, Address(t0, 0));
6043   ror(yz_idx2, yz_idx2, 32, tmp);
6044 
6045   add2_with_carry(carry, tmp4, tmp3, carry, yz_idx2, tmp);
6046 
6047   ror(tmp3, tmp3, 32, tmp);
6048   sd(tmp3, Address(t0, 0));
6049 
6050   bind(L_check_1);
6051 
6052   andi(idx, idx, 0x1);
6053   subiw(idx, idx, 1);
6054   bltz(idx, L_post_third_loop_done);
6055   shadd(t0, idx, y, t0, LogBytesPerInt);
6056   lwu(tmp4, Address(t0, 0));
6057   mul(tmp3, tmp4, product_hi); //  tmp4 * product_hi -> carry2:tmp3
6058   mulhu(carry2, tmp4, product_hi);
6059 
6060   shadd(t0, idx, z, t0, LogBytesPerInt);
6061   lwu(tmp4, Address(t0, 0));
6062 
6063   add2_with_carry(carry2, carry2, tmp3, tmp4, carry, t0);
6064 
6065   shadd(t0, idx, z, t0, LogBytesPerInt);
6066   sw(tmp3, Address(t0, 0));
6067 
6068   slli(t0, carry2, 32);
6069   srli(carry, tmp3, 32);
6070   orr(carry, carry, t0);
6071 
6072   bind(L_post_third_loop_done);
6073 }
6074 
6075 /**
6076  * Code for BigInteger::multiplyToLen() intrinsic.
6077  *
6078  * x10: x
6079  * x11: xlen
6080  * x12: y
6081  * x13: ylen
6082  * x14: z
6083  * x15: tmp0
6084  * x16: tmp1
6085  * x17: tmp2
6086  * x7:  tmp3
6087  * x28: tmp4
6088  * x29: tmp5
6089  * x30: tmp6
6090  * x31: tmp7
6091  */
6092 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen,
6093                                      Register z, Register tmp0,
6094                                      Register tmp1, Register tmp2, Register tmp3, Register tmp4,
6095                                      Register tmp5, Register tmp6, Register product_hi) {
6096   assert_different_registers(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6);
6097 
6098   const Register idx = tmp1;
6099   const Register kdx = tmp2;
6100   const Register xstart = tmp3;
6101 
6102   const Register y_idx = tmp4;
6103   const Register carry = tmp5;
6104   const Register product = xlen;
6105   const Register x_xstart = tmp0;
6106   const Register jdx = tmp1;
6107 
6108   mv(idx, ylen);         // idx = ylen;
6109   addw(kdx, xlen, ylen); // kdx = xlen+ylen;
6110   mv(carry, zr);         // carry = 0;
6111 
6112   Label L_done;
6113   subiw(xstart, xlen, 1);
6114   bltz(xstart, L_done);
6115 
6116   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
6117 
6118   Label L_second_loop_aligned;
6119   beqz(kdx, L_second_loop_aligned);
6120 
6121   Label L_carry;
6122   subiw(kdx, kdx, 1);
6123   beqz(kdx, L_carry);
6124 
6125   shadd(t0, kdx, z, t0, LogBytesPerInt);
6126   sw(carry, Address(t0, 0));
6127   srli(carry, carry, 32);
6128   subiw(kdx, kdx, 1);
6129 
6130   bind(L_carry);
6131   shadd(t0, kdx, z, t0, LogBytesPerInt);
6132   sw(carry, Address(t0, 0));
6133 
6134   // Second and third (nested) loops.
6135   //
6136   // for (int i = xstart-1; i >= 0; i--) { // Second loop
6137   //   carry = 0;
6138   //   for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
6139   //     long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
6140   //                    (z[k] & LONG_MASK) + carry;
6141   //     z[k] = (int)product;
6142   //     carry = product >>> 32;
6143   //   }
6144   //   z[i] = (int)carry;
6145   // }
6146   //
6147   // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = product_hi
6148 
6149   bind(L_second_loop_aligned);
6150   mv(carry, zr); // carry = 0;
6151   mv(jdx, ylen); // j = ystart+1
6152 
6153   subiw(xstart, xstart, 1); // i = xstart-1;
6154   bltz(xstart, L_done);
6155 
6156   subi(sp, sp, 4 * wordSize);
6157   sd(z, Address(sp, 0));
6158 
6159   Label L_last_x;
6160   shadd(t0, xstart, z, t0, LogBytesPerInt);
6161   addi(z, t0, 4);
6162   subiw(xstart, xstart, 1); // i = xstart-1;
6163   bltz(xstart, L_last_x);
6164 
6165   shadd(t0, xstart, x, t0, LogBytesPerInt);
6166   ld(product_hi, Address(t0, 0));
6167   ror(product_hi, product_hi, 32); // convert big-endian to little-endian
6168 
6169   Label L_third_loop_prologue;
6170   bind(L_third_loop_prologue);
6171 
6172   sd(ylen, Address(sp, wordSize));
6173   sd(x, Address(sp, 2 * wordSize));
6174   sd(xstart, Address(sp, 3 * wordSize));
6175   multiply_128_x_128_loop(y, z, carry, x, jdx, ylen, product,
6176                           tmp2, x_xstart, tmp3, tmp4, tmp6, product_hi);
6177   ld(z, Address(sp, 0));
6178   ld(ylen, Address(sp, wordSize));
6179   ld(x, Address(sp, 2 * wordSize));
6180   ld(xlen, Address(sp, 3 * wordSize)); // copy old xstart -> xlen
6181   addi(sp, sp, 4 * wordSize);
6182 
6183   addiw(tmp3, xlen, 1);
6184   shadd(t0, tmp3, z, t0, LogBytesPerInt);
6185   sw(carry, Address(t0, 0));
6186 
6187   subiw(tmp3, tmp3, 1);
6188   bltz(tmp3, L_done);
6189 
6190   srli(carry, carry, 32);
6191   shadd(t0, tmp3, z, t0, LogBytesPerInt);
6192   sw(carry, Address(t0, 0));
6193   j(L_second_loop_aligned);
6194 
6195   // Next infrequent code is moved outside loops.
6196   bind(L_last_x);
6197   lwu(product_hi, Address(x, 0));
6198   j(L_third_loop_prologue);
6199 
6200   bind(L_done);
6201 }
6202 #endif
6203 
6204 // Count bits of trailing zero chars from lsb to msb until first non-zero
6205 // char seen. For the LL case, shift 8 bits once as there is only one byte
6206 // per each char. For other cases, shift 16 bits once.
6207 void MacroAssembler::ctzc_bits(Register Rd, Register Rs, bool isLL,
6208                                Register tmp1, Register tmp2) {
6209   int step = isLL ? 8 : 16;
6210   if (UseZbb) {
6211     ctz(Rd, Rs);
6212     andi(Rd, Rd, -step);
6213     return;
6214   }
6215 
6216   assert_different_registers(Rd, tmp1, tmp2);
6217   Label Loop;
6218   mv(tmp2, Rs);
6219   mv(Rd, -step);
6220 
6221   bind(Loop);
6222   addi(Rd, Rd, step);
6223   zext(tmp1, tmp2, step);
6224   srli(tmp2, tmp2, step);
6225   beqz(tmp1, Loop);
6226 }
6227 
6228 // This instruction reads adjacent 4 bytes from the lower half of source register,
6229 // inflate into a register, for example:
6230 // Rs: A7A6A5A4A3A2A1A0
6231 // Rd: 00A300A200A100A0
6232 void MacroAssembler::inflate_lo32(Register Rd, Register Rs, Register tmp1, Register tmp2) {
6233   assert_different_registers(Rd, Rs, tmp1, tmp2);
6234 
6235   mv(tmp1, 0xFF000000); // first byte mask at lower word
6236   andr(Rd, Rs, tmp1);
6237   for (int i = 0; i < 2; i++) {
6238     slli(Rd, Rd, wordSize);
6239     srli(tmp1, tmp1, wordSize);
6240     andr(tmp2, Rs, tmp1);
6241     orr(Rd, Rd, tmp2);
6242   }
6243   slli(Rd, Rd, wordSize);
6244   zext(tmp2, Rs, 8); // last byte mask at lower word
6245   orr(Rd, Rd, tmp2);
6246 }
6247 
6248 // This instruction reads adjacent 4 bytes from the upper half of source register,
6249 // inflate into a register, for example:
6250 // Rs: A7A6A5A4A3A2A1A0
6251 // Rd: 00A700A600A500A4
6252 void MacroAssembler::inflate_hi32(Register Rd, Register Rs, Register tmp1, Register tmp2) {
6253   assert_different_registers(Rd, Rs, tmp1, tmp2);
6254   srli(Rs, Rs, 32);   // only upper 32 bits are needed
6255   inflate_lo32(Rd, Rs, tmp1, tmp2);
6256 }
6257 
6258 // The size of the blocks erased by the zero_blocks stub.  We must
6259 // handle anything smaller than this ourselves in zero_words().
6260 const int MacroAssembler::zero_words_block_size = 8;
6261 
6262 // zero_words() is used by C2 ClearArray patterns.  It is as small as
6263 // possible, handling small word counts locally and delegating
6264 // anything larger to the zero_blocks stub.  It is expanded many times
6265 // in compiled code, so it is important to keep it short.
6266 
6267 // ptr:   Address of a buffer to be zeroed.
6268 // cnt:   Count in HeapWords.
6269 //
6270 // ptr, cnt, t1, and t0 are clobbered.
6271 address MacroAssembler::zero_words(Register ptr, Register cnt) {
6272   assert(is_power_of_2(zero_words_block_size), "adjust this");
6273   assert(ptr == x28 && cnt == x29, "mismatch in register usage");
6274   assert_different_registers(cnt, t0, t1);
6275 
6276   BLOCK_COMMENT("zero_words {");
6277 
6278   mv(t0, zero_words_block_size);
6279   Label around, done, done16;
6280   bltu(cnt, t0, around);
6281   {
6282     RuntimeAddress zero_blocks(StubRoutines::riscv::zero_blocks());
6283     assert(zero_blocks.target() != nullptr, "zero_blocks stub has not been generated");
6284     if (StubRoutines::riscv::complete()) {
6285       address tpc = reloc_call(zero_blocks);
6286       if (tpc == nullptr) {
6287         DEBUG_ONLY(reset_labels(around));
6288         postcond(pc() == badAddress);
6289         return nullptr;
6290       }
6291     } else {
6292       // Clobbers t1
6293       rt_call(zero_blocks.target());
6294     }
6295   }
6296   bind(around);
6297   for (int i = zero_words_block_size >> 1; i > 1; i >>= 1) {
6298     Label l;
6299     test_bit(t0, cnt, exact_log2(i));
6300     beqz(t0, l);
6301     for (int j = 0; j < i; j++) {
6302       sd(zr, Address(ptr, j * wordSize));
6303     }
6304     addi(ptr, ptr, i * wordSize);
6305     bind(l);
6306   }
6307   {
6308     Label l;
6309     test_bit(t0, cnt, 0);
6310     beqz(t0, l);
6311     sd(zr, Address(ptr, 0));
6312     bind(l);
6313   }
6314 
6315   BLOCK_COMMENT("} zero_words");
6316   postcond(pc() != badAddress);
6317   return pc();
6318 }
6319 
6320 #define SmallArraySize (18 * BytesPerLong)
6321 
6322 // base:  Address of a buffer to be zeroed, 8 bytes aligned.
6323 // cnt:   Immediate count in HeapWords.
6324 void MacroAssembler::zero_words(Register base, uint64_t cnt) {
6325   assert_different_registers(base, t0, t1);
6326 
6327   BLOCK_COMMENT("zero_words {");
6328 
6329   if (cnt <= SmallArraySize / BytesPerLong) {
6330     for (int i = 0; i < (int)cnt; i++) {
6331       sd(zr, Address(base, i * wordSize));
6332     }
6333   } else {
6334     const int unroll = 8; // Number of sd(zr, adr), instructions we'll unroll
6335     int remainder = cnt % unroll;
6336     for (int i = 0; i < remainder; i++) {
6337       sd(zr, Address(base, i * wordSize));
6338     }
6339 
6340     Label loop;
6341     Register cnt_reg = t0;
6342     Register loop_base = t1;
6343     cnt = cnt - remainder;
6344     mv(cnt_reg, cnt);
6345     addi(loop_base, base, remainder * wordSize);
6346     bind(loop);
6347     sub(cnt_reg, cnt_reg, unroll);
6348     for (int i = 0; i < unroll; i++) {
6349       sd(zr, Address(loop_base, i * wordSize));
6350     }
6351     addi(loop_base, loop_base, unroll * wordSize);
6352     bnez(cnt_reg, loop);
6353   }
6354 
6355   BLOCK_COMMENT("} zero_words");
6356 }
6357 
6358 // base:   Address of a buffer to be filled, 8 bytes aligned.
6359 // cnt:    Count in 8-byte unit.
6360 // value:  Value to be filled with.
6361 // base will point to the end of the buffer after filling.
6362 void MacroAssembler::fill_words(Register base, Register cnt, Register value) {
6363 //  Algorithm:
6364 //
6365 //    t0 = cnt & 7
6366 //    cnt -= t0
6367 //    p += t0
6368 //    switch (t0):
6369 //      switch start:
6370 //      do while cnt
6371 //        cnt -= 8
6372 //          p[-8] = value
6373 //        case 7:
6374 //          p[-7] = value
6375 //        case 6:
6376 //          p[-6] = value
6377 //          // ...
6378 //        case 1:
6379 //          p[-1] = value
6380 //        case 0:
6381 //          p += 8
6382 //      do-while end
6383 //    switch end
6384 
6385   assert_different_registers(base, cnt, value, t0, t1);
6386 
6387   Label fini, skip, entry, loop;
6388   const int unroll = 8; // Number of sd instructions we'll unroll
6389 
6390   beqz(cnt, fini);
6391 
6392   andi(t0, cnt, unroll - 1);
6393   sub(cnt, cnt, t0);
6394   shadd(base, t0, base, t1, 3);
6395   la(t1, entry);
6396   slli(t0, t0, 2);
6397   sub(t1, t1, t0);
6398   jr(t1);
6399 
6400   bind(loop);
6401   addi(base, base, unroll * wordSize);
6402   {
6403     IncompressibleScope scope(this); // Fixed length
6404     for (int i = -unroll; i < 0; i++) {
6405       sd(value, Address(base, i * 8));
6406     }
6407   }
6408   bind(entry);
6409   subi(cnt, cnt, unroll);
6410   bgez(cnt, loop);
6411 
6412   bind(fini);
6413 }
6414 
6415 // Zero blocks of memory by using CBO.ZERO.
6416 //
6417 // Aligns the base address first sufficiently for CBO.ZERO, then uses
6418 // CBO.ZERO repeatedly for every full block.  cnt is the size to be
6419 // zeroed in HeapWords.  Returns the count of words left to be zeroed
6420 // in cnt.
6421 //
6422 // NOTE: This is intended to be used in the zero_blocks() stub.  If
6423 // you want to use it elsewhere, note that cnt must be >= zicboz_block_size.
6424 void MacroAssembler::zero_dcache_blocks(Register base, Register cnt, Register tmp1, Register tmp2) {
6425   int zicboz_block_size = VM_Version::zicboz_block_size.value();
6426   Label initial_table_end, loop;
6427 
6428   // Align base with cache line size.
6429   neg(tmp1, base);
6430   andi(tmp1, tmp1, zicboz_block_size - 1);
6431 
6432   // tmp1: the number of bytes to be filled to align the base with cache line size.
6433   add(base, base, tmp1);
6434   srai(tmp2, tmp1, 3);
6435   sub(cnt, cnt, tmp2);
6436   srli(tmp2, tmp1, 1);
6437   la(tmp1, initial_table_end);
6438   sub(tmp2, tmp1, tmp2);
6439   jr(tmp2);
6440   for (int i = -zicboz_block_size + wordSize; i < 0; i += wordSize) {
6441     sd(zr, Address(base, i));
6442   }
6443   bind(initial_table_end);
6444 
6445   mv(tmp1, zicboz_block_size / wordSize);
6446   bind(loop);
6447   cbo_zero(base);
6448   sub(cnt, cnt, tmp1);
6449   addi(base, base, zicboz_block_size);
6450   bge(cnt, tmp1, loop);
6451 }
6452 
6453 // java.lang.Math.round(float a)
6454 // Returns the closest int to the argument, with ties rounding to positive infinity.
6455 void MacroAssembler::java_round_float(Register dst, FloatRegister src, FloatRegister ftmp) {
6456   // this instructions calling sequence provides performance improvement on all tested devices;
6457   // don't change it without re-verification
6458   Label done;
6459   mv(t0, jint_cast(0.5f));
6460   fmv_w_x(ftmp, t0);
6461 
6462   // dst = 0 if NaN
6463   feq_s(t0, src, src); // replacing fclass with feq as performance optimization
6464   mv(dst, zr);
6465   beqz(t0, done);
6466 
6467   // dst = (src + 0.5f) rounded down towards negative infinity
6468   //   Adding 0.5f to some floats exceeds the precision limits for a float and rounding takes place.
6469   //   RDN is required for fadd_s, RNE gives incorrect results:
6470   //     --------------------------------------------------------------------
6471   //     fadd.s rne (src + 0.5f): src = 8388609.000000  ftmp = 8388610.000000
6472   //     fcvt.w.s rdn: ftmp = 8388610.000000 dst = 8388610
6473   //     --------------------------------------------------------------------
6474   //     fadd.s rdn (src + 0.5f): src = 8388609.000000  ftmp = 8388609.000000
6475   //     fcvt.w.s rdn: ftmp = 8388609.000000 dst = 8388609
6476   //     --------------------------------------------------------------------
6477   fadd_s(ftmp, src, ftmp, RoundingMode::rdn);
6478   fcvt_w_s(dst, ftmp, RoundingMode::rdn);
6479 
6480   bind(done);
6481 }
6482 
6483 // java.lang.Math.round(double a)
6484 // Returns the closest long to the argument, with ties rounding to positive infinity.
6485 void MacroAssembler::java_round_double(Register dst, FloatRegister src, FloatRegister ftmp) {
6486   // this instructions calling sequence provides performance improvement on all tested devices;
6487   // don't change it without re-verification
6488   Label done;
6489   mv(t0, julong_cast(0.5));
6490   fmv_d_x(ftmp, t0);
6491 
6492   // dst = 0 if NaN
6493   feq_d(t0, src, src); // replacing fclass with feq as performance optimization
6494   mv(dst, zr);
6495   beqz(t0, done);
6496 
6497   // dst = (src + 0.5) rounded down towards negative infinity
6498   fadd_d(ftmp, src, ftmp, RoundingMode::rdn); // RDN is required here otherwise some inputs produce incorrect results
6499   fcvt_l_d(dst, ftmp, RoundingMode::rdn);
6500 
6501   bind(done);
6502 }
6503 
6504 // Helper routine processing the slow path of NaN when converting float to float16
6505 void MacroAssembler::float_to_float16_NaN(Register dst, FloatRegister src,
6506                                           Register tmp1, Register tmp2) {
6507   fmv_x_w(dst, src);
6508 
6509   //  Float (32 bits)
6510   //    Bit:     31        30 to 23          22 to 0
6511   //          +---+------------------+-----------------------------+
6512   //          | S |     Exponent     |      Mantissa (Fraction)    |
6513   //          +---+------------------+-----------------------------+
6514   //          1 bit       8 bits                  23 bits
6515   //
6516   //  Float (16 bits)
6517   //    Bit:    15        14 to 10         9 to 0
6518   //          +---+----------------+------------------+
6519   //          | S |    Exponent    |     Mantissa     |
6520   //          +---+----------------+------------------+
6521   //          1 bit      5 bits          10 bits
6522   const int fp_sign_bits = 1;
6523   const int fp32_bits = 32;
6524   const int fp32_exponent_bits = 8;
6525   const int fp32_mantissa_1st_part_bits = 10;
6526   const int fp32_mantissa_2nd_part_bits = 9;
6527   const int fp32_mantissa_3rd_part_bits = 4;
6528   const int fp16_exponent_bits = 5;
6529   const int fp16_mantissa_bits = 10;
6530 
6531   // preserve the sign bit and exponent, clear mantissa.
6532   srai(tmp2, dst, fp32_bits - fp_sign_bits - fp16_exponent_bits);
6533   slli(tmp2, tmp2, fp16_mantissa_bits);
6534 
6535   // Preserve high order bit of float NaN in the
6536   // binary16 result NaN (tenth bit); OR in remaining
6537   // bits into lower 9 bits of binary 16 significand.
6538   //   | (doppel & 0x007f_e000) >> 13 // 10 bits
6539   //   | (doppel & 0x0000_1ff0) >> 4  //  9 bits
6540   //   | (doppel & 0x0000_000f));     //  4 bits
6541   //
6542   // Check j.l.Float.floatToFloat16 for more information.
6543   // 10 bits
6544   int left_shift = fp_sign_bits + fp32_exponent_bits + 32;
6545   int right_shift = left_shift + fp32_mantissa_2nd_part_bits + fp32_mantissa_3rd_part_bits;
6546   slli(tmp1, dst, left_shift);
6547   srli(tmp1, tmp1, right_shift);
6548   orr(tmp2, tmp2, tmp1);
6549   // 9 bits
6550   left_shift += fp32_mantissa_1st_part_bits;
6551   right_shift = left_shift + fp32_mantissa_3rd_part_bits;
6552   slli(tmp1, dst, left_shift);
6553   srli(tmp1, tmp1, right_shift);
6554   orr(tmp2, tmp2, tmp1);
6555   // 4 bits
6556   andi(tmp1, dst, 0xf);
6557   orr(dst, tmp2, tmp1);
6558 }
6559 
6560 #define FCVT_SAFE(FLOATCVT, FLOATSIG)                                                     \
6561 void MacroAssembler::FLOATCVT##_safe(Register dst, FloatRegister src, Register tmp) {     \
6562   Label done;                                                                             \
6563   assert_different_registers(dst, tmp);                                                   \
6564   fclass_##FLOATSIG(tmp, src);                                                            \
6565   mv(dst, zr);                                                                            \
6566   /* check if src is NaN */                                                               \
6567   andi(tmp, tmp, FClassBits::nan);                                                        \
6568   bnez(tmp, done);                                                                        \
6569   FLOATCVT(dst, src);                                                                     \
6570   bind(done);                                                                             \
6571 }
6572 
6573 FCVT_SAFE(fcvt_w_s, s);
6574 FCVT_SAFE(fcvt_l_s, s);
6575 FCVT_SAFE(fcvt_w_d, d);
6576 FCVT_SAFE(fcvt_l_d, d);
6577 
6578 #undef FCVT_SAFE
6579 
6580 #define FCMP(FLOATTYPE, FLOATSIG)                                                       \
6581 void MacroAssembler::FLOATTYPE##_compare(Register result, FloatRegister Rs1,            \
6582                                          FloatRegister Rs2, int unordered_result) {     \
6583   Label Ldone;                                                                          \
6584   if (unordered_result < 0) {                                                           \
6585     /* we want -1 for unordered or less than, 0 for equal and 1 for greater than. */    \
6586     /* installs 1 if gt else 0 */                                                       \
6587     flt_##FLOATSIG(result, Rs2, Rs1);                                                   \
6588     /* Rs1 > Rs2, install 1 */                                                          \
6589     bgtz(result, Ldone);                                                                \
6590     feq_##FLOATSIG(result, Rs1, Rs2);                                                   \
6591     subi(result, result, 1);                                                            \
6592     /* Rs1 = Rs2, install 0 */                                                          \
6593     /* NaN or Rs1 < Rs2, install -1 */                                                  \
6594     bind(Ldone);                                                                        \
6595   } else {                                                                              \
6596     /* we want -1 for less than, 0 for equal and 1 for unordered or greater than. */    \
6597     /* installs 1 if gt or unordered else 0 */                                          \
6598     flt_##FLOATSIG(result, Rs1, Rs2);                                                   \
6599     /* Rs1 < Rs2, install -1 */                                                         \
6600     bgtz(result, Ldone);                                                                \
6601     feq_##FLOATSIG(result, Rs1, Rs2);                                                   \
6602     subi(result, result, 1);                                                            \
6603     /* Rs1 = Rs2, install 0 */                                                          \
6604     /* NaN or Rs1 > Rs2, install 1 */                                                   \
6605     bind(Ldone);                                                                        \
6606     neg(result, result);                                                                \
6607   }                                                                                     \
6608 }
6609 
6610 FCMP(float, s);
6611 FCMP(double, d);
6612 
6613 #undef FCMP
6614 
6615 // Zero words; len is in bytes
6616 // Destroys all registers except addr
6617 // len must be a nonzero multiple of wordSize
6618 void MacroAssembler::zero_memory(Register addr, Register len, Register tmp) {
6619   assert_different_registers(addr, len, tmp, t0, t1);
6620 
6621 #ifdef ASSERT
6622   {
6623     Label L;
6624     andi(t0, len, BytesPerWord - 1);
6625     beqz(t0, L);
6626     stop("len is not a multiple of BytesPerWord");
6627     bind(L);
6628   }
6629 #endif // ASSERT
6630 
6631 #ifndef PRODUCT
6632   block_comment("zero memory");
6633 #endif // PRODUCT
6634 
6635   Label loop;
6636   Label entry;
6637 
6638   // Algorithm:
6639   //
6640   //  t0 = cnt & 7
6641   //  cnt -= t0
6642   //  p += t0
6643   //  switch (t0) {
6644   //    do {
6645   //      cnt -= 8
6646   //        p[-8] = 0
6647   //      case 7:
6648   //        p[-7] = 0
6649   //      case 6:
6650   //        p[-6] = 0
6651   //        ...
6652   //      case 1:
6653   //        p[-1] = 0
6654   //      case 0:
6655   //        p += 8
6656   //     } while (cnt)
6657   //  }
6658 
6659   const int unroll = 8;   // Number of sd(zr) instructions we'll unroll
6660 
6661   srli(len, len, LogBytesPerWord);
6662   andi(t0, len, unroll - 1);  // t0 = cnt % unroll
6663   sub(len, len, t0);          // cnt -= unroll
6664   // tmp always points to the end of the region we're about to zero
6665   shadd(tmp, t0, addr, t1, LogBytesPerWord);
6666   la(t1, entry);
6667   slli(t0, t0, 2);
6668   sub(t1, t1, t0);
6669   jr(t1);
6670 
6671   bind(loop);
6672   sub(len, len, unroll);
6673   {
6674     IncompressibleScope scope(this); // Fixed length
6675     for (int i = -unroll; i < 0; i++) {
6676       sd(zr, Address(tmp, i * wordSize));
6677     }
6678   }
6679   bind(entry);
6680   add(tmp, tmp, unroll * wordSize);
6681   bnez(len, loop);
6682 }
6683 
6684 // shift left by shamt and add
6685 // Rd = (Rs1 << shamt) + Rs2
6686 void MacroAssembler::shadd(Register Rd, Register Rs1, Register Rs2, Register tmp, int shamt) {
6687   if (UseZba) {
6688     if (shamt == 1) {
6689       sh1add(Rd, Rs1, Rs2);
6690       return;
6691     } else if (shamt == 2) {
6692       sh2add(Rd, Rs1, Rs2);
6693       return;
6694     } else if (shamt == 3) {
6695       sh3add(Rd, Rs1, Rs2);
6696       return;
6697     }
6698   }
6699 
6700   if (shamt != 0) {
6701     assert_different_registers(Rs2, tmp);
6702     slli(tmp, Rs1, shamt);
6703     add(Rd, Rs2, tmp);
6704   } else {
6705     add(Rd, Rs1, Rs2);
6706   }
6707 }
6708 
6709 void MacroAssembler::zext(Register dst, Register src, int bits) {
6710   switch (bits) {
6711     case 32:
6712       if (UseZba) {
6713         zext_w(dst, src);
6714         return;
6715       }
6716       break;
6717     case 16:
6718       if (UseZbb) {
6719         zext_h(dst, src);
6720         return;
6721       }
6722       break;
6723     case 8:
6724       zext_b(dst, src);
6725       return;
6726     default:
6727       break;
6728   }
6729 
6730   slli(dst, src, XLEN - bits);
6731   srli(dst, dst, XLEN - bits);
6732 }
6733 
6734 void MacroAssembler::sext(Register dst, Register src, int bits) {
6735   switch (bits) {
6736     case 32:
6737       sext_w(dst, src);
6738       return;
6739     case 16:
6740       if (UseZbb) {
6741         sext_h(dst, src);
6742         return;
6743       }
6744       break;
6745     case 8:
6746       if (UseZbb) {
6747         sext_b(dst, src);
6748         return;
6749       }
6750       break;
6751     default:
6752       break;
6753   }
6754 
6755   slli(dst, src, XLEN - bits);
6756   srai(dst, dst, XLEN - bits);
6757 }
6758 
6759 void MacroAssembler::cmp_x2i(Register dst, Register src1, Register src2,
6760                              Register tmp, bool is_signed) {
6761   if (src1 == src2) {
6762     mv(dst, zr);
6763     return;
6764   }
6765   Label done;
6766   Register left = src1;
6767   Register right = src2;
6768   if (dst == src1) {
6769     assert_different_registers(dst, src2, tmp);
6770     mv(tmp, src1);
6771     left = tmp;
6772   } else if (dst == src2) {
6773     assert_different_registers(dst, src1, tmp);
6774     mv(tmp, src2);
6775     right = tmp;
6776   }
6777 
6778   // installs 1 if gt else 0
6779   if (is_signed) {
6780     slt(dst, right, left);
6781   } else {
6782     sltu(dst, right, left);
6783   }
6784   bnez(dst, done);
6785   if (is_signed) {
6786     slt(dst, left, right);
6787   } else {
6788     sltu(dst, left, right);
6789   }
6790   // dst = -1 if lt; else if eq , dst = 0
6791   neg(dst, dst);
6792   bind(done);
6793 }
6794 
6795 void MacroAssembler::cmp_l2i(Register dst, Register src1, Register src2, Register tmp)
6796 {
6797   cmp_x2i(dst, src1, src2, tmp);
6798 }
6799 
6800 void MacroAssembler::cmp_ul2i(Register dst, Register src1, Register src2, Register tmp) {
6801   cmp_x2i(dst, src1, src2, tmp, false);
6802 }
6803 
6804 void MacroAssembler::cmp_uw2i(Register dst, Register src1, Register src2, Register tmp) {
6805   cmp_x2i(dst, src1, src2, tmp, false);
6806 }
6807 
6808 // The java_calling_convention describes stack locations as ideal slots on
6809 // a frame with no abi restrictions. Since we must observe abi restrictions
6810 // (like the placement of the register window) the slots must be biased by
6811 // the following value.
6812 static int reg2offset_in(VMReg r) {
6813   // Account for saved fp and ra
6814   // This should really be in_preserve_stack_slots
6815   return r->reg2stack() * VMRegImpl::stack_slot_size;
6816 }
6817 
6818 static int reg2offset_out(VMReg r) {
6819   return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
6820 }
6821 
6822 // The C ABI specifies:
6823 // "integer scalars narrower than XLEN bits are widened according to the sign
6824 // of their type up to 32 bits, then sign-extended to XLEN bits."
6825 // Applies for both passed in register and stack.
6826 //
6827 // Java uses 32-bit stack slots; jint, jshort, jchar, jbyte uses one slot.
6828 // Native uses 64-bit stack slots for all integer scalar types.
6829 //
6830 // lw loads the Java stack slot, sign-extends and
6831 // sd store this widened integer into a 64 bit native stack slot.
6832 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp) {
6833   if (src.first()->is_stack()) {
6834     if (dst.first()->is_stack()) {
6835       // stack to stack
6836       lw(tmp, Address(fp, reg2offset_in(src.first())));
6837       sd(tmp, Address(sp, reg2offset_out(dst.first())));
6838     } else {
6839       // stack to reg
6840       lw(dst.first()->as_Register(), Address(fp, reg2offset_in(src.first())));
6841     }
6842   } else if (dst.first()->is_stack()) {
6843     // reg to stack
6844     sd(src.first()->as_Register(), Address(sp, reg2offset_out(dst.first())));
6845   } else {
6846     if (dst.first() != src.first()) {
6847       sext(dst.first()->as_Register(), src.first()->as_Register(), 32);
6848     }
6849   }
6850 }
6851 
6852 // An oop arg. Must pass a handle not the oop itself
6853 void MacroAssembler::object_move(OopMap* map,
6854                                  int oop_handle_offset,
6855                                  int framesize_in_slots,
6856                                  VMRegPair src,
6857                                  VMRegPair dst,
6858                                  bool is_receiver,
6859                                  int* receiver_offset) {
6860   assert_cond(map != nullptr && receiver_offset != nullptr);
6861 
6862   // must pass a handle. First figure out the location we use as a handle
6863   Register rHandle = dst.first()->is_stack() ? t1 : dst.first()->as_Register();
6864 
6865   // See if oop is null if it is we need no handle
6866 
6867   if (src.first()->is_stack()) {
6868     // Oop is already on the stack as an argument
6869     int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
6870     map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots));
6871     if (is_receiver) {
6872       *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size;
6873     }
6874 
6875     ld(t0, Address(fp, reg2offset_in(src.first())));
6876     la(rHandle, Address(fp, reg2offset_in(src.first())));
6877     // conditionally move a null
6878     Label notZero1;
6879     bnez(t0, notZero1);
6880     mv(rHandle, zr);
6881     bind(notZero1);
6882   } else {
6883 
6884     // Oop is in a register we must store it to the space we reserve
6885     // on the stack for oop_handles and pass a handle if oop is non-null
6886 
6887     const Register rOop = src.first()->as_Register();
6888     int oop_slot = -1;
6889     if (rOop == j_rarg0) {
6890       oop_slot = 0;
6891     } else if (rOop == j_rarg1) {
6892       oop_slot = 1;
6893     } else if (rOop == j_rarg2) {
6894       oop_slot = 2;
6895     } else if (rOop == j_rarg3) {
6896       oop_slot = 3;
6897     } else if (rOop == j_rarg4) {
6898       oop_slot = 4;
6899     } else if (rOop == j_rarg5) {
6900       oop_slot = 5;
6901     } else if (rOop == j_rarg6) {
6902       oop_slot = 6;
6903     } else {
6904       assert(rOop == j_rarg7, "wrong register");
6905       oop_slot = 7;
6906     }
6907 
6908     oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset;
6909     int offset = oop_slot * VMRegImpl::stack_slot_size;
6910 
6911     map->set_oop(VMRegImpl::stack2reg(oop_slot));
6912     // Store oop in handle area, may be null
6913     sd(rOop, Address(sp, offset));
6914     if (is_receiver) {
6915       *receiver_offset = offset;
6916     }
6917 
6918     //rOop maybe the same as rHandle
6919     if (rOop == rHandle) {
6920       Label isZero;
6921       beqz(rOop, isZero);
6922       la(rHandle, Address(sp, offset));
6923       bind(isZero);
6924     } else {
6925       Label notZero2;
6926       la(rHandle, Address(sp, offset));
6927       bnez(rOop, notZero2);
6928       mv(rHandle, zr);
6929       bind(notZero2);
6930     }
6931   }
6932 
6933   // If arg is on the stack then place it otherwise it is already in correct reg.
6934   if (dst.first()->is_stack()) {
6935     sd(rHandle, Address(sp, reg2offset_out(dst.first())));
6936   }
6937 }
6938 
6939 // A float arg may have to do float reg int reg conversion
6940 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp) {
6941   assert((src.first()->is_stack() && dst.first()->is_stack()) ||
6942          (src.first()->is_reg() && dst.first()->is_reg()) ||
6943          (src.first()->is_stack() && dst.first()->is_reg()), "Unexpected error");
6944   if (src.first()->is_stack()) {
6945     if (dst.first()->is_stack()) {
6946       lwu(tmp, Address(fp, reg2offset_in(src.first())));
6947       sw(tmp, Address(sp, reg2offset_out(dst.first())));
6948     } else if (dst.first()->is_Register()) {
6949       lwu(dst.first()->as_Register(), Address(fp, reg2offset_in(src.first())));
6950     } else {
6951       ShouldNotReachHere();
6952     }
6953   } else if (src.first() != dst.first()) {
6954     if (src.is_single_phys_reg() && dst.is_single_phys_reg()) {
6955       fmv_s(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister());
6956     } else {
6957       ShouldNotReachHere();
6958     }
6959   }
6960 }
6961 
6962 // A long move
6963 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp) {
6964   if (src.first()->is_stack()) {
6965     if (dst.first()->is_stack()) {
6966       // stack to stack
6967       ld(tmp, Address(fp, reg2offset_in(src.first())));
6968       sd(tmp, Address(sp, reg2offset_out(dst.first())));
6969     } else {
6970       // stack to reg
6971       ld(dst.first()->as_Register(), Address(fp, reg2offset_in(src.first())));
6972     }
6973   } else if (dst.first()->is_stack()) {
6974     // reg to stack
6975     sd(src.first()->as_Register(), Address(sp, reg2offset_out(dst.first())));
6976   } else {
6977     if (dst.first() != src.first()) {
6978       mv(dst.first()->as_Register(), src.first()->as_Register());
6979     }
6980   }
6981 }
6982 
6983 // A double move
6984 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp) {
6985   assert((src.first()->is_stack() && dst.first()->is_stack()) ||
6986          (src.first()->is_reg() && dst.first()->is_reg()) ||
6987          (src.first()->is_stack() && dst.first()->is_reg()), "Unexpected error");
6988   if (src.first()->is_stack()) {
6989     if (dst.first()->is_stack()) {
6990       ld(tmp, Address(fp, reg2offset_in(src.first())));
6991       sd(tmp, Address(sp, reg2offset_out(dst.first())));
6992     } else if (dst.first()-> is_Register()) {
6993       ld(dst.first()->as_Register(), Address(fp, reg2offset_in(src.first())));
6994     } else {
6995       ShouldNotReachHere();
6996     }
6997   } else if (src.first() != dst.first()) {
6998     if (src.is_single_phys_reg() && dst.is_single_phys_reg()) {
6999       fmv_d(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister());
7000     } else {
7001       ShouldNotReachHere();
7002     }
7003   }
7004 }
7005 
7006 void MacroAssembler::test_bit(Register Rd, Register Rs, uint32_t bit_pos) {
7007   assert(bit_pos < 64, "invalid bit range");
7008   if (UseZbs) {
7009     bexti(Rd, Rs, bit_pos);
7010     return;
7011   }
7012   int64_t imm = (int64_t)(1UL << bit_pos);
7013   if (is_simm12(imm)) {
7014     andi(Rd, Rs, imm);
7015   } else {
7016     srli(Rd, Rs, bit_pos);
7017     andi(Rd, Rd, 1);
7018   }
7019 }
7020 
7021 // Implements fast-locking.
7022 //
7023 //  - obj: the object to be locked
7024 //  - tmp1, tmp2, tmp3: temporary registers, will be destroyed
7025 //  - slow: branched to if locking fails
7026 void MacroAssembler::fast_lock(Register basic_lock, Register obj, Register tmp1, Register tmp2, Register tmp3, Label& slow) {
7027   assert_different_registers(basic_lock, obj, tmp1, tmp2, tmp3, t0);
7028 
7029   Label push;
7030   const Register top = tmp1;
7031   const Register mark = tmp2;
7032   const Register t = tmp3;
7033 
7034   // Preload the markWord. It is important that this is the first
7035   // instruction emitted as it is part of C1's null check semantics.
7036   ld(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
7037 
7038   if (UseObjectMonitorTable) {
7039     // Clear cache in case fast locking succeeds or we need to take the slow-path.
7040     sd(zr, Address(basic_lock, BasicObjectLock::lock_offset() + in_ByteSize((BasicLock::object_monitor_cache_offset_in_bytes()))));
7041   }
7042 
7043   if (DiagnoseSyncOnValueBasedClasses != 0) {
7044     load_klass(tmp1, obj);
7045     lbu(tmp1, Address(tmp1, Klass::misc_flags_offset()));
7046     test_bit(tmp1, tmp1, exact_log2(KlassFlags::_misc_is_value_based_class));
7047     bnez(tmp1, slow, /* is_far */ true);
7048   }
7049 
7050   // Check if the lock-stack is full.
7051   lwu(top, Address(xthread, JavaThread::lock_stack_top_offset()));
7052   mv(t, (unsigned)LockStack::end_offset());
7053   bge(top, t, slow, /* is_far */ true);
7054 
7055   // Check for recursion.
7056   add(t, xthread, top);
7057   ld(t, Address(t, -oopSize));
7058   beq(obj, t, push);
7059 
7060   // Check header for monitor (0b10).
7061   test_bit(t, mark, exact_log2(markWord::monitor_value));
7062   bnez(t, slow, /* is_far */ true);
7063 
7064   // Try to lock. Transition lock-bits 0b01 => 0b00
7065   assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a la");
7066   ori(mark, mark, markWord::unlocked_value);
7067   // Mask inline_type bit such that we go to the slow path if object is an inline type
7068   andi(mark, mark, ~((int) markWord::inline_type_bit_in_place));
7069   xori(t, mark, markWord::unlocked_value);
7070   cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::int64,
7071           /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ t);
7072   bne(mark, t, slow, /* is_far */ true);
7073 
7074   bind(push);
7075   // After successful lock, push object on lock-stack.
7076   add(t, xthread, top);
7077   sd(obj, Address(t));
7078   addiw(top, top, oopSize);
7079   sw(top, Address(xthread, JavaThread::lock_stack_top_offset()));
7080 }
7081 
7082 // Implements ligthweight-unlocking.
7083 //
7084 // - obj: the object to be unlocked
7085 // - tmp1, tmp2, tmp3: temporary registers
7086 // - slow: branched to if unlocking fails
7087 void MacroAssembler::fast_unlock(Register obj, Register tmp1, Register tmp2, Register tmp3, Label& slow) {
7088   assert_different_registers(obj, tmp1, tmp2, tmp3, t0);
7089 
7090 #ifdef ASSERT
7091   {
7092     // Check for lock-stack underflow.
7093     Label stack_ok;
7094     lwu(tmp1, Address(xthread, JavaThread::lock_stack_top_offset()));
7095     mv(tmp2, (unsigned)LockStack::start_offset());
7096     bge(tmp1, tmp2, stack_ok);
7097     STOP("Lock-stack underflow");
7098     bind(stack_ok);
7099   }
7100 #endif
7101 
7102   Label unlocked, push_and_slow;
7103   const Register top = tmp1;
7104   const Register mark = tmp2;
7105   const Register t = tmp3;
7106 
7107   // Check if obj is top of lock-stack.
7108   lwu(top, Address(xthread, JavaThread::lock_stack_top_offset()));
7109   subiw(top, top, oopSize);
7110   add(t, xthread, top);
7111   ld(t, Address(t));
7112   bne(obj, t, slow, /* is_far */ true);
7113 
7114   // Pop lock-stack.
7115   DEBUG_ONLY(add(t, xthread, top);)
7116   DEBUG_ONLY(sd(zr, Address(t));)
7117   sw(top, Address(xthread, JavaThread::lock_stack_top_offset()));
7118 
7119   // Check if recursive.
7120   add(t, xthread, top);
7121   ld(t, Address(t, -oopSize));
7122   beq(obj, t, unlocked);
7123 
7124   // Not recursive. Check header for monitor (0b10).
7125   ld(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
7126   test_bit(t, mark, exact_log2(markWord::monitor_value));
7127   bnez(t, push_and_slow);
7128 
7129 #ifdef ASSERT
7130   // Check header not unlocked (0b01).
7131   Label not_unlocked;
7132   test_bit(t, mark, exact_log2(markWord::unlocked_value));
7133   beqz(t, not_unlocked);
7134   stop("fast_unlock already unlocked");
7135   bind(not_unlocked);
7136 #endif
7137 
7138   // Try to unlock. Transition lock bits 0b00 => 0b01
7139   assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
7140   ori(t, mark, markWord::unlocked_value);
7141   cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::int64,
7142           /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, /*result*/ t);
7143   beq(mark, t, unlocked);
7144 
7145   bind(push_and_slow);
7146   // Restore lock-stack and handle the unlock in runtime.
7147   DEBUG_ONLY(add(t, xthread, top);)
7148   DEBUG_ONLY(sd(obj, Address(t));)
7149   addiw(top, top, oopSize);
7150   sw(top, Address(xthread, JavaThread::lock_stack_top_offset()));
7151   j(slow);
7152 
7153   bind(unlocked);
7154 }
--- EOF ---