1 /*
   2  * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
   4  * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #ifndef CPU_RISCV_ASSEMBLER_RISCV_HPP
  28 #define CPU_RISCV_ASSEMBLER_RISCV_HPP
  29 
  30 #include "asm/register.hpp"
  31 #include "assembler_riscv.inline.hpp"
  32 
  33 #define registerSize 64
  34 
  35 // definitions of various symbolic names for machine registers
  36 
  37 // First intercalls between C and Java which use 8 general registers
  38 // and 8 floating registers
  39 
  40 class Argument {
  41  public:
  42   enum {
  43     n_int_register_parameters_c   = 8,  // x10, x11, ... x17 (c_rarg0, c_rarg1, ...)
  44     n_float_register_parameters_c = 8,  // f10, f11, ... f17 (c_farg0, c_farg1, ... )
  45 
  46     n_int_register_parameters_j   = 8, // x11, ... x17, x10 (rj_rarg0, j_rarg1, ...)
  47     n_float_register_parameters_j = 8  // f10, f11, ... f17 (j_farg0, j_farg1, ...)
  48   };
  49 };
  50 
  51 // function argument(caller-save registers)
  52 REGISTER_DECLARATION(Register, c_rarg0, x10);
  53 REGISTER_DECLARATION(Register, c_rarg1, x11);
  54 REGISTER_DECLARATION(Register, c_rarg2, x12);
  55 REGISTER_DECLARATION(Register, c_rarg3, x13);
  56 REGISTER_DECLARATION(Register, c_rarg4, x14);
  57 REGISTER_DECLARATION(Register, c_rarg5, x15);
  58 REGISTER_DECLARATION(Register, c_rarg6, x16);
  59 REGISTER_DECLARATION(Register, c_rarg7, x17);
  60 
  61 REGISTER_DECLARATION(FloatRegister, c_farg0, f10);
  62 REGISTER_DECLARATION(FloatRegister, c_farg1, f11);
  63 REGISTER_DECLARATION(FloatRegister, c_farg2, f12);
  64 REGISTER_DECLARATION(FloatRegister, c_farg3, f13);
  65 REGISTER_DECLARATION(FloatRegister, c_farg4, f14);
  66 REGISTER_DECLARATION(FloatRegister, c_farg5, f15);
  67 REGISTER_DECLARATION(FloatRegister, c_farg6, f16);
  68 REGISTER_DECLARATION(FloatRegister, c_farg7, f17);
  69 
  70 // java function register(caller-save registers)
  71 REGISTER_DECLARATION(Register, j_rarg0, c_rarg1);
  72 REGISTER_DECLARATION(Register, j_rarg1, c_rarg2);
  73 REGISTER_DECLARATION(Register, j_rarg2, c_rarg3);
  74 REGISTER_DECLARATION(Register, j_rarg3, c_rarg4);
  75 REGISTER_DECLARATION(Register, j_rarg4, c_rarg5);
  76 REGISTER_DECLARATION(Register, j_rarg5, c_rarg6);
  77 REGISTER_DECLARATION(Register, j_rarg6, c_rarg7);
  78 REGISTER_DECLARATION(Register, j_rarg7, c_rarg0);
  79 
  80 REGISTER_DECLARATION(FloatRegister, j_farg0, f10);
  81 REGISTER_DECLARATION(FloatRegister, j_farg1, f11);
  82 REGISTER_DECLARATION(FloatRegister, j_farg2, f12);
  83 REGISTER_DECLARATION(FloatRegister, j_farg3, f13);
  84 REGISTER_DECLARATION(FloatRegister, j_farg4, f14);
  85 REGISTER_DECLARATION(FloatRegister, j_farg5, f15);
  86 REGISTER_DECLARATION(FloatRegister, j_farg6, f16);
  87 REGISTER_DECLARATION(FloatRegister, j_farg7, f17);
  88 
  89 // zero rigster
  90 REGISTER_DECLARATION(Register, zr,        x0);
  91 // global pointer
  92 REGISTER_DECLARATION(Register, gp,        x3);
  93 // thread pointer
  94 REGISTER_DECLARATION(Register, tp,        x4);
  95 
  96 // volatile (caller-save) registers
  97 
  98 // current method -- must be in a call-clobbered register
  99 REGISTER_DECLARATION(Register, xmethod,   x31);
 100 // return address
 101 REGISTER_DECLARATION(Register, ra,        x1);
 102 // link rigster
 103 REGISTER_DECLARATION(Register, lr,        x1);
 104 
 105 
 106 // non-volatile (callee-save) registers
 107 
 108 // stack pointer
 109 REGISTER_DECLARATION(Register, sp,        x2);
 110 // frame pointer
 111 REGISTER_DECLARATION(Register, fp,        x8);
 112 // base of heap
 113 REGISTER_DECLARATION(Register, xheapbase, x27);
 114 // constant pool cache
 115 REGISTER_DECLARATION(Register, xcpool,    x26);
 116 // monitors allocated on stack
 117 REGISTER_DECLARATION(Register, xmonitors, x25);
 118 // locals on stack
 119 REGISTER_DECLARATION(Register, xlocals,   x24);
 120 
 121 /* If you use x4(tp) as java thread pointer according to the instruction manual,
 122  * it overlaps with the register used by c++ thread.
 123  */
 124 // java thread pointer
 125 REGISTER_DECLARATION(Register, xthread,   x23);
 126 // bytecode pointer
 127 REGISTER_DECLARATION(Register, xbcp,      x22);
 128 // Dispatch table base
 129 REGISTER_DECLARATION(Register, xdispatch, x21);
 130 // Java stack pointer
 131 REGISTER_DECLARATION(Register, esp,       x20);
 132 
 133 // tempory register(caller-save registers)
 134 REGISTER_DECLARATION(Register, t0, x5);
 135 REGISTER_DECLARATION(Register, t1, x6);
 136 REGISTER_DECLARATION(Register, t2, x7);
 137 
 138 const Register g_INTArgReg[Argument::n_int_register_parameters_c] = {
 139   c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5,  c_rarg6,  c_rarg7
 140 };
 141 
 142 const FloatRegister g_FPArgReg[Argument::n_float_register_parameters_c] = {
 143   c_farg0, c_farg1, c_farg2, c_farg3, c_farg4, c_farg5, c_farg6, c_farg7
 144 };
 145 
 146 #define assert_cond(ARG1) assert(ARG1, #ARG1)
 147 
 148 // Addressing modes
 149 class Address {
 150  public:
 151 
 152   enum mode { no_mode, base_plus_offset, pcrel, literal };
 153 
 154  private:
 155   Register _base;
 156   int64_t _offset;
 157   enum mode _mode;
 158 
 159   RelocationHolder _rspec;
 160 
 161   // If the target is far we'll need to load the ea of this to a
 162   // register to reach it. Otherwise if near we can do PC-relative
 163   // addressing.
 164   address          _target;
 165 
 166  public:
 167   Address()
 168     : _base(noreg), _offset(0), _mode(no_mode),          _target(NULL) { }
 169   Address(Register r)
 170     : _base(r),     _offset(0), _mode(base_plus_offset), _target(NULL) { }
 171   Address(Register r, int o)
 172     : _base(r),     _offset(o), _mode(base_plus_offset), _target(NULL) { }
 173   Address(Register r, long o)
 174     : _base(r),     _offset(o), _mode(base_plus_offset), _target(NULL) { }
 175   Address(Register r, long long o)
 176     : _base(r),     _offset(o), _mode(base_plus_offset), _target(NULL) { }
 177   Address(Register r, unsigned int o)
 178     : _base(r),     _offset(o), _mode(base_plus_offset), _target(NULL) { }
 179   Address(Register r, unsigned long o)
 180     : _base(r),     _offset(o), _mode(base_plus_offset), _target(NULL) { }
 181   Address(Register r, unsigned long long o)
 182     : _base(r),     _offset(o), _mode(base_plus_offset), _target(NULL) { }
 183   Address(Register r, ByteSize disp)
 184     : Address(r, in_bytes(disp)) {}
 185   Address(address target, RelocationHolder const& rspec)
 186     : _base(noreg),
 187       _offset(0),
 188       _mode(literal),
 189       _rspec(rspec),
 190       _target(target) { }
 191   Address(address target, relocInfo::relocType rtype = relocInfo::external_word_type);
 192 
 193   const Register base() const {
 194     guarantee((_mode == base_plus_offset | _mode == pcrel | _mode == literal), "wrong mode");
 195     return _base;
 196   }
 197   long offset() const {
 198     return _offset;
 199   }
 200 
 201   mode getMode() const {
 202     return _mode;
 203   }
 204 
 205   bool uses(Register reg) const { return _base == reg;}
 206   const address target() const { return _target; }
 207   const RelocationHolder& rspec() const { return _rspec; }
 208   ~Address() {
 209     _target = NULL;
 210     _base = NULL;
 211   }
 212 };
 213 
 214 // Convience classes
 215 class RuntimeAddress: public Address {
 216 
 217   public:
 218 
 219   RuntimeAddress(address target) : Address(target, relocInfo::runtime_call_type) {}
 220   ~RuntimeAddress() {}
 221 };
 222 
 223 class OopAddress: public Address {
 224 
 225   public:
 226 
 227   OopAddress(address target) : Address(target, relocInfo::oop_type) {}
 228   ~OopAddress() {}
 229 };
 230 
 231 class ExternalAddress: public Address {
 232  private:
 233   static relocInfo::relocType reloc_for_target(address target) {
 234     // Sometimes ExternalAddress is used for values which aren't
 235     // exactly addresses, like the card table base.
 236     // external_word_type can't be used for values in the first page
 237     // so just skip the reloc in that case.
 238     return external_word_Relocation::can_be_relocated(target) ? relocInfo::external_word_type : relocInfo::none;
 239   }
 240 
 241  public:
 242 
 243   ExternalAddress(address target) : Address(target, reloc_for_target(target)) {}
 244   ~ExternalAddress() {}
 245 };
 246 
 247 class InternalAddress: public Address {
 248 
 249   public:
 250 
 251   InternalAddress(address target) : Address(target, relocInfo::internal_word_type) {}
 252   ~InternalAddress() {}
 253 };
 254 
 255 class Assembler : public AbstractAssembler {
 256 public:
 257 
 258   enum { instruction_size = 4 };
 259 
 260   //---<  calculate length of instruction  >---
 261   // We just use the values set above.
 262   // instruction must start at passed address
 263   static unsigned int instr_len(unsigned char *instr) { return instruction_size; }
 264 
 265   //---<  longest instructions  >---
 266   static unsigned int instr_maxlen() { return instruction_size; }
 267 
 268   enum RoundingMode {
 269     rne = 0b000,     // round to Nearest, ties to Even
 270     rtz = 0b001,     // round towards Zero
 271     rdn = 0b010,     // round Down (towards eegative infinity)
 272     rup = 0b011,     // round Up (towards infinity)
 273     rmm = 0b100,     // round to Nearest, ties to Max Magnitude
 274     rdy = 0b111,     // in instruction's rm field, selects dynamic rounding mode.In Rounding Mode register, Invalid.
 275   };
 276 
 277   void baseOffset32(Register temp, const Address &adr, int32_t &offset) {
 278     assert(temp != noreg, "temp must not be empty register!");
 279     guarantee(adr.base() != temp, "should use different registers!");
 280     if (is_offset_in_range(adr.offset(), 32)) {
 281       int32_t imm = adr.offset();
 282       int32_t upper = imm, lower = imm;
 283       lower = (imm << 20) >> 20;
 284       upper -= lower;
 285       lui(temp, upper);
 286       offset = lower;
 287     } else {
 288       movptr_with_offset(temp, (address)(uintptr_t)adr.offset(), offset);
 289     }
 290     add(temp, temp, adr.base());
 291   }
 292 
 293   void baseOffset(Register temp, const Address &adr, int32_t &offset) {
 294     if (is_offset_in_range(adr.offset(), 12)) {
 295       assert(temp != noreg, "temp must not be empty register!");
 296       addi(temp, adr.base(), adr.offset());
 297       offset = 0;
 298     } else {
 299       baseOffset32(temp, adr, offset);
 300     }
 301   }
 302 
 303   void li(Register Rd, int64_t imm);  // optimized load immediate
 304   void li32(Register Rd, int32_t imm);
 305   void li64(Register Rd, int64_t imm);
 306   void movptr(Register Rd, address addr);
 307   void movptr_with_offset(Register Rd, address addr, int32_t &offset);
 308   void movptr(Register Rd, uintptr_t imm64);
 309   void ifence();
 310   void j(const address &dest, Register temp = t0);
 311   void j(const Address &adr, Register temp = t0) ;
 312   void j(Label &l, Register temp = t0);
 313   void jal(Label &l, Register temp = t0);
 314   void jal(const address &dest, Register temp = t0);
 315   void jal(const Address &adr, Register temp = t0);
 316   void jr(Register Rs);
 317   void jalr(Register Rs);
 318   void ret();
 319   void call(const address &dest, Register temp = t0);
 320   void call(const Address &adr, Register temp = t0);
 321   void tail(const address &dest, Register temp = t0);
 322   void tail(const Address &adr, Register temp = t0);
 323   void call(Label &l, Register temp) {
 324     call(target(l), temp);
 325   }
 326   void tail(Label &l, Register temp) {
 327     tail(target(l), temp);
 328   }
 329 
 330   static inline uint32_t extract(uint32_t val, unsigned msb, unsigned lsb) {
 331     assert_cond(msb >= lsb && msb <= 31);
 332     unsigned nbits = msb - lsb + 1;
 333     uint32_t mask = (1U << nbits) - 1;
 334     uint32_t result = val >> lsb;
 335     result &= mask;
 336     return result;
 337   }
 338 
 339   static inline int32_t sextract(uint32_t val, unsigned msb, unsigned lsb) {
 340     assert_cond(msb >= lsb && msb <= 31);
 341     int32_t result = val << (31 - msb);
 342     result >>= (31 - msb + lsb);
 343     return result;
 344   }
 345 
 346   static void patch(address a, unsigned msb, unsigned lsb, unsigned val) {
 347     assert_cond(a != NULL);
 348     assert_cond(msb >= lsb && msb <= 31);
 349     unsigned nbits = msb - lsb + 1;
 350     guarantee(val < (1U << nbits), "Field too big for insn");
 351     unsigned mask = (1U << nbits) - 1;
 352     val <<= lsb;
 353     mask <<= lsb;
 354     unsigned target = *(unsigned *)a;
 355     target &= ~mask;
 356     target |= val;
 357     *(unsigned *)a = target;
 358   }
 359 
 360   static void patch(address a, unsigned bit, unsigned val) {
 361     patch(a, bit, bit, val);
 362   }
 363 
 364   static void patch_reg(address a, unsigned lsb, Register reg) {
 365     patch(a, lsb + 4, lsb, reg->encoding_nocheck());
 366   }
 367 
 368   static void patch_reg(address a, unsigned lsb, FloatRegister reg) {
 369     patch(a, lsb + 4, lsb, reg->encoding_nocheck());
 370   }
 371 
 372   static void patch_reg(address a, unsigned lsb, VectorRegister reg) {
 373     patch(a, lsb + 4, lsb, reg->encoding_nocheck());
 374   }
 375 
 376   void emit(unsigned insn) {
 377     emit_int32((jint)insn);
 378   }
 379 
 380   void halt() {
 381     emit_int32(0);
 382   }
 383 
 384 // Rigster Instruction
 385 #define INSN(NAME, op, funct3, funct7)                          \
 386   void NAME(Register Rd, Register Rs1, Register Rs2) {          \
 387     unsigned insn = 0;                                          \
 388     patch((address)&insn, 6,  0, op);                           \
 389     patch((address)&insn, 14, 12, funct3);                      \
 390     patch((address)&insn, 31, 25, funct7);                      \
 391     patch_reg((address)&insn, 7, Rd);                           \
 392     patch_reg((address)&insn, 15, Rs1);                         \
 393     patch_reg((address)&insn, 20, Rs2);                         \
 394     emit(insn);                                                 \
 395   }
 396 
 397   INSN(add,   0b0110011, 0b000, 0b0000000);
 398   INSN(sub,   0b0110011, 0b000, 0b0100000);
 399   INSN(andr,  0b0110011, 0b111, 0b0000000);
 400   INSN(orr,   0b0110011, 0b110, 0b0000000);
 401   INSN(xorr,  0b0110011, 0b100, 0b0000000);
 402   INSN(sll,   0b0110011, 0b001, 0b0000000);
 403   INSN(sra,   0b0110011, 0b101, 0b0100000);
 404   INSN(srl,   0b0110011, 0b101, 0b0000000);
 405   INSN(slt,   0b0110011, 0b010, 0b0000000);
 406   INSN(sltu,  0b0110011, 0b011, 0b0000000);
 407   INSN(addw,  0b0111011, 0b000, 0b0000000);
 408   INSN(subw,  0b0111011, 0b000, 0b0100000);
 409   INSN(sllw,  0b0111011, 0b001, 0b0000000);
 410   INSN(sraw,  0b0111011, 0b101, 0b0100000);
 411   INSN(srlw,  0b0111011, 0b101, 0b0000000);
 412   INSN(mul,   0b0110011, 0b000, 0b0000001);
 413   INSN(mulh,  0b0110011, 0b001, 0b0000001);
 414   INSN(mulhsu,0b0110011, 0b010, 0b0000001);
 415   INSN(mulhu, 0b0110011, 0b011, 0b0000001);
 416   INSN(mulw,  0b0111011, 0b000, 0b0000001);
 417   INSN(div,   0b0110011, 0b100, 0b0000001);
 418   INSN(divu,  0b0110011, 0b101, 0b0000001);
 419   INSN(divw,  0b0111011, 0b100, 0b0000001);
 420   INSN(divuw, 0b0111011, 0b101, 0b0000001);
 421   INSN(rem,   0b0110011, 0b110, 0b0000001);
 422   INSN(remu,  0b0110011, 0b111, 0b0000001);
 423   INSN(remw,  0b0111011, 0b110, 0b0000001);
 424   INSN(remuw, 0b0111011, 0b111, 0b0000001);
 425 
 426   // Vector Configuration Instruction
 427   INSN(vsetvl, 0b1010111, 0b111, 0b1000000);
 428 
 429 #undef INSN
 430 
 431 #define INSN_ENTRY_RELOC(result_type, header)                               \
 432   result_type header {                                                      \
 433     InstructionMark im(this);                                               \
 434     guarantee(rtype == relocInfo::internal_word_type,                       \
 435               "only internal_word_type relocs make sense here");            \
 436     code_section()->relocate(inst_mark(), InternalAddress(dest).rspec());
 437 
 438   // Load/store register (all modes)
 439 #define INSN(NAME, op, funct3)                                                                     \
 440   void NAME(Register Rd, Register Rs, const int32_t offset) {                                      \
 441     unsigned insn = 0;                                                                             \
 442     guarantee(is_offset_in_range(offset, 12), "offset is invalid.");                               \
 443     int32_t val = offset & 0xfff;                                                                  \
 444     patch((address)&insn, 6, 0, op);                                                               \
 445     patch((address)&insn, 14, 12, funct3);                                                         \
 446     patch_reg((address)&insn, 15, Rs);                                                             \
 447     patch_reg((address)&insn, 7, Rd);                                                              \
 448     patch((address)&insn, 31, 20, val);                                                            \
 449     emit(insn);                                                                                    \
 450   }                                                                                                \
 451   void NAME(Register Rd, address dest) {                                                           \
 452     assert_cond(dest != NULL);                                                                     \
 453     int64_t distance = (dest - pc());                                                              \
 454     if (is_offset_in_range(distance, 32)) {                                                        \
 455       auipc(Rd, (int32_t)distance + 0x800);                                                        \
 456       NAME(Rd, Rd, ((int32_t)distance << 20) >> 20);                                               \
 457     } else {                                                                                       \
 458       int32_t offset = 0;                                                                          \
 459       movptr_with_offset(Rd, dest, offset);                                                        \
 460       NAME(Rd, Rd, offset);                                                                        \
 461     }                                                                                              \
 462   }                                                                                                \
 463   INSN_ENTRY_RELOC(void, NAME(Register Rd, address dest, relocInfo::relocType rtype))              \
 464     NAME(Rd, dest);                                                                                \
 465   }                                                                                                \
 466   void NAME(Register Rd, const Address &adr, Register temp = t0) {                                 \
 467     switch(adr.getMode()) {                                                                        \
 468       case Address::literal: {                                                                     \
 469         code_section()->relocate(pc(), adr.rspec());                                               \
 470         NAME(Rd, adr.target());                                                                    \
 471         break;                                                                                     \
 472       }                                                                                            \
 473       case Address::base_plus_offset:{                                                             \
 474         if (is_offset_in_range(adr.offset(), 12)) {                                                \
 475           NAME(Rd, adr.base(), adr.offset());                                                      \
 476         } else {                                                                                   \
 477           int32_t offset = 0;                                                                      \
 478           if (Rd == adr.base()) {                                                                  \
 479             baseOffset32(temp, adr, offset);                                                       \
 480             NAME(Rd, temp, offset);                                                                \
 481           } else {                                                                                 \
 482             baseOffset32(Rd, adr, offset);                                                         \
 483             NAME(Rd, Rd, offset);                                                                  \
 484           }                                                                                        \
 485         }                                                                                          \
 486         break;                                                                                     \
 487       }                                                                                            \
 488       default:                                                                                     \
 489         ShouldNotReachHere();                                                                      \
 490     }                                                                                              \
 491   }                                                                                                \
 492   void NAME(Register Rd, Label &L) {                                                               \
 493     wrap_label(Rd, L, &Assembler::NAME);                                                           \
 494   }
 495 
 496   INSN(lb,  0b0000011, 0b000);
 497   INSN(lbu, 0b0000011, 0b100);
 498   INSN(ld,  0b0000011, 0b011);
 499   INSN(lh,  0b0000011, 0b001);
 500   INSN(lhu, 0b0000011, 0b101);
 501   INSN(lw,  0b0000011, 0b010);
 502   INSN(lwu, 0b0000011, 0b110);
 503 
 504 #undef INSN
 505 
 506 #define INSN(NAME, op, funct3)                                                                     \
 507   void NAME(FloatRegister Rd, Register Rs, const int32_t offset) {                                 \
 508     unsigned insn = 0;                                                                             \
 509     guarantee(is_offset_in_range(offset, 12), "offset is invalid.");                               \
 510     uint32_t val = offset & 0xfff;                                                                 \
 511     patch((address)&insn, 6, 0, op);                                                               \
 512     patch((address)&insn, 14, 12, funct3);                                                         \
 513     patch_reg((address)&insn, 15, Rs);                                                             \
 514     patch_reg((address)&insn, 7, Rd);                                                              \
 515     patch((address)&insn, 31, 20, val);                                                            \
 516     emit(insn);                                                                                    \
 517   }                                                                                                \
 518   void NAME(FloatRegister Rd, address dest, Register temp = t0) {                                  \
 519     assert_cond(dest != NULL);                                                                     \
 520     int64_t distance = (dest - pc());                                                              \
 521     if (is_offset_in_range(distance, 32)) {                                                        \
 522       auipc(temp, (int32_t)distance + 0x800);                                                      \
 523       NAME(Rd, temp, ((int32_t)distance << 20) >> 20);                                             \
 524     } else {                                                                                       \
 525       int32_t offset = 0;                                                                          \
 526       movptr_with_offset(temp, dest, offset);                                                      \
 527       NAME(Rd, temp, offset);                                                                      \
 528     }                                                                                              \
 529   }                                                                                                \
 530   INSN_ENTRY_RELOC(void, NAME(FloatRegister Rd, address dest, relocInfo::relocType rtype, Register temp = t0)) \
 531     NAME(Rd, dest, temp);                                                                          \
 532   }                                                                                                \
 533   void NAME(FloatRegister Rd, const Address &adr, Register temp = t0) {                            \
 534     switch(adr.getMode()) {                                                                        \
 535       case Address::literal: {                                                                     \
 536         code_section()->relocate(pc(), adr.rspec());                                               \
 537         NAME(Rd, adr.target(), temp);                                                              \
 538         break;                                                                                     \
 539       }                                                                                            \
 540       case Address::base_plus_offset:{                                                             \
 541         if (is_offset_in_range(adr.offset(), 12)) {                                                \
 542           NAME(Rd, adr.base(), adr.offset());                                                      \
 543         } else {                                                                                   \
 544           int32_t offset = 0;                                                                      \
 545           baseOffset32(temp, adr, offset);                                                         \
 546           NAME(Rd, temp, offset);                                                                  \
 547         }                                                                                          \
 548         break;                                                                                     \
 549       }                                                                                            \
 550       default:                                                                                     \
 551         ShouldNotReachHere();                                                                      \
 552     }                                                                                              \
 553   }
 554 
 555   INSN(flw, 0b0000111, 0b010);
 556   INSN(fld, 0b0000111, 0b011);
 557 #undef INSN
 558 
 559 #define INSN(NAME, op, funct3)                                                                           \
 560   void NAME(Register Rs1, Register Rs2, const int64_t offset) {                                          \
 561     unsigned insn = 0;                                                                                   \
 562     guarantee(is_imm_in_range(offset, 12, 1), "offset is invalid.");                                     \
 563     uint32_t val  = offset & 0x1fff;                                                                     \
 564     uint32_t val11 = (val >> 11) & 0x1;                                                                  \
 565     uint32_t val12 = (val >> 12) & 0x1;                                                                  \
 566     uint32_t low  = (val >> 1) & 0xf;                                                                    \
 567     uint32_t high = (val >> 5) & 0x3f;                                                                   \
 568     patch((address)&insn, 6, 0, op);                                                                     \
 569     patch((address)&insn, 14, 12, funct3);                                                               \
 570     patch_reg((address)&insn, 15, Rs1);                                                                  \
 571     patch_reg((address)&insn, 20, Rs2);                                                                  \
 572     patch((address)&insn, 7, val11);                                                                     \
 573     patch((address)&insn, 11, 8, low);                                                                   \
 574     patch((address)&insn, 30, 25, high);                                                                 \
 575     patch((address)&insn, 31, val12);                                                                    \
 576     emit(insn);                                                                                          \
 577   }                                                                                                      \
 578   void NAME(Register Rs1, Register Rs2, const address dest) {                                            \
 579     assert_cond(dest != NULL);                                                                           \
 580     int64_t offset = (dest - pc());                                                                      \
 581     guarantee(is_imm_in_range(offset, 12, 1), "offset is invalid.");                                     \
 582     NAME(Rs1, Rs2, offset);                                                                              \
 583   }                                                                                                      \
 584   INSN_ENTRY_RELOC(void, NAME(Register Rs1, Register Rs2, address dest, relocInfo::relocType rtype))     \
 585     NAME(Rs1, Rs2, dest);                                                                                \
 586   }
 587 
 588   INSN(beq,  0b1100011, 0b000);
 589   INSN(bge,  0b1100011, 0b101);
 590   INSN(bgeu, 0b1100011, 0b111);
 591   INSN(blt,  0b1100011, 0b100);
 592   INSN(bltu, 0b1100011, 0b110);
 593   INSN(bne,  0b1100011, 0b001);
 594 
 595 #undef INSN
 596 
 597 #define INSN(NAME, NEG_INSN)                                                                \
 598   void NAME(Register Rs1, Register Rs2, Label &L, bool is_far = false) {                    \
 599     wrap_label(Rs1, Rs2, L, &Assembler::NAME, &Assembler::NEG_INSN, is_far);                \
 600   }
 601 
 602   INSN(beq,  bne);
 603   INSN(bne,  beq);
 604   INSN(blt,  bge);
 605   INSN(bge,  blt);
 606   INSN(bltu, bgeu);
 607   INSN(bgeu, bltu);
 608 
 609 #undef INSN
 610 
 611 #define INSN(NAME, REGISTER, op, funct3)                                                                    \
 612   void NAME(REGISTER Rs1, Register Rs2, const int32_t offset) {                                             \
 613     unsigned insn = 0;                                                                                      \
 614     guarantee(is_offset_in_range(offset, 12), "offset is invalid.");                                        \
 615     uint32_t val  = offset & 0xfff;                                                                         \
 616     uint32_t low  = val & 0x1f;                                                                             \
 617     uint32_t high = (val >> 5) & 0x7f;                                                                      \
 618     patch((address)&insn, 6, 0, op);                                                                        \
 619     patch((address)&insn, 14, 12, funct3);                                                                  \
 620     patch_reg((address)&insn, 15, Rs2);                                                                     \
 621     patch_reg((address)&insn, 20, Rs1);                                                                     \
 622     patch((address)&insn, 11, 7, low);                                                                      \
 623     patch((address)&insn, 31, 25, high);                                                                    \
 624     emit(insn);                                                                                             \
 625   }                                                                                                         \
 626   INSN_ENTRY_RELOC(void, NAME(REGISTER Rs, address dest, relocInfo::relocType rtype, Register temp = t0))   \
 627     NAME(Rs, dest, temp);                                                                                   \
 628   }
 629 
 630   INSN(sb,  Register,      0b0100011, 0b000);
 631   INSN(sh,  Register,      0b0100011, 0b001);
 632   INSN(sw,  Register,      0b0100011, 0b010);
 633   INSN(sd,  Register,      0b0100011, 0b011);
 634   INSN(fsw, FloatRegister, 0b0100111, 0b010);
 635   INSN(fsd, FloatRegister, 0b0100111, 0b011);
 636 
 637 #undef INSN
 638 
 639 #define INSN(NAME)                                                                                 \
 640   void NAME(Register Rs, address dest, Register temp = t0) {                                       \
 641     assert_cond(dest != NULL);                                                                     \
 642     assert_different_registers(Rs, temp);                                                          \
 643     int64_t distance = (dest - pc());                                                              \
 644     if (is_offset_in_range(distance, 32)) {                                                        \
 645       auipc(temp, (int32_t)distance + 0x800);                                                      \
 646       NAME(Rs, temp, ((int32_t)distance << 20) >> 20);                                             \
 647     } else {                                                                                       \
 648       int32_t offset = 0;                                                                          \
 649       movptr_with_offset(temp, dest, offset);                                                      \
 650       NAME(Rs, temp, offset);                                                                      \
 651     }                                                                                              \
 652   }                                                                                                \
 653   void NAME(Register Rs, const Address &adr, Register temp = t0) {                                 \
 654     switch(adr.getMode()) {                                                                        \
 655       case Address::literal: {                                                                     \
 656         assert_different_registers(Rs, temp);                                                      \
 657         code_section()->relocate(pc(), adr.rspec());                                               \
 658         NAME(Rs, adr.target(), temp);                                                              \
 659         break;                                                                                     \
 660       }                                                                                            \
 661       case Address::base_plus_offset:{                                                             \
 662         if (is_offset_in_range(adr.offset(), 12)) {                                                \
 663           NAME(Rs, adr.base(), adr.offset());                                                      \
 664         } else {                                                                                   \
 665           int32_t offset= 0;                                                                       \
 666           assert_different_registers(Rs, temp);                                                    \
 667           baseOffset32(temp, adr, offset);                                                         \
 668           NAME(Rs, temp, offset);                                                                  \
 669         }                                                                                          \
 670         break;                                                                                     \
 671       }                                                                                            \
 672       default:                                                                                     \
 673         ShouldNotReachHere();                                                                      \
 674     }                                                                                              \
 675   }
 676 
 677   INSN(sb);
 678   INSN(sh);
 679   INSN(sw);
 680   INSN(sd);
 681 
 682 #undef INSN
 683 
 684 #define INSN(NAME)                                                                                 \
 685   void NAME(FloatRegister Rs, address dest, Register temp = t0) {                                  \
 686     assert_cond(dest != NULL);                                                                     \
 687     int64_t distance = (dest - pc());                                                              \
 688     if (is_offset_in_range(distance, 32)) {                                                        \
 689       auipc(temp, (int32_t)distance + 0x800);                                                      \
 690       NAME(Rs, temp, ((int32_t)distance << 20) >> 20);                                             \
 691     } else {                                                                                       \
 692       int32_t offset = 0;                                                                          \
 693       movptr_with_offset(temp, dest, offset);                                                      \
 694       NAME(Rs, temp, offset);                                                                      \
 695     }                                                                                              \
 696   }                                                                                                \
 697   void NAME(FloatRegister Rs, const Address &adr, Register temp = t0) {                            \
 698     switch(adr.getMode()) {                                                                        \
 699       case Address::literal: {                                                                     \
 700         code_section()->relocate(pc(), adr.rspec());                                               \
 701         NAME(Rs, adr.target(), temp);                                                              \
 702         break;                                                                                     \
 703       }                                                                                            \
 704       case Address::base_plus_offset:{                                                             \
 705         if (is_offset_in_range(adr.offset(), 12)) {                                                \
 706           NAME(Rs, adr.base(), adr.offset());                                                      \
 707         } else {                                                                                   \
 708           int32_t offset = 0;                                                                      \
 709           baseOffset32(temp, adr, offset);                                                         \
 710           NAME(Rs, temp, offset);                                                                  \
 711         }                                                                                          \
 712         break;                                                                                     \
 713       }                                                                                            \
 714       default:                                                                                     \
 715         ShouldNotReachHere();                                                                      \
 716     }                                                                                              \
 717   }
 718 
 719   INSN(fsw);
 720   INSN(fsd);
 721 
 722 #undef INSN
 723 
 724 #define INSN(NAME, op, funct3)                                                        \
 725   void NAME(Register Rd, const uint32_t csr, Register Rs1) {                          \
 726     guarantee(is_unsigned_imm_in_range(csr, 12, 0), "csr is invalid");                \
 727     unsigned insn = 0;                                                                \
 728     patch((address)&insn, 6, 0, op);                                                  \
 729     patch((address)&insn, 14, 12, funct3);                                            \
 730     patch_reg((address)&insn, 7, Rd);                                                 \
 731     patch_reg((address)&insn, 15, Rs1);                                               \
 732     patch((address)&insn, 31, 20, csr);                                               \
 733     emit(insn);                                                                       \
 734   }
 735 
 736   INSN(csrrw, 0b1110011, 0b001);
 737   INSN(csrrs, 0b1110011, 0b010);
 738   INSN(csrrc, 0b1110011, 0b011);
 739 
 740 #undef INSN
 741 
 742 #define INSN(NAME, op, funct3)                                                        \
 743   void NAME(Register Rd, const uint32_t csr, const uint32_t uimm) {                   \
 744     guarantee(is_unsigned_imm_in_range(csr, 12, 0), "csr is invalid");                \
 745     guarantee(is_unsigned_imm_in_range(uimm, 5, 0), "uimm is invalid");               \
 746     unsigned insn = 0;                                                                \
 747     uint32_t val  = uimm & 0x1f;                                                      \
 748     patch((address)&insn, 6, 0, op);                                                  \
 749     patch((address)&insn, 14, 12, funct3);                                            \
 750     patch_reg((address)&insn, 7, Rd);                                                 \
 751     patch((address)&insn, 19, 15, val);                                               \
 752     patch((address)&insn, 31, 20, csr);                                               \
 753     emit(insn);                                                                       \
 754   }
 755 
 756   INSN(csrrwi, 0b1110011, 0b101);
 757   INSN(csrrsi, 0b1110011, 0b110);
 758   INSN(csrrci, 0b1110011, 0b111);
 759 
 760 #undef INSN
 761 
 762 #define INSN(NAME, op)                                                                        \
 763   void NAME(Register Rd, const int32_t offset) {                                              \
 764     unsigned insn = 0;                                                                        \
 765     guarantee(is_imm_in_range(offset, 20, 1), "offset is invalid.");                          \
 766     patch((address)&insn, 6, 0, op);                                                          \
 767     patch_reg((address)&insn, 7, Rd);                                                         \
 768     patch((address)&insn, 19, 12, (uint32_t)((offset >> 12) & 0xff));                         \
 769     patch((address)&insn, 20, (uint32_t)((offset >> 11) & 0x1));                              \
 770     patch((address)&insn, 30, 21, (uint32_t)((offset >> 1) & 0x3ff));                         \
 771     patch((address)&insn, 31, (uint32_t)((offset >> 20) & 0x1));                              \
 772     emit(insn);                                                                               \
 773   }                                                                                           \
 774   void NAME(Register Rd, const address dest, Register temp = t0) {                            \
 775     assert_cond(dest != NULL);                                                                \
 776     int64_t offset = dest - pc();                                                             \
 777     if (is_imm_in_range(offset, 20, 1)) {                                                     \
 778       NAME(Rd, offset);                                                                       \
 779     } else {                                                                                  \
 780       assert_different_registers(Rd, temp);                                                   \
 781       int32_t off = 0;                                                                        \
 782       movptr_with_offset(temp, dest, off);                                                    \
 783       jalr(Rd, temp, off);                                                                    \
 784     }                                                                                         \
 785   }                                                                                           \
 786   void NAME(Register Rd, Label &L, Register temp = t0) {                                      \
 787     assert_different_registers(Rd, temp);                                                     \
 788     wrap_label(Rd, L, temp, &Assembler::NAME);                                                \
 789   }
 790 
 791   INSN(jal, 0b1101111);
 792 
 793 #undef INSN
 794 
 795 #undef INSN_ENTRY_RELOC
 796 
 797 #define INSN(NAME, op, funct)                                                              \
 798   void NAME(Register Rd, Register Rs, const int32_t offset) {                              \
 799     unsigned insn = 0;                                                                     \
 800     guarantee(is_offset_in_range(offset, 12), "offset is invalid.");                       \
 801     patch((address)&insn, 6, 0, op);                                                       \
 802     patch_reg((address)&insn, 7, Rd);                                                      \
 803     patch((address)&insn, 14, 12, funct);                                                  \
 804     patch_reg((address)&insn, 15, Rs);                                                     \
 805     int32_t val = offset & 0xfff;                                                          \
 806     patch((address)&insn, 31, 20, val);                                                    \
 807     emit(insn);                                                                            \
 808   }
 809 
 810   INSN(jalr, 0b1100111, 0b000);
 811 
 812 #undef INSN
 813 
 814   enum barrier {
 815     i = 0b1000, o = 0b0100, r = 0b0010, w = 0b0001,
 816     ir = i | r, ow = o | w, iorw = i | o | r | w
 817   };
 818 
 819   void fence(const uint32_t predecessor, const uint32_t successor) {
 820     unsigned insn = 0;
 821     guarantee(predecessor < 16, "predecessor is invalid");
 822     guarantee(successor < 16, "successor is invalid");
 823     patch((address)&insn, 6, 0, 0b001111);
 824     patch((address)&insn, 11, 7, 0b00000);
 825     patch((address)&insn, 14, 12, 0b000);
 826     patch((address)&insn, 19, 15, 0b00000);
 827     patch((address)&insn, 23, 20, successor);
 828     patch((address)&insn, 27, 24, predecessor);
 829     patch((address)&insn, 31, 28, 0b0000);
 830     emit(insn);
 831   }
 832 
 833 #define INSN(NAME, op, funct3, funct7)                      \
 834   void NAME() {                                             \
 835     unsigned insn = 0;                                      \
 836     patch((address)&insn, 6, 0, op);                        \
 837     patch((address)&insn, 11, 7, 0b00000);                  \
 838     patch((address)&insn, 14, 12, funct3);                  \
 839     patch((address)&insn, 19, 15, 0b00000);                 \
 840     patch((address)&insn, 31, 20, funct7);                  \
 841     emit(insn);                                             \
 842   }
 843 
 844   INSN(fence_i, 0b0001111, 0b001, 0b000000000000);
 845   INSN(ecall,   0b1110011, 0b000, 0b000000000000);
 846   INSN(ebreak,  0b1110011, 0b000, 0b000000000001);
 847 #undef INSN
 848 
 849 enum Aqrl {relaxed = 0b00, rl = 0b01, aq = 0b10, aqrl = 0b11};
 850 
 851 #define INSN(NAME, op, funct3, funct7)                                                  \
 852   void NAME(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {        \
 853     unsigned insn = 0;                                                                  \
 854     patch((address)&insn, 6, 0, op);                                                    \
 855     patch((address)&insn, 14, 12, funct3);                                              \
 856     patch_reg((address)&insn, 7, Rd);                                                   \
 857     patch_reg((address)&insn, 15, Rs1);                                                 \
 858     patch_reg((address)&insn, 20, Rs2);                                                 \
 859     patch((address)&insn, 31, 27, funct7);                                              \
 860     patch((address)&insn, 26, 25, memory_order);                                        \
 861     emit(insn);                                                                         \
 862   }
 863 
 864   INSN(amoswap_w, 0b0101111, 0b010, 0b00001);
 865   INSN(amoadd_w,  0b0101111, 0b010, 0b00000);
 866   INSN(amoxor_w,  0b0101111, 0b010, 0b00100);
 867   INSN(amoand_w,  0b0101111, 0b010, 0b01100);
 868   INSN(amoor_w,   0b0101111, 0b010, 0b01000);
 869   INSN(amomin_w,  0b0101111, 0b010, 0b10000);
 870   INSN(amomax_w,  0b0101111, 0b010, 0b10100);
 871   INSN(amominu_w, 0b0101111, 0b010, 0b11000);
 872   INSN(amomaxu_w, 0b0101111, 0b010, 0b11100);
 873   INSN(amoswap_d, 0b0101111, 0b011, 0b00001);
 874   INSN(amoadd_d,  0b0101111, 0b011, 0b00000);
 875   INSN(amoxor_d,  0b0101111, 0b011, 0b00100);
 876   INSN(amoand_d,  0b0101111, 0b011, 0b01100);
 877   INSN(amoor_d,   0b0101111, 0b011, 0b01000);
 878   INSN(amomin_d,  0b0101111, 0b011, 0b10000);
 879   INSN(amomax_d , 0b0101111, 0b011, 0b10100);
 880   INSN(amominu_d, 0b0101111, 0b011, 0b11000);
 881   INSN(amomaxu_d, 0b0101111, 0b011, 0b11100);
 882 #undef INSN
 883 
 884 enum operand_size { int8, int16, int32, uint32, int64 };
 885 
 886 #define INSN(NAME, op, funct3, funct7)                                              \
 887   void NAME(Register Rd, Register Rs1, Aqrl memory_order = relaxed) {               \
 888     unsigned insn = 0;                                                              \
 889     uint32_t val = memory_order & 0x3;                                              \
 890     patch((address)&insn, 6, 0, op);                                                \
 891     patch((address)&insn, 14, 12, funct3);                                          \
 892     patch_reg((address)&insn, 7, Rd);                                               \
 893     patch_reg((address)&insn, 15, Rs1);                                             \
 894     patch((address)&insn, 25, 20, 0b00000);                                         \
 895     patch((address)&insn, 31, 27, funct7);                                          \
 896     patch((address)&insn, 26, 25, val);                                             \
 897     emit(insn);                                                                     \
 898   }
 899 
 900   INSN(lr_w, 0b0101111, 0b010, 0b00010);
 901   INSN(lr_d, 0b0101111, 0b011, 0b00010);
 902 
 903 #undef INSN
 904 
 905 #define INSN(NAME, op, funct3, funct7)                                                      \
 906   void NAME(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = relaxed) {         \
 907     unsigned insn = 0;                                                                      \
 908     uint32_t val = memory_order & 0x3;                                                      \
 909     patch((address)&insn, 6, 0, op);                                                        \
 910     patch((address)&insn, 14, 12, funct3);                                                  \
 911     patch_reg((address)&insn, 7, Rd);                                                       \
 912     patch_reg((address)&insn, 15, Rs2);                                                     \
 913     patch_reg((address)&insn, 20, Rs1);                                                     \
 914     patch((address)&insn, 31, 27, funct7);                                                  \
 915     patch((address)&insn, 26, 25, val);                                                     \
 916     emit(insn);                                                                             \
 917   }
 918 
 919   INSN(sc_w, 0b0101111, 0b010, 0b00011);
 920   INSN(sc_d, 0b0101111, 0b011, 0b00011);
 921 #undef INSN
 922 
 923 #define INSN(NAME, op, funct5, funct7)                                                      \
 924   void NAME(FloatRegister Rd, FloatRegister Rs1, RoundingMode rm = rne) {                   \
 925     unsigned insn = 0;                                                                      \
 926     patch((address)&insn, 6, 0, op);                                                        \
 927     patch((address)&insn, 14, 12, rm);                                                      \
 928     patch((address)&insn, 24, 20, funct5);                                                  \
 929     patch((address)&insn, 31, 25, funct7);                                                  \
 930     patch_reg((address)&insn, 7, Rd);                                                       \
 931     patch_reg((address)&insn, 15, Rs1);                                                     \
 932     emit(insn);                                                                             \
 933   }
 934 
 935   INSN(fsqrt_s,   0b1010011, 0b00000, 0b0101100);
 936   INSN(fsqrt_d,   0b1010011, 0b00000, 0b0101101);
 937   INSN(fcvt_s_d,  0b1010011, 0b00001, 0b0100000);
 938   INSN(fcvt_d_s,  0b1010011, 0b00000, 0b0100001);
 939 #undef INSN
 940 
 941 // Immediate Instruction
 942 #define INSN(NAME, op, funct3)                                                              \
 943   void NAME(Register Rd, Register Rs1, int32_t imm) {                                       \
 944     guarantee(is_imm_in_range(imm, 12, 0), "Immediate is out of validity");                 \
 945     unsigned insn = 0;                                                                      \
 946     patch((address)&insn, 6, 0, op);                                                        \
 947     patch((address)&insn, 14, 12, funct3);                                                  \
 948     patch((address)&insn, 31, 20, imm & 0x00000fff);                                        \
 949     patch_reg((address)&insn, 7, Rd);                                                       \
 950     patch_reg((address)&insn, 15, Rs1);                                                     \
 951     emit(insn);                                                                             \
 952   }
 953 
 954   INSN(addi,  0b0010011, 0b000);
 955   INSN(slti,  0b0010011, 0b010);
 956   INSN(addiw, 0b0011011, 0b000);
 957   INSN(and_imm12,  0b0010011, 0b111);
 958   INSN(ori,   0b0010011, 0b110);
 959   INSN(xori,  0b0010011, 0b100);
 960 
 961 #undef INSN
 962 
 963 #define INSN(NAME, op, funct3)                                                              \
 964   void NAME(Register Rd, Register Rs1, uint32_t imm) {                                      \
 965     guarantee(is_unsigned_imm_in_range(imm, 12, 0), "Immediate is out of validity");        \
 966     unsigned insn = 0;                                                                      \
 967     patch((address)&insn,6, 0,  op);                                                        \
 968     patch((address)&insn, 14, 12, funct3);                                                  \
 969     patch((address)&insn, 31, 20, imm & 0x00000fff);                                        \
 970     patch_reg((address)&insn, 7, Rd);                                                       \
 971     patch_reg((address)&insn, 15, Rs1);                                                     \
 972     emit(insn);                                                                             \
 973   }
 974 
 975   INSN(sltiu, 0b0010011, 0b011);
 976 
 977 #undef INSN
 978 
 979 // Shift Immediate Instruction
 980 #define INSN(NAME, op, funct3, funct6)                                   \
 981   void NAME(Register Rd, Register Rs1, unsigned shamt) {                 \
 982     guarantee(shamt <= 0x3f, "Shamt is invalid");                        \
 983     unsigned insn = 0;                                                   \
 984     patch((address)&insn, 6, 0, op);                                     \
 985     patch((address)&insn, 14, 12, funct3);                               \
 986     patch((address)&insn, 25, 20, shamt);                                \
 987     patch((address)&insn, 31, 26, funct6);                               \
 988     patch_reg((address)&insn, 7, Rd);                                    \
 989     patch_reg((address)&insn, 15, Rs1);                                  \
 990     emit(insn);                                                          \
 991   }
 992 
 993   INSN(slli,  0b0010011, 0b001, 0b000000);
 994   INSN(srai,  0b0010011, 0b101, 0b010000);
 995   INSN(srli,  0b0010011, 0b101, 0b000000);
 996 
 997 #undef INSN
 998 
 999 // Shift Word Immediate Instruction
1000 #define INSN(NAME, op, funct3, funct7)                                  \
1001   void NAME(Register Rd, Register Rs1, unsigned shamt) {                \
1002     guarantee(shamt <= 0x1f, "Shamt is invalid");                       \
1003     unsigned insn = 0;                                                  \
1004     patch((address)&insn, 6, 0, op);                                    \
1005     patch((address)&insn, 14, 12, funct3);                              \
1006     patch((address)&insn, 24, 20, shamt);                               \
1007     patch((address)&insn, 31, 25, funct7);                              \
1008     patch_reg((address)&insn, 7, Rd);                                   \
1009     patch_reg((address)&insn, 15, Rs1);                                 \
1010     emit(insn);                                                         \
1011   }
1012 
1013   INSN(slliw, 0b0011011, 0b001, 0b0000000);
1014   INSN(sraiw, 0b0011011, 0b101, 0b0100000);
1015   INSN(srliw, 0b0011011, 0b101, 0b0000000);
1016 
1017 #undef INSN
1018 
1019 // Upper Immediate Instruction
1020 #define INSN(NAME, op)                                                  \
1021   void NAME(Register Rd, int32_t imm) {                                 \
1022     int32_t upperImm = imm >> 12;                                       \
1023     unsigned insn = 0;                                                  \
1024     patch((address)&insn, 6, 0, op);                                    \
1025     patch_reg((address)&insn, 7, Rd);                                   \
1026     upperImm &= 0x000fffff;                                             \
1027     patch((address)&insn, 31, 12, upperImm);                            \
1028     emit(insn);                                                         \
1029   }
1030 
1031   INSN(lui,   0b0110111);
1032   INSN(auipc, 0b0010111);
1033 
1034 #undef INSN
1035 
1036 // Float and Double Rigster Instruction
1037 #define INSN(NAME, op, funct2)                                                                                     \
1038   void NAME(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, FloatRegister Rs3, RoundingMode rm = rne) {    \
1039     unsigned insn = 0;                                                                                             \
1040     patch((address)&insn, 6, 0, op);                                                                               \
1041     patch((address)&insn, 14, 12, rm);                                                                             \
1042     patch((address)&insn, 26, 25, funct2);                                                                         \
1043     patch_reg((address)&insn, 7, Rd);                                                                              \
1044     patch_reg((address)&insn, 15, Rs1);                                                                            \
1045     patch_reg((address)&insn, 20, Rs2);                                                                            \
1046     patch_reg((address)&insn, 27, Rs3);                                                                            \
1047     emit(insn);                                                                                                    \
1048   }
1049 
1050   INSN(fmadd_s,   0b1000011,  0b00);
1051   INSN(fmsub_s,   0b1000111,  0b00);
1052   INSN(fnmsub_s,  0b1001011,  0b00);
1053   INSN(fnmadd_s,  0b1001111,  0b00);
1054   INSN(fmadd_d,   0b1000011,  0b01);
1055   INSN(fmsub_d,   0b1000111,  0b01);
1056   INSN(fnmsub_d,  0b1001011,  0b01);
1057   INSN(fnmadd_d,  0b1001111,  0b01);
1058 
1059 #undef INSN
1060 
1061 // Float and Double Rigster Instruction
1062 #define INSN(NAME, op, funct3, funct7)                                        \
1063   void NAME(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2) {         \
1064     unsigned insn = 0;                                                        \
1065     patch((address)&insn, 6, 0, op);                                          \
1066     patch((address)&insn, 14, 12, funct3);                                    \
1067     patch((address)&insn, 31, 25, funct7);                                    \
1068     patch_reg((address)&insn, 7, Rd);                                         \
1069     patch_reg((address)&insn, 15, Rs1);                                       \
1070     patch_reg((address)&insn, 20, Rs2);                                       \
1071     emit(insn);                                                               \
1072   }
1073 
1074   INSN(fsgnj_s,  0b1010011, 0b000, 0b0010000);
1075   INSN(fsgnjn_s, 0b1010011, 0b001, 0b0010000);
1076   INSN(fsgnjx_s, 0b1010011, 0b010, 0b0010000);
1077   INSN(fmin_s,   0b1010011, 0b000, 0b0010100);
1078   INSN(fmax_s,   0b1010011, 0b001, 0b0010100);
1079   INSN(fsgnj_d,  0b1010011, 0b000, 0b0010001);
1080   INSN(fsgnjn_d, 0b1010011, 0b001, 0b0010001);
1081   INSN(fsgnjx_d, 0b1010011, 0b010, 0b0010001);
1082   INSN(fmin_d,   0b1010011, 0b000, 0b0010101);
1083   INSN(fmax_d,   0b1010011, 0b001, 0b0010101);
1084 
1085 #undef INSN
1086 
1087 // Float and Double Rigster Arith Instruction
1088 #define INSN(NAME, op, funct3, funct7)                                    \
1089   void NAME(Register Rd, FloatRegister Rs1, FloatRegister Rs2) {          \
1090     unsigned insn = 0;                                                    \
1091     patch((address)&insn, 6, 0, op);                                      \
1092     patch((address)&insn, 14, 12, funct3);                                \
1093     patch((address)&insn, 31, 25, funct7);                                \
1094     patch_reg((address)&insn, 7, Rd);                                     \
1095     patch_reg((address)&insn, 15, Rs1);                                   \
1096     patch_reg((address)&insn, 20, Rs2);                                   \
1097     emit(insn);                                                           \
1098   }
1099 
1100   INSN(feq_s,    0b1010011, 0b010, 0b1010000);
1101   INSN(flt_s,    0b1010011, 0b001, 0b1010000);
1102   INSN(fle_s,    0b1010011, 0b000, 0b1010000);
1103   INSN(feq_d,    0b1010011, 0b010, 0b1010001);
1104   INSN(fle_d,    0b1010011, 0b000, 0b1010001);
1105   INSN(flt_d,    0b1010011, 0b001, 0b1010001);
1106 #undef INSN
1107 
1108 // Float and Double Arith Instruction
1109 #define INSN(NAME, op, funct7)                                                                  \
1110   void NAME(FloatRegister Rd, FloatRegister Rs1, FloatRegister Rs2, RoundingMode rm = rne) {    \
1111     unsigned insn = 0;                                                                          \
1112     patch((address)&insn, 6, 0, op);                                                            \
1113     patch((address)&insn, 14, 12, rm);                                                          \
1114     patch((address)&insn, 31, 25, funct7);                                                      \
1115     patch_reg((address)&insn, 7, Rd);                                                           \
1116     patch_reg((address)&insn, 15, Rs1);                                                         \
1117     patch_reg((address)&insn, 20, Rs2);                                                         \
1118     emit(insn);                                                                                 \
1119   }
1120 
1121   INSN(fadd_s,   0b1010011, 0b0000000);
1122   INSN(fsub_s,   0b1010011, 0b0000100);
1123   INSN(fmul_s,   0b1010011, 0b0001000);
1124   INSN(fdiv_s,   0b1010011, 0b0001100);
1125   INSN(fadd_d,   0b1010011, 0b0000001);
1126   INSN(fsub_d,   0b1010011, 0b0000101);
1127   INSN(fmul_d,   0b1010011, 0b0001001);
1128   INSN(fdiv_d,   0b1010011, 0b0001101);
1129 
1130 #undef INSN
1131 
1132 // Whole Float and Double Conversion Instruction
1133 #define INSN(NAME, op, funct5, funct7)                                  \
1134   void NAME(FloatRegister Rd, Register Rs1, RoundingMode rm = rne) {    \
1135     unsigned insn = 0;                                                  \
1136     patch((address)&insn, 6, 0, op);                                    \
1137     patch((address)&insn, 14, 12, rm);                                  \
1138     patch((address)&insn, 24, 20, funct5);                              \
1139     patch((address)&insn, 31, 25, funct7);                              \
1140     patch_reg((address)&insn, 7, Rd);                                   \
1141     patch_reg((address)&insn, 15, Rs1);                                 \
1142     emit(insn);                                                         \
1143   }
1144 
1145   INSN(fcvt_s_w,   0b1010011, 0b00000, 0b1101000);
1146   INSN(fcvt_s_wu,  0b1010011, 0b00001, 0b1101000);
1147   INSN(fcvt_s_l,   0b1010011, 0b00010, 0b1101000);
1148   INSN(fcvt_s_lu,  0b1010011, 0b00011, 0b1101000);
1149   INSN(fcvt_d_w,   0b1010011, 0b00000, 0b1101001);
1150   INSN(fcvt_d_wu,  0b1010011, 0b00001, 0b1101001);
1151   INSN(fcvt_d_l,   0b1010011, 0b00010, 0b1101001);
1152   INSN(fcvt_d_lu,  0b1010011, 0b00011, 0b1101001);
1153 
1154 #undef INSN
1155 
1156 // Float and Double Conversion Instruction
1157 #define INSN(NAME, op, funct5, funct7)                                  \
1158   void NAME(Register Rd, FloatRegister Rs1, RoundingMode rm = rtz) {    \
1159     unsigned insn = 0;                                                  \
1160     patch((address)&insn, 6, 0, op);                                    \
1161     patch((address)&insn, 14, 12, rm);                                  \
1162     patch((address)&insn, 24, 20, funct5);                              \
1163     patch((address)&insn, 31, 25, funct7);                              \
1164     patch_reg((address)&insn, 7, Rd);                                   \
1165     patch_reg((address)&insn, 15, Rs1);                                 \
1166     emit(insn);                                                         \
1167   }
1168 
1169   INSN(fcvt_w_s,   0b1010011, 0b00000, 0b1100000);
1170   INSN(fcvt_l_s,   0b1010011, 0b00010, 0b1100000);
1171   INSN(fcvt_wu_s,  0b1010011, 0b00001, 0b1100000);
1172   INSN(fcvt_lu_s,  0b1010011, 0b00011, 0b1100000);
1173   INSN(fcvt_w_d,   0b1010011, 0b00000, 0b1100001);
1174   INSN(fcvt_wu_d,  0b1010011, 0b00001, 0b1100001);
1175   INSN(fcvt_l_d,   0b1010011, 0b00010, 0b1100001);
1176   INSN(fcvt_lu_d,  0b1010011, 0b00011, 0b1100001);
1177 
1178 #undef INSN
1179 
1180 // Float and Double Move Instruction
1181 #define INSN(NAME, op, funct3, funct5, funct7)       \
1182   void NAME(FloatRegister Rd, Register Rs1) {        \
1183     unsigned insn = 0;                               \
1184     patch((address)&insn, 6, 0, op);                 \
1185     patch((address)&insn, 14, 12, funct3);           \
1186     patch((address)&insn, 20, funct5);               \
1187     patch((address)&insn, 31, 25, funct7);           \
1188     patch_reg((address)&insn, 7, Rd);                \
1189     patch_reg((address)&insn, 15, Rs1);              \
1190     emit(insn);                                      \
1191   }
1192 
1193   INSN(fmv_w_x,  0b1010011, 0b000, 0b00000, 0b1111000);
1194   INSN(fmv_d_x,  0b1010011, 0b000, 0b00000, 0b1111001);
1195 
1196 #undef INSN
1197 
1198 // Float and Double Conversion Instruction
1199 #define INSN(NAME, op, funct3, funct5, funct7)            \
1200   void NAME(Register Rd, FloatRegister Rs1) {             \
1201     unsigned insn = 0;                                    \
1202     patch((address)&insn, 6, 0, op);                      \
1203     patch((address)&insn, 14, 12, funct3);                \
1204     patch((address)&insn, 20, funct5);                    \
1205     patch((address)&insn, 31, 25, funct7);                \
1206     patch_reg((address)&insn, 7, Rd);                     \
1207     patch_reg((address)&insn, 15, Rs1);                   \
1208     emit(insn);                                           \
1209   }
1210 
1211   INSN(fclass_s, 0b1010011, 0b001, 0b00000, 0b1110000);
1212   INSN(fclass_d, 0b1010011, 0b001, 0b00000, 0b1110001);
1213   INSN(fmv_x_w,  0b1010011, 0b000, 0b00000, 0b1110000);
1214   INSN(fmv_x_d,  0b1010011, 0b000, 0b00000, 0b1110001);
1215 
1216 #undef INSN
1217 
1218 enum SEW {
1219   e8    = 0b000,
1220   e16   = 0b001,
1221   e32   = 0b010,
1222   e64   = 0b011,
1223   e128  = 0b100,
1224   e256  = 0b101,
1225   e512  = 0b110,
1226   e1024 = 0b111,
1227 };
1228 
1229 enum LMUL {
1230   mf8 = 0b101,
1231   mf4 = 0b110,
1232   mf2 = 0b111,
1233   m1  = 0b000,
1234   m2  = 0b001,
1235   m4  = 0b010,
1236   m8  = 0b011,
1237 };
1238 
1239 enum VMA {
1240   mu, // undisturbed
1241   ma, // agnostic
1242 };
1243 
1244 enum VTA {
1245   tu, // undisturbed
1246   ta, // agnostic
1247 };
1248 
1249 #define patch_vtype(hsb, lsb, vlmul, vsew, vta, vma, vill)   \
1250     if (vill == 1) {                                         \
1251       guarantee((vlmul | vsew | vsew | vta | vma == 0),      \
1252                 "the other bits in vtype shall be zero");    \
1253     }                                                        \
1254     patch((address)&insn, lsb + 2, lsb, vlmul);              \
1255     patch((address)&insn, lsb + 5, lsb + 3, vsew);           \
1256     patch((address)&insn, lsb + 6, vta);                     \
1257     patch((address)&insn, lsb + 7, vma);                     \
1258     patch((address)&insn, hsb - 1, lsb + 8, 0);              \
1259     patch((address)&insn, hsb, vill)
1260 
1261 #define INSN(NAME, op, funct3)                                            \
1262   void NAME(Register Rd, Register Rs1, SEW sew, LMUL lmul = m1,           \
1263             VMA vma = mu, VTA vta = tu, bool vill = false) {              \
1264     unsigned insn = 0;                                                    \
1265     patch((address)&insn, 6, 0, op);                                      \
1266     patch((address)&insn, 14, 12, funct3);                                \
1267     patch_vtype(30, 20, lmul, sew, vta, vma, vill);                       \
1268     patch((address)&insn, 31, 0);                                         \
1269     patch_reg((address)&insn, 7, Rd);                                     \
1270     patch_reg((address)&insn, 15, Rs1);                                   \
1271     emit(insn);                                                           \
1272   }
1273 
1274   INSN(vsetvli, 0b1010111, 0b111);
1275 
1276 #undef INSN
1277 
1278 #define INSN(NAME, op, funct3)                                            \
1279   void NAME(Register Rd, uint32_t imm, SEW sew, LMUL lmul = m1,           \
1280             VMA vma = mu, VTA vta = tu, bool vill = false) {              \
1281     unsigned insn = 0;                                                    \
1282     guarantee(is_unsigned_imm_in_range(imm, 5, 0), "imm is invalid");     \
1283     patch((address)&insn, 6, 0, op);                                      \
1284     patch((address)&insn, 14, 12, funct3);                                \
1285     patch((address)&insn, 19, 15, imm);                                   \
1286     patch_vtype(29, 20, lmul, sew, vta, vma, vill);                       \
1287     patch((address)&insn, 31, 30, 0b11);                                  \
1288     patch_reg((address)&insn, 7, Rd);                                     \
1289     emit(insn);                                                           \
1290   }
1291 
1292   INSN(vsetivli, 0b1010111, 0b111);
1293 
1294 #undef INSN
1295 
1296 #undef patch_vtype
1297 
1298 enum VectorMask {
1299   v0_t = 0b0,
1300   unmasked = 0b1
1301 };
1302 
1303 // Vector AMO operations
1304 #define INSN(NAME, op, funct3, funct5)                                   \
1305   void NAME(VectorRegister vSrc, Register rBase, VectorRegister vOffset, \
1306             bool src_as_dst, VectorMask vm = unmasked) {                 \
1307     unsigned insn = 0;                                                   \
1308     patch((address)&insn, 6, 0, op);                                     \
1309     patch((address)&insn, 14, 12, funct3);                               \
1310     patch((address)&insn, 25, vm);                                       \
1311     patch((address)&insn, 26, (uint32_t)src_as_dst);                     \
1312     patch((address)&insn, 31, 27, funct5);                               \
1313     patch_reg((address)&insn, 7, vSrc);                                  \
1314     patch_reg((address)&insn, 15, rBase);                                \
1315     patch_reg((address)&insn, 20, vOffset);                              \
1316     emit(insn);                                                          \
1317   }
1318 
1319   INSN(vamoswapei8_v,  0b0101111, 0b000, 0b00001);
1320   INSN(vamoswapei16_v, 0b0101111, 0b101, 0b00001);
1321   INSN(vamoswapei32_v, 0b0101111, 0b110, 0b00001);
1322   INSN(vamoaddei8_v,   0b0101111, 0b000, 0b00000);
1323   INSN(vamoaddei16_v,  0b0101111, 0b101, 0b00000);
1324   INSN(vamoaddei32_v,  0b0101111, 0b110, 0b00000);
1325   INSN(vamoxorei8_v,   0b0101111, 0b000, 0b00100);
1326   INSN(vamoxorei16_v,  0b0101111, 0b101, 0b00100);
1327   INSN(vamoxorei32_v,  0b0101111, 0b110, 0b00100);
1328   INSN(vamoandei8_v,   0b0101111, 0b000, 0b01100);
1329   INSN(vamoandei16_v,  0b0101111, 0b101, 0b01100);
1330   INSN(vamoandei32_v,  0b0101111, 0b110, 0b01100);
1331   INSN(vamoorei8_v,    0b0101111, 0b000, 0b01000);
1332   INSN(vamoorei16_v,   0b0101111, 0b101, 0b01000);
1333   INSN(vamoorei32_v,   0b0101111, 0b110, 0b01000);
1334   INSN(vamominei8_v,   0b0101111, 0b000, 0b10000);
1335   INSN(vamominei16_v,  0b0101111, 0b101, 0b10000);
1336   INSN(vamominei32_v,  0b0101111, 0b110, 0b10000);
1337   INSN(vamomaxei8_v,   0b0101111, 0b000, 0b10100);
1338   INSN(vamomaxei16_v,  0b0101111, 0b101, 0b10100);
1339   INSN(vamomaxei32_v,  0b0101111, 0b110, 0b10100);
1340   INSN(vamominuei8_v,  0b0101111, 0b000, 0b11000);
1341   INSN(vamominuei16_v, 0b0101111, 0b101, 0b11000);
1342   INSN(vamominuei32_v, 0b0101111, 0b110, 0b11000);
1343   INSN(vamomaxuei8_v,  0b0101111, 0b000, 0b11100);
1344   INSN(vamomaxuei16_v, 0b0101111, 0b101, 0b11100);
1345   INSN(vamomaxuei32_v, 0b0101111, 0b110, 0b11100);
1346 
1347 #undef INSN
1348 
1349 #define patch_VArith(op, Reg, funct3, Reg_or_Imm5, Vs2, vm, funct6)            \
1350     unsigned insn = 0;                                                         \
1351     patch((address)&insn, 6, 0, op);                                           \
1352     patch((address)&insn, 14, 12, funct3);                                     \
1353     patch((address)&insn, 19, 15, Reg_or_Imm5);                                \
1354     patch((address)&insn, 25, vm);                                             \
1355     patch((address)&insn, 31, 26, funct6);                                     \
1356     patch_reg((address)&insn, 7, Reg);                                         \
1357     patch_reg((address)&insn, 20, Vs2);                                        \
1358     emit(insn)
1359 
1360 // r2_vm
1361 #define INSN(NAME, op, funct3, Vs1, funct6)                                    \
1362   void NAME(Register Rd, VectorRegister Vs2, VectorMask vm = unmasked) {       \
1363     patch_VArith(op, Rd, funct3, Vs1, Vs2, vm, funct6);                        \
1364   }
1365 
1366   // Vector Mask
1367   INSN(vpopc_m,  0b1010111, 0b010, 0b10000, 0b010000);
1368   INSN(vfirst_m, 0b1010111, 0b010, 0b10001, 0b010000);
1369 #undef INSN
1370 
1371 #define INSN(NAME, op, funct3, Vs1, funct6)                                    \
1372   void NAME(VectorRegister Vd, VectorRegister Vs2, VectorMask vm = unmasked) { \
1373     patch_VArith(op, Vd, funct3, Vs1, Vs2, vm, funct6);                        \
1374   }
1375 
1376   // Vector Integer Extension
1377   INSN(vzext_vf2, 0b1010111, 0b010, 0b00110, 0b010010);
1378   INSN(vzext_vf4, 0b1010111, 0b010, 0b00100, 0b010010);
1379   INSN(vzext_vf8, 0b1010111, 0b010, 0b00010, 0b010010);
1380   INSN(vsext_vf2, 0b1010111, 0b010, 0b00111, 0b010010);
1381   INSN(vsext_vf4, 0b1010111, 0b010, 0b00101, 0b010010);
1382   INSN(vsext_vf8, 0b1010111, 0b010, 0b00011, 0b010010);
1383 
1384   // Vector Mask
1385   INSN(vmsbf_m,   0b1010111, 0b010, 0b00001, 0b010100);
1386   INSN(vmsif_m,   0b1010111, 0b010, 0b00011, 0b010100);
1387   INSN(vmsof_m,   0b1010111, 0b010, 0b00010, 0b010100);
1388   INSN(viota_m,   0b1010111, 0b010, 0b10000, 0b010100);
1389 
1390   // Vector Single-Width Floating-Point/Integer Type-Convert Instructions
1391   INSN(vfcvt_xu_f_v, 0b1010111, 0b001, 0b00000, 0b010010);
1392   INSN(vfcvt_x_f_v,  0b1010111, 0b001, 0b00001, 0b010010);
1393   INSN(vfcvt_f_xu_v, 0b1010111, 0b001, 0b00010, 0b010010);
1394   INSN(vfcvt_f_x_v,  0b1010111, 0b001, 0b00011, 0b010010);
1395   INSN(vfcvt_rtz_xu_f_v, 0b1010111, 0b001, 0b00110, 0b010010);
1396   INSN(vfcvt_rtz_x_f_v,  0b1010111, 0b001, 0b00111, 0b010010);
1397 
1398   // Vector Widening Floating-Point/Integer Type-Convert Instructions
1399   INSN(vfwcvt_xu_f_v, 0b1010111, 0b001, 0b01000, 0b010010);
1400   INSN(vfwcvt_x_f_v,  0b1010111, 0b001, 0b01001, 0b010010);
1401   INSN(vfwcvt_f_xu_v, 0b1010111, 0b001, 0b01010, 0b010010);
1402   INSN(vfwcvt_f_x_v,  0b1010111, 0b001, 0b01011, 0b010010);
1403   INSN(vfwcvt_f_f_v,  0b1010111, 0b001, 0b01100, 0b010010);
1404   INSN(vfwcvt_rtz_xu_f_v, 0b1010111, 0b001, 0b01110, 0b010010);
1405   INSN(vfwcvt_rtz_x_f_v,  0b1010111, 0b001, 0b01111, 0b010010);
1406 
1407   // Vector Narrowing Floating-Point/Integer Type-Convert Instructions
1408   INSN(vfncvt_xu_f_w, 0b1010111, 0b001, 0b10000, 0b010010);
1409   INSN(vfncvt_x_f_w,  0b1010111, 0b001, 0b10001, 0b010010);
1410   INSN(vfncvt_f_xu_w, 0b1010111, 0b001, 0b10010, 0b010010);
1411   INSN(vfncvt_f_x_w,  0b1010111, 0b001, 0b10011, 0b010010);
1412   INSN(vfncvt_f_f_w,  0b1010111, 0b001, 0b10100, 0b010010);
1413   INSN(vfncvt_rod_f_f_w,  0b1010111, 0b001, 0b10101, 0b010010);
1414   INSN(vfncvt_rtz_xu_f_w, 0b1010111, 0b001, 0b10110, 0b010010);
1415   INSN(vfncvt_rtz_x_f_w,  0b1010111, 0b001, 0b10111, 0b010010);
1416 
1417   // Vector Floating-Point Instruction
1418   INSN(vfsqrt_v,  0b1010111, 0b001, 0b00000, 0b010011);
1419   INSN(vfclass_v, 0b1010111, 0b001, 0b10000, 0b010011);
1420 
1421 #undef INSN
1422 
1423 // r2rd
1424 #define INSN(NAME, op, funct3, simm5, vm, funct6)         \
1425   void NAME(VectorRegister Vd, VectorRegister Vs2) {      \
1426     patch_VArith(op, Vd, funct3, simm5, Vs2, vm, funct6); \
1427   }
1428 
1429   // Vector Whole Vector Register Move
1430   INSN(vmv1r_v, 0b1010111, 0b011, 0b00000, 0b1, 0b100111);
1431   INSN(vmv2r_v, 0b1010111, 0b011, 0b00001, 0b1, 0b100111);
1432   INSN(vmv4r_v, 0b1010111, 0b011, 0b00011, 0b1, 0b100111);
1433   INSN(vmv8r_v, 0b1010111, 0b011, 0b00111, 0b1, 0b100111);
1434 
1435 #undef INSN
1436 
1437 #define INSN(NAME, op, funct3, Vs1, vm, funct6)           \
1438   void NAME(FloatRegister Rd, VectorRegister Vs2) {       \
1439     patch_VArith(op, Rd, funct3, Vs1, Vs2, vm, funct6);   \
1440   }
1441 
1442   // Vector Floating-Point Move Instruction
1443   INSN(vfmv_f_s, 0b1010111, 0b001, 0b00000, 0b1, 0b010000);
1444 
1445 #undef INSN
1446 
1447 #define INSN(NAME, op, funct3, Vs1, vm, funct6)          \
1448   void NAME(Register Rd, VectorRegister Vs2) {           \
1449     patch_VArith(op, Rd, funct3, Vs1, Vs2, vm, funct6);  \
1450   }
1451 
1452   // Vector Integer Scalar Move Instructions
1453   INSN(vmv_x_s, 0b1010111, 0b010, 0b00000, 0b1, 0b010000);
1454 
1455 #undef INSN
1456 
1457 // r_vm
1458 #define INSN(NAME, op, funct3, funct6)                                                             \
1459   void NAME(VectorRegister Vd, VectorRegister Vs2, uint32_t imm, VectorMask vm = unmasked) {       \
1460     guarantee(is_unsigned_imm_in_range(imm, 5, 0), "imm is invalid");                              \
1461     patch_VArith(op, Vd, funct3, (uint32_t)(imm & 0x1f), Vs2, vm, funct6);                         \
1462   }
1463 
1464   // Vector Register Gather Instruction
1465   INSN(vrgather_vi,   0b1010111, 0b011, 0b001100);
1466 
1467   // Vector Slide Instructions
1468   INSN(vslidedown_vi, 0b1010111, 0b011, 0b001111);
1469   INSN(vslideup_vi,   0b1010111, 0b011, 0b001110);
1470 
1471   // Vector Narrowing Fixed-Point Clip Instructions
1472   INSN(vnclip_wi,  0b1010111, 0b011, 0b101111);
1473   INSN(vnclipu_wi, 0b1010111, 0b011, 0b101110);
1474 
1475   // Vector Single-Width Scaling Shift Instructions
1476   INSN(vssra_vi,   0b1010111, 0b011, 0b101011);
1477   INSN(vssrl_vi,   0b1010111, 0b011, 0b101010);
1478 
1479   // Vector Narrowing Integer Right Shift Instructions
1480   INSN(vnsra_wi,   0b1010111, 0b011, 0b101101);
1481   INSN(vnsrl_wi,   0b1010111, 0b011, 0b101100);
1482 
1483   // Vector Single-Width Bit Shift Instructions
1484   INSN(vsra_vi,    0b1010111, 0b011, 0b101001);
1485   INSN(vsrl_vi,    0b1010111, 0b011, 0b101000);
1486   INSN(vsll_vi,    0b1010111, 0b011, 0b100101);
1487 
1488 #undef INSN
1489 
1490 #define INSN(NAME, op, funct3, funct6)                                                             \
1491   void NAME(VectorRegister Vd, VectorRegister Vs1, VectorRegister Vs2, VectorMask vm = unmasked) { \
1492     patch_VArith(op, Vd, funct3, Vs1->encoding_nocheck(), Vs2, vm, funct6);                        \
1493   }
1494 
1495   // Vector Widening Floating-Point Fused Multiply-Add Instructions
1496   INSN(vfwnmsac_vv, 0b1010111, 0b001, 0b111111);
1497   INSN(vfwmsac_vv,  0b1010111, 0b001, 0b111110);
1498   INSN(vfwnmacc_vv, 0b1010111, 0b001, 0b111101);
1499   INSN(vfwmacc_vv,  0b1010111, 0b001, 0b111100);
1500 
1501   // Vector Single-Width Floating-Point Fused Multiply-Add Instructions
1502   INSN(vfnmsub_vv, 0b1010111, 0b001, 0b101011);
1503   INSN(vfmsub_vv,  0b1010111, 0b001, 0b101010);
1504   INSN(vfnmadd_vv, 0b1010111, 0b001, 0b101001);
1505   INSN(vfmadd_vv,  0b1010111, 0b001, 0b101000);
1506   INSN(vfnmsac_vv, 0b1010111, 0b001, 0b101111);
1507   INSN(vfmsac_vv,  0b1010111, 0b001, 0b101110);
1508   INSN(vfmacc_vv,  0b1010111, 0b001, 0b101100);
1509   INSN(vfnmacc_vv, 0b1010111, 0b001, 0b101101);
1510 
1511   // Vector Widening Integer Multiply-Add Instructions
1512   INSN(vwmaccsu_vv, 0b1010111, 0b010, 0b111111);
1513   INSN(vwmacc_vv,   0b1010111, 0b010, 0b111101);
1514   INSN(vwmaccu_vv,  0b1010111, 0b010, 0b111100);
1515 
1516   // Vector Single-Width Integer Multiply-Add Instructions
1517   INSN(vnmsub_vv, 0b1010111, 0b010, 0b101011);
1518   INSN(vmadd_vv,  0b1010111, 0b010, 0b101001);
1519   INSN(vnmsac_vv, 0b1010111, 0b010, 0b101111);
1520   INSN(vmacc_vv,  0b1010111, 0b010, 0b101101);
1521 
1522 #undef INSN
1523 
1524 #define INSN(NAME, op, funct3, funct6)                                                             \
1525   void NAME(VectorRegister Vd, Register Rs1, VectorRegister Vs2, VectorMask vm = unmasked) {       \
1526     patch_VArith(op, Vd, funct3, Rs1->encoding_nocheck(), Vs2, vm, funct6);                        \
1527   }
1528 
1529   // Vector Widening Integer Multiply-Add Instructions
1530   INSN(vwmaccsu_vx, 0b1010111, 0b110, 0b111111);
1531   INSN(vwmacc_vx,   0b1010111, 0b110, 0b111101);
1532   INSN(vwmaccu_vx,  0b1010111, 0b110, 0b111100);
1533   INSN(vwmaccus_vx, 0b1010111, 0b110, 0b111110);
1534 
1535   // Vector Single-Width Integer Multiply-Add Instructions
1536   INSN(vnmsub_vx, 0b1010111, 0b110, 0b101011);
1537   INSN(vmadd_vx,  0b1010111, 0b110, 0b101001);
1538   INSN(vnmsac_vx, 0b1010111, 0b110, 0b101111);
1539   INSN(vmacc_vx,  0b1010111, 0b110, 0b101101);
1540 
1541   INSN(vrsub_vx,  0b1010111, 0b100, 0b000011);
1542 
1543 #undef INSN
1544 
1545 #define INSN(NAME, op, funct3, funct6)                                                             \
1546   void NAME(VectorRegister Vd, FloatRegister Rs1, VectorRegister Vs2, VectorMask vm = unmasked) {  \
1547     patch_VArith(op, Vd, funct3, Rs1->encoding_nocheck(), Vs2, vm, funct6);                        \
1548   }
1549 
1550   // Vector Widening Floating-Point Fused Multiply-Add Instructions
1551   INSN(vfwnmsac_vf, 0b1010111, 0b101, 0b111111);
1552   INSN(vfwmsac_vf,  0b1010111, 0b101, 0b111110);
1553   INSN(vfwnmacc_vf, 0b1010111, 0b101, 0b111101);
1554   INSN(vfwmacc_vf,  0b1010111, 0b101, 0b111100);
1555 
1556   // Vector Single-Width Floating-Point Fused Multiply-Add Instructions
1557   INSN(vfnmsub_vf, 0b1010111, 0b101, 0b101011);
1558   INSN(vfmsub_vf,  0b1010111, 0b101, 0b101010);
1559   INSN(vfnmadd_vf, 0b1010111, 0b101, 0b101001);
1560   INSN(vfmadd_vf,  0b1010111, 0b101, 0b101000);
1561   INSN(vfnmsac_vf, 0b1010111, 0b101, 0b101111);
1562   INSN(vfmsac_vf,  0b1010111, 0b101, 0b101110);
1563   INSN(vfmacc_vf,  0b1010111, 0b101, 0b101100);
1564   INSN(vfnmacc_vf, 0b1010111, 0b101, 0b101101);
1565 
1566 #undef INSN
1567 
1568 #define INSN(NAME, op, funct3, funct6)                                                             \
1569   void NAME(VectorRegister Vd, VectorRegister Vs2, VectorRegister Vs1, VectorMask vm = unmasked) { \
1570     patch_VArith(op, Vd, funct3, Vs1->encoding_nocheck(), Vs2, vm, funct6);                        \
1571   }
1572 
1573   // Vector Register Gather Instruction
1574   INSN(vrgather_vv,     0b1010111, 0b000, 0b001100);
1575   INSN(vrgatherei16_vv, 0b1010111, 0b000, 0b001110);
1576 
1577   // Vector Widening Floating-Point Reduction Instructions
1578   INSN(vfwredsum_vs,  0b1010111, 0b001, 0b110001);
1579   INSN(vfwredosum_vs, 0b1010111, 0b001, 0b110011);
1580 
1581   // Vector Single-Width Floating-Point Reduction Instructions
1582   INSN(vfredsum_vs,   0b1010111, 0b001, 0b000001);
1583   INSN(vfredosum_vs,  0b1010111, 0b001, 0b000011);
1584   INSN(vfredmin_vs,   0b1010111, 0b001, 0b000101);
1585   INSN(vfredmax_vs,   0b1010111, 0b001, 0b000111);
1586 
1587   // Vector Single-Width Integer Reduction Instructions
1588   INSN(vredsum_vs,    0b1010111, 0b010, 0b000000);
1589   INSN(vredand_vs,    0b1010111, 0b010, 0b000001);
1590   INSN(vredor_vs,     0b1010111, 0b010, 0b000010);
1591   INSN(vredxor_vs,    0b1010111, 0b010, 0b000011);
1592   INSN(vredminu_vs,   0b1010111, 0b010, 0b000100);
1593   INSN(vredmin_vs,    0b1010111, 0b010, 0b000101);
1594   INSN(vredmaxu_vs,   0b1010111, 0b010, 0b000110);
1595   INSN(vredmax_vs,    0b1010111, 0b010, 0b000111);
1596 
1597   // Vector Widening Integer Reduction Instructions
1598   INSN(vwredsumu_vs,  0b1010111, 0b000, 0b110000);
1599   INSN(vwredsum_vs,   0b1010111, 0b000, 0b110001);
1600 
1601   // Vector Floating-Point Compare Instructions
1602   INSN(vmfle_vv, 0b1010111, 0b001, 0b011001);
1603   INSN(vmflt_vv, 0b1010111, 0b001, 0b011011);
1604   INSN(vmfne_vv, 0b1010111, 0b001, 0b011100);
1605   INSN(vmfeq_vv, 0b1010111, 0b001, 0b011000);
1606 
1607   // Vector Floating-Point Sign-Injection Instructions
1608   INSN(vfsgnjx_vv, 0b1010111, 0b001, 0b001010);
1609   INSN(vfsgnjn_vv, 0b1010111, 0b001, 0b001001);
1610   INSN(vfsgnj_vv,  0b1010111, 0b001, 0b001000);
1611 
1612   // Vector Floating-Point MIN/MAX Instructions
1613   INSN(vfmax_vv,   0b1010111, 0b001, 0b000110);
1614   INSN(vfmin_vv,   0b1010111, 0b001, 0b000100);
1615 
1616   // Vector Widening Floating-Point Multiply
1617   INSN(vfwmul_vv,  0b1010111, 0b001, 0b111000);
1618 
1619   // Vector Single-Width Floating-Point Multiply/Divide Instructions
1620   INSN(vfdiv_vv,   0b1010111, 0b001, 0b100000);
1621   INSN(vfmul_vv,   0b1010111, 0b001, 0b100100);
1622 
1623   // Vector Widening Floating-Point Add/Subtract Instructions
1624   INSN(vfwsub_wv, 0b1010111, 0b001, 0b110110);
1625   INSN(vfwsub_vv, 0b1010111, 0b001, 0b110010);
1626   INSN(vfwadd_wv, 0b1010111, 0b001, 0b110100);
1627   INSN(vfwadd_vv, 0b1010111, 0b001, 0b110000);
1628 
1629   // Vector Single-Width Floating-Point Add/Subtract Instructions
1630   INSN(vfsub_vv, 0b1010111, 0b001, 0b000010);
1631   INSN(vfadd_vv, 0b1010111, 0b001, 0b000000);
1632 
1633   // Vector Narrowing Fixed-Point Clip Instructions
1634   INSN(vnclip_wv,  0b1010111, 0b000, 0b101111);
1635   INSN(vnclipu_wv, 0b1010111, 0b000, 0b101110);
1636 
1637   // Vector Single-Width Scaling Shift Instructions
1638   INSN(vssra_vv, 0b1010111, 0b000, 0b101011);
1639   INSN(vssrl_vv, 0b1010111, 0b000, 0b101010);
1640 
1641   // Vector Single-Width Fractional Multiply with Rounding and Saturation
1642   INSN(vsmul_vv, 0b1010111, 0b000, 0b100111);
1643 
1644   // Vector Single-Width Averaging Add and Subtract
1645   INSN(vasubu_vv, 0b1010111, 0b010, 0b001010);
1646   INSN(vasub_vv,  0b1010111, 0b010, 0b001011);
1647   INSN(vaaddu_vv, 0b1010111, 0b010, 0b001000);
1648   INSN(vaadd_vv,  0b1010111, 0b010, 0b001001);
1649 
1650   // Vector Single-Width Saturating Add and Subtract
1651   INSN(vssub_vv,  0b1010111, 0b000, 0b100011);
1652   INSN(vssubu_vv, 0b1010111, 0b000, 0b100010);
1653   INSN(vsadd_vv,  0b1010111, 0b000, 0b100001);
1654   INSN(vsaddu_vv, 0b1010111, 0b000, 0b100000);
1655 
1656   // Vector Widening Integer Multiply Instructions
1657   INSN(vwmul_vv,   0b1010111, 0b010, 0b111011);
1658   INSN(vwmulsu_vv, 0b1010111, 0b010, 0b111010);
1659   INSN(vwmulu_vv,  0b1010111, 0b010, 0b111000);
1660 
1661   // Vector Integer Divide Instructions
1662   INSN(vrem_vv,  0b1010111, 0b010, 0b100011);
1663   INSN(vremu_vv, 0b1010111, 0b010, 0b100010);
1664   INSN(vdiv_vv,  0b1010111, 0b010, 0b100001);
1665   INSN(vdivu_vv, 0b1010111, 0b010, 0b100000);
1666 
1667   // Vector Single-Width Integer Multiply Instructions
1668   INSN(vmulhsu_vv, 0b1010111, 0b010, 0b100110);
1669   INSN(vmulhu_vv,  0b1010111, 0b010, 0b100100);
1670   INSN(vmulh_vv,   0b1010111, 0b010, 0b100111);
1671   INSN(vmul_vv,    0b1010111, 0b010, 0b100101);
1672 
1673   // Vector Integer Min/Max Instructions
1674   INSN(vmax_vv,  0b1010111, 0b000, 0b000111);
1675   INSN(vmaxu_vv, 0b1010111, 0b000, 0b000110);
1676   INSN(vmin_vv,  0b1010111, 0b000, 0b000101);
1677   INSN(vminu_vv, 0b1010111, 0b000, 0b000100);
1678 
1679   // Vector Integer Comparison Instructions
1680   INSN(vmsle_vv,  0b1010111, 0b000, 0b011101);
1681   INSN(vmsleu_vv, 0b1010111, 0b000, 0b011100);
1682   INSN(vmslt_vv,  0b1010111, 0b000, 0b011011);
1683   INSN(vmsltu_vv, 0b1010111, 0b000, 0b011010);
1684   INSN(vmsne_vv,  0b1010111, 0b000, 0b011001);
1685   INSN(vmseq_vv,  0b1010111, 0b000, 0b011000);
1686 
1687   // Vector Narrowing Integer Right Shift Instructions
1688   INSN(vnsra_wv, 0b1010111, 0b000, 0b101101);
1689   INSN(vnsrl_wv, 0b1010111, 0b000, 0b101100);
1690 
1691   // Vector Single-Width Bit Shift Instructions
1692   INSN(vsra_vv, 0b1010111, 0b000, 0b101001);
1693   INSN(vsrl_vv, 0b1010111, 0b000, 0b101000);
1694   INSN(vsll_vv, 0b1010111, 0b000, 0b100101);
1695 
1696   // Vector Bitwise Logical Instructions
1697   INSN(vxor_vv, 0b1010111, 0b000, 0b001011);
1698   INSN(vor_vv,  0b1010111, 0b000, 0b001010);
1699   INSN(vand_vv, 0b1010111, 0b000, 0b001001);
1700 
1701   // Vector Widening Integer Add/Subtract
1702   INSN(vwsub_wv,  0b1010111, 0b010, 0b110111);
1703   INSN(vwsubu_wv, 0b1010111, 0b010, 0b110110);
1704   INSN(vwadd_wv,  0b1010111, 0b010, 0b110101);
1705   INSN(vwaddu_wv, 0b1010111, 0b010, 0b110100);
1706   INSN(vwsub_vv,  0b1010111, 0b010, 0b110011);
1707   INSN(vwsubu_vv, 0b1010111, 0b010, 0b110010);
1708   INSN(vwadd_vv,  0b1010111, 0b010, 0b110001);
1709   INSN(vwaddu_vv, 0b1010111, 0b010, 0b110000);
1710 
1711   // Vector Single-Width Integer Add and Subtract
1712   INSN(vsub_vv, 0b1010111, 0b000, 0b000010);
1713   INSN(vadd_vv, 0b1010111, 0b000, 0b000000);
1714 
1715 #undef INSN
1716 
1717 
1718 #define INSN(NAME, op, funct3, funct6)                                                             \
1719   void NAME(VectorRegister Vd, VectorRegister Vs2, Register Rs1, VectorMask vm = unmasked) {       \
1720     patch_VArith(op, Vd, funct3, Rs1->encoding_nocheck(), Vs2, vm, funct6);                        \
1721   }
1722 
1723   // Vector Register Gather Instruction
1724   INSN(vrgather_vx, 0b1010111, 0b100, 0b001100);
1725 
1726   // Vector Slide Instructions
1727   INSN(vslide1down_vx, 0b1010111, 0b110, 0b001111);
1728   INSN(vslidedown_vx,  0b1010111, 0b100, 0b001111);
1729   INSN(vslide1up_vx,   0b1010111, 0b110, 0b001110);
1730   INSN(vslideup_vx,    0b1010111, 0b100, 0b001110);
1731 
1732   // Vector Narrowing Fixed-Point Clip Instructions
1733   INSN(vnclip_wx,  0b1010111, 0b100, 0b101111);
1734   INSN(vnclipu_wx, 0b1010111, 0b100, 0b101110);
1735 
1736   // Vector Single-Width Scaling Shift Instructions
1737   INSN(vssra_vx, 0b1010111, 0b100, 0b101011);
1738   INSN(vssrl_vx, 0b1010111, 0b100, 0b101010);
1739 
1740   // Vector Single-Width Fractional Multiply with Rounding and Saturation
1741   INSN(vsmul_vx, 0b1010111, 0b100, 0b100111);
1742 
1743   // Vector Single-Width Averaging Add and Subtract
1744   INSN(vasubu_vx, 0b1010111, 0b110, 0b001010);
1745   INSN(vasub_vx,  0b1010111, 0b110, 0b001011);
1746   INSN(vaaddu_vx, 0b1010111, 0b110, 0b001000);
1747   INSN(vaadd_vx,  0b1010111, 0b110, 0b001001);
1748 
1749   // Vector Single-Width Saturating Add and Subtract
1750   INSN(vssub_vx,  0b1010111, 0b100, 0b100011);
1751   INSN(vssubu_vx, 0b1010111, 0b100, 0b100010);
1752   INSN(vsadd_vx,  0b1010111, 0b100, 0b100001);
1753   INSN(vsaddu_vx, 0b1010111, 0b100, 0b100000);
1754 
1755   // Vector Widening Integer Multiply Instructions
1756   INSN(vwmul_vx,   0b1010111, 0b110, 0b111011);
1757   INSN(vwmulsu_vx, 0b1010111, 0b110, 0b111010);
1758   INSN(vwmulu_vx,  0b1010111, 0b110, 0b111000);
1759 
1760   // Vector Integer Divide Instructions
1761   INSN(vrem_vx,  0b1010111, 0b110, 0b100011);
1762   INSN(vremu_vx, 0b1010111, 0b110, 0b100010);
1763   INSN(vdiv_vx,  0b1010111, 0b110, 0b100001);
1764   INSN(vdivu_vx, 0b1010111, 0b110, 0b100000);
1765 
1766   // Vector Single-Width Integer Multiply Instructions
1767   INSN(vmulhsu_vx, 0b1010111, 0b110, 0b100110);
1768   INSN(vmulhu_vx,  0b1010111, 0b110, 0b100100);
1769   INSN(vmulh_vx,   0b1010111, 0b110, 0b100111);
1770   INSN(vmul_vx,    0b1010111, 0b110, 0b100101);
1771 
1772   // Vector Integer Min/Max Instructions
1773   INSN(vmax_vx,  0b1010111, 0b100, 0b000111);
1774   INSN(vmaxu_vx, 0b1010111, 0b100, 0b000110);
1775   INSN(vmin_vx,  0b1010111, 0b100, 0b000101);
1776   INSN(vminu_vx, 0b1010111, 0b100, 0b000100);
1777 
1778   // Vector Integer Comparison Instructions
1779   INSN(vmsgt_vx,  0b1010111, 0b100, 0b011111);
1780   INSN(vmsgtu_vx, 0b1010111, 0b100, 0b011110);
1781   INSN(vmsle_vx,  0b1010111, 0b100, 0b011101);
1782   INSN(vmsleu_vx, 0b1010111, 0b100, 0b011100);
1783   INSN(vmslt_vx,  0b1010111, 0b100, 0b011011);
1784   INSN(vmsltu_vx, 0b1010111, 0b100, 0b011010);
1785   INSN(vmsne_vx,  0b1010111, 0b100, 0b011001);
1786   INSN(vmseq_vx,  0b1010111, 0b100, 0b011000);
1787 
1788   // Vector Narrowing Integer Right Shift Instructions
1789   INSN(vnsra_wx, 0b1010111, 0b100, 0b101101);
1790   INSN(vnsrl_wx, 0b1010111, 0b100, 0b101100);
1791 
1792   // Vector Single-Width Bit Shift Instructions
1793   INSN(vsra_vx, 0b1010111, 0b100, 0b101001);
1794   INSN(vsrl_vx, 0b1010111, 0b100, 0b101000);
1795   INSN(vsll_vx, 0b1010111, 0b100, 0b100101);
1796 
1797   // Vector Bitwise Logical Instructions
1798   INSN(vxor_vx, 0b1010111, 0b100, 0b001011);
1799   INSN(vor_vx,  0b1010111, 0b100, 0b001010);
1800   INSN(vand_vx, 0b1010111, 0b100, 0b001001);
1801 
1802   // Vector Widening Integer Add/Subtract
1803   INSN(vwsub_wx,  0b1010111, 0b110, 0b110111);
1804   INSN(vwsubu_wx, 0b1010111, 0b110, 0b110110);
1805   INSN(vwadd_wx,  0b1010111, 0b110, 0b110101);
1806   INSN(vwadd_wv,  0b1010111, 0b010, 0b110101);
1807   INSN(vwaddu_wx, 0b1010111, 0b110, 0b110100);
1808   INSN(vwsub_vx,  0b1010111, 0b110, 0b110011);
1809   INSN(vwsubu_vx, 0b1010111, 0b110, 0b110010);
1810   INSN(vwadd_vx,  0b1010111, 0b110, 0b110001);
1811   INSN(vwaddu_vx, 0b1010111, 0b110, 0b110000);
1812 
1813   // Vector Single-Width Integer Add and Subtract
1814   INSN(vsub_vx, 0b1010111, 0b100, 0b000010);
1815   INSN(vadd_vx, 0b1010111, 0b100, 0b000000);
1816 
1817 #undef INSN
1818 
1819 #define INSN(NAME, op, funct3, funct6)                                                             \
1820   void NAME(VectorRegister Vd, VectorRegister Vs2, FloatRegister Rs1, VectorMask vm = unmasked) {  \
1821     patch_VArith(op, Vd, funct3, Rs1->encoding_nocheck(), Vs2, vm, funct6);                        \
1822   }
1823 
1824   // Vector Floating-Point Compare Instructions
1825   INSN(vmfge_vf, 0b1010111, 0b101, 0b011111);
1826   INSN(vmfgt_vf, 0b1010111, 0b101, 0b011101);
1827   INSN(vmfle_vf, 0b1010111, 0b101, 0b011001);
1828   INSN(vmflt_vf, 0b1010111, 0b101, 0b011011);
1829   INSN(vmfne_vf, 0b1010111, 0b101, 0b011100);
1830   INSN(vmfeq_vf, 0b1010111, 0b101, 0b011000);
1831 
1832   // Vector Slide1up/Slide1down Instruction
1833   INSN(vfslide1down_vf, 0b1010111, 0b101, 0b001111);
1834   INSN(vfslide1up_vf,   0b1010111, 0b101, 0b001110);
1835 
1836   // Vector Floating-Point Sign-Injection Instructions
1837   INSN(vfsgnjx_vf, 0b1010111, 0b101, 0b001010);
1838   INSN(vfsgnjn_vf, 0b1010111, 0b101, 0b001001);
1839   INSN(vfsgnj_vf,  0b1010111, 0b101, 0b001000);
1840 
1841   // Vector Floating-Point MIN/MAX Instructions
1842   INSN(vfmax_vf, 0b1010111, 0b101, 0b000110);
1843   INSN(vfmin_vf, 0b1010111, 0b101, 0b000100);
1844 
1845   // Vector Widening Floating-Point Multiply
1846   INSN(vfwmul_vf, 0b1010111, 0b101, 0b111000);
1847 
1848   // Vector Single-Width Floating-Point Multiply/Divide Instructions
1849   INSN(vfdiv_vf,  0b1010111, 0b101, 0b100000);
1850   INSN(vfmul_vf,  0b1010111, 0b101, 0b100100);
1851   INSN(vfrdiv_vf, 0b1010111, 0b101, 0b100001);
1852 
1853   // Vector Widening Floating-Point Add/Subtract Instructions
1854   INSN(vfwsub_wf, 0b1010111, 0b101, 0b110110);
1855   INSN(vfwsub_vf, 0b1010111, 0b101, 0b110010);
1856   INSN(vfwadd_wf, 0b1010111, 0b101, 0b110100);
1857   INSN(vfwadd_vf, 0b1010111, 0b101, 0b110000);
1858 
1859   // Vector Single-Width Floating-Point Add/Subtract Instructions
1860   INSN(vfsub_vf,  0b1010111, 0b101, 0b000010);
1861   INSN(vfadd_vf,  0b1010111, 0b101, 0b000000);
1862   INSN(vfrsub_vf, 0b1010111, 0b101, 0b100111);
1863 
1864 #undef INSN
1865 
1866 #define INSN(NAME, op, funct3, funct6)                                                             \
1867   void NAME(VectorRegister Vd, VectorRegister Vs2, int32_t imm, VectorMask vm = unmasked) {        \
1868     guarantee(is_imm_in_range(imm, 5, 0), "imm is invalid");                                       \
1869     patch_VArith(op, Vd, funct3, (uint32_t)imm & 0x1f, Vs2, vm, funct6);                           \
1870   }
1871 
1872   INSN(vsadd_vi,  0b1010111, 0b011, 0b100001);
1873   INSN(vsaddu_vi, 0b1010111, 0b011, 0b100000);
1874   INSN(vmsgt_vi,  0b1010111, 0b011, 0b011111);
1875   INSN(vmsgtu_vi, 0b1010111, 0b011, 0b011110);
1876   INSN(vmsle_vi,  0b1010111, 0b011, 0b011101);
1877   INSN(vmsleu_vi, 0b1010111, 0b011, 0b011100);
1878   INSN(vmsne_vi,  0b1010111, 0b011, 0b011001);
1879   INSN(vmseq_vi,  0b1010111, 0b011, 0b011000);
1880   INSN(vxor_vi,   0b1010111, 0b011, 0b001011);
1881   INSN(vor_vi,    0b1010111, 0b011, 0b001010);
1882   INSN(vand_vi,   0b1010111, 0b011, 0b001001);
1883   INSN(vadd_vi,   0b1010111, 0b011, 0b000000);
1884 
1885 #undef INSN
1886 
1887 #define INSN(NAME, op, funct3, funct6)                                                             \
1888   void NAME(VectorRegister Vd, int32_t imm, VectorRegister Vs2, VectorMask vm = unmasked) {        \
1889     guarantee(is_imm_in_range(imm, 5, 0), "imm is invalid");                                       \
1890     patch_VArith(op, Vd, funct3, (uint32_t)(imm & 0x1f), Vs2, vm, funct6);                         \
1891   }
1892 
1893   INSN(vrsub_vi, 0b1010111, 0b011, 0b000011);
1894 
1895 #undef INSN
1896 
1897 #define INSN(NAME, op, funct3, vm, funct6)                                   \
1898   void NAME(VectorRegister Vd, VectorRegister Vs2, VectorRegister Vs1) {     \
1899     patch_VArith(op, Vd, funct3, Vs1->encoding_nocheck(), Vs2, vm, funct6);  \
1900   }
1901 
1902   // Vector Compress Instruction
1903   INSN(vcompress_vm, 0b1010111, 0b010, 0b1, 0b010111);
1904 
1905   // Vector Mask-Register Logical Instructions
1906   INSN(vmxnor_mm,   0b1010111, 0b010, 0b1, 0b011111);
1907   INSN(vmornot_mm,  0b1010111, 0b010, 0b1, 0b011100);
1908   INSN(vmnor_mm,    0b1010111, 0b010, 0b1, 0b011110);
1909   INSN(vmor_mm,     0b1010111, 0b010, 0b1, 0b011010);
1910   INSN(vmxor_mm,    0b1010111, 0b010, 0b1, 0b011011);
1911   INSN(vmandnot_mm, 0b1010111, 0b010, 0b1, 0b011000);
1912   INSN(vmnand_mm,   0b1010111, 0b010, 0b1, 0b011101);
1913   INSN(vmand_mm,    0b1010111, 0b010, 0b1, 0b011001);
1914 
1915 #undef INSN
1916 
1917 #define INSN(NAME, op, funct3, vm, funct6)                                                  \
1918   void NAME(VectorRegister Vd, VectorRegister Vs2, int32_t imm, VectorRegister V0) {        \
1919     guarantee(is_imm_in_range(imm, 5, 0), "imm is invalid");                                \
1920     patch_VArith(op, Vd, funct3, (uint32_t)(imm & 0x1f), Vs2, vm, funct6);                  \
1921   }
1922 
1923   // Vector Integer Merge Instructions
1924   INSN(vmerge_vim, 0b1010111, 0b011, 0b0, 0b010111);
1925 
1926   // Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
1927   INSN(vadc_vim,   0b1010111, 0b011, 0b0, 0b010000);
1928   INSN(vmadc_vim,  0b1010111, 0b011, 0b0, 0b010001);
1929 
1930 #undef INSN
1931 
1932 #define INSN(NAME, op, funct3, vm, funct6)                                                  \
1933   void NAME(VectorRegister Vd, VectorRegister Vs2, VectorRegister Vs1, VectorRegister V0) { \
1934     patch_VArith(op, Vd, funct3, Vs1->encoding_nocheck(), Vs2, vm, funct6);                 \
1935   }
1936 
1937   // Vector Integer Merge Instructions
1938   INSN(vmerge_vvm, 0b1010111, 0b000, 0b0, 0b010111);
1939 
1940   // Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
1941   INSN(vsbc_vvm, 0b1010111, 0b000, 0b0, 0b010010);
1942   INSN(vadc_vvm, 0b1010111, 0b000, 0b0, 0b010000);
1943 
1944   // Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
1945   INSN(vmadc_vvm, 0b1010111, 0b000, 0b0, 0b010001);
1946   INSN(vmsbc_vvm, 0b1010111, 0b000, 0b0, 0b010011);
1947 
1948 #undef INSN
1949 
1950 #define INSN(NAME, op, funct3, vm, funct6)                                                  \
1951   void NAME(VectorRegister Vd, VectorRegister Vs2, FloatRegister Rs1, VectorRegister V0) {  \
1952     patch_VArith(op, Vd, funct3, Rs1->encoding_nocheck(), Vs2, vm, funct6);                 \
1953   }
1954 
1955   // Vector Floating-Point Merge Instruction
1956   INSN(vfmerge_vfm, 0b1010111, 0b101, 0b0, 0b010111);
1957 
1958 #undef INSN
1959 
1960 #define INSN(NAME, op, funct3, vm, funct6)                                                  \
1961   void NAME(VectorRegister Vd, VectorRegister Vs2, Register Rs1, VectorRegister V0) {       \
1962     patch_VArith(op, Vd, funct3, Rs1->encoding_nocheck(), Vs2, vm, funct6);                 \
1963   }
1964 
1965   // Vector Integer Merge Instructions
1966   INSN(vmerge_vxm, 0b1010111, 0b100, 0b0, 0b010111);
1967 
1968   // Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
1969   INSN(vsbc_vxm, 0b1010111, 0b100, 0b0, 0b010010);
1970   INSN(vadc_vxm, 0b1010111, 0b100, 0b0, 0b010000);
1971 
1972   // Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
1973   INSN(vmadc_vxm, 0b1010111, 0b100, 0b0, 0b010001);
1974   INSN(vmsbc_vxm, 0b1010111, 0b100, 0b0, 0b010011);
1975 
1976 #undef INSN
1977 
1978 #define INSN(NAME, op, funct3, Vs2, vm, funct6)                            \
1979   void NAME(VectorRegister Vd, int32_t imm) {                              \
1980     guarantee(is_imm_in_range(imm, 5, 0), "imm is invalid");               \
1981     patch_VArith(op, Vd, funct3, (uint32_t)(imm & 0x1f), Vs2, vm, funct6); \
1982   }
1983 
1984   // Vector Integer Move Instructions
1985   INSN(vmv_v_i, 0b1010111, 0b011, v0, 0b1, 0b010111);
1986 
1987 #undef INSN
1988 
1989 #define INSN(NAME, op, funct3, Vs2, vm, funct6)                             \
1990   void NAME(VectorRegister Vd, FloatRegister Rs1) {                         \
1991     patch_VArith(op, Vd, funct3, Rs1->encoding_nocheck(), Vs2, vm, funct6); \
1992   }
1993 
1994   // Floating-Point Scalar Move Instructions
1995   INSN(vfmv_s_f, 0b1010111, 0b101, v0, 0b1, 0b010000);
1996   // Vector Floating-Point Move Instruction
1997   INSN(vfmv_v_f, 0b1010111, 0b101, v0, 0b1, 0b010111);
1998 
1999 #undef INSN
2000 
2001 #define INSN(NAME, op, funct3, Vs2, vm, funct6)                             \
2002   void NAME(VectorRegister Vd, VectorRegister Vs1) {                        \
2003     patch_VArith(op, Vd, funct3, Vs1->encoding_nocheck(), Vs2, vm, funct6); \
2004   }
2005 
2006   // Vector Integer Move Instructions
2007   INSN(vmv_v_v, 0b1010111, 0b000, v0, 0b1, 0b010111);
2008 
2009 #undef INSN
2010 
2011 #define INSN(NAME, op, funct3, Vs2, vm, funct6)                             \
2012    void NAME(VectorRegister Vd, Register Rs1) {                             \
2013     patch_VArith(op, Vd, funct3, Rs1->encoding_nocheck(), Vs2, vm, funct6); \
2014    }
2015 
2016   // Integer Scalar Move Instructions
2017   INSN(vmv_s_x, 0b1010111, 0b110, v0, 0b1, 0b010000);
2018 
2019   // Vector Integer Move Instructions
2020   INSN(vmv_v_x, 0b1010111, 0b100, v0, 0b1, 0b010111);
2021 
2022 #undef INSN
2023 #undef patch_VArith
2024 
2025 #define INSN(NAME, op, funct13, funct6)                    \
2026   void NAME(VectorRegister Vd, VectorMask vm = unmasked) { \
2027     unsigned insn = 0;                                     \
2028     patch((address)&insn, 6, 0, op);                       \
2029     patch((address)&insn, 24, 12, funct13);                \
2030     patch((address)&insn, 25, vm);                         \
2031     patch((address)&insn, 31, 26, funct6);                 \
2032     patch_reg((address)&insn, 7, Vd);                      \
2033     emit(insn);                                            \
2034   }
2035 
2036   // Vector Element Index Instruction
2037   INSN(vid_v, 0b1010111, 0b0000010001010, 0b010100);
2038 
2039 #undef INSN
2040 
2041 enum Nf {
2042   g1 = 0b000,
2043   g2 = 0b001,
2044   g3 = 0b010,
2045   g4 = 0b011,
2046   g5 = 0b100,
2047   g6 = 0b101,
2048   g7 = 0b110,
2049   g8 = 0b111
2050 };
2051 
2052 #define patch_VLdSt(op, VReg, width, Rs1, Reg_or_umop, vm, mop, mew, nf) \
2053     unsigned insn = 0;                                                   \
2054     patch((address)&insn, 6, 0, op);                                     \
2055     patch((address)&insn, 14, 12, width);                                \
2056     patch((address)&insn, 24, 20, Reg_or_umop);                          \
2057     patch((address)&insn, 25, vm);                                       \
2058     patch((address)&insn, 27, 26, mop);                                  \
2059     patch((address)&insn, 28, mew);                                      \
2060     patch((address)&insn, 31, 29, nf);                                   \
2061     patch_reg((address)&insn, 7, VReg);                                  \
2062     patch_reg((address)&insn, 15, Rs1);                                  \
2063     emit(insn)
2064 
2065 #define INSN(NAME, op, lumop, vm, mop, nf)                                           \
2066   void NAME(VectorRegister Vd, Register Rs1, uint32_t width = 0, bool mew = false) { \
2067     guarantee(is_unsigned_imm_in_range(width, 3, 0), "width is invalid");            \
2068     patch_VLdSt(op, Vd, width, Rs1, lumop, vm, mop, mew, nf);                        \
2069   }
2070 
2071   // Vector Load/Store Instructions
2072   INSN(vl1r_v, 0b0000111, 0b01000, 0b1, 0b00, g1);
2073 
2074 #undef INSN
2075 
2076 #define INSN(NAME, op, width, sumop, vm, mop, mew, nf)           \
2077   void NAME(VectorRegister Vs3, Register Rs1) {                  \
2078     patch_VLdSt(op, Vs3, width, Rs1, sumop, vm, mop, mew, nf);   \
2079   }
2080 
2081   // Vector Load/Store Instructions
2082   INSN(vs1r_v, 0b0100111, 0b000, 0b01000, 0b1, 0b00, 0b0, g1);
2083 
2084 #undef INSN
2085 
2086 // r2_nfvm
2087 #define INSN(NAME, op, width, umop, mop, mew)                         \
2088   void NAME(VectorRegister Vd_or_Vs3, Register Rs1, Nf nf = g1) {     \
2089     patch_VLdSt(op, Vd_or_Vs3, width, Rs1, umop, 1, mop, mew, nf);    \
2090   }
2091 
2092   // Vector Unit-Stride Instructions
2093   INSN(vle1_v, 0b0000111, 0b000, 0b01011, 0b00, 0b0);
2094   INSN(vse1_v, 0b0100111, 0b000, 0b01011, 0b00, 0b0);
2095 
2096 #undef INSN
2097 
2098 #define INSN(NAME, op, width, umop, mop, mew)                                               \
2099   void NAME(VectorRegister Vd_or_Vs3, Register Rs1, VectorMask vm = unmasked, Nf nf = g1) { \
2100     patch_VLdSt(op, Vd_or_Vs3, width, Rs1, umop, vm, mop, mew, nf);                         \
2101   }
2102 
2103   // Vector Unit-Stride Instructions
2104   INSN(vle8_v,    0b0000111, 0b000, 0b00000, 0b00, 0b0);
2105   INSN(vle16_v,   0b0000111, 0b101, 0b00000, 0b00, 0b0);
2106   INSN(vle32_v,   0b0000111, 0b110, 0b00000, 0b00, 0b0);
2107   INSN(vle64_v,   0b0000111, 0b111, 0b00000, 0b00, 0b0);
2108 
2109   // Vector unit-stride fault-only-first Instructions
2110   INSN(vle8ff_v,  0b0000111, 0b000, 0b10000, 0b00, 0b0);
2111   INSN(vle16ff_v, 0b0000111, 0b101, 0b10000, 0b00, 0b0);
2112   INSN(vle32ff_v, 0b0000111, 0b110, 0b10000, 0b00, 0b0);
2113   INSN(vle64ff_v, 0b0000111, 0b111, 0b10000, 0b00, 0b0);
2114 
2115   INSN(vse8_v,  0b0100111, 0b000, 0b00000, 0b00, 0b0);
2116   INSN(vse16_v, 0b0100111, 0b101, 0b00000, 0b00, 0b0);
2117   INSN(vse32_v, 0b0100111, 0b110, 0b00000, 0b00, 0b0);
2118   INSN(vse64_v, 0b0100111, 0b111, 0b00000, 0b00, 0b0);
2119 
2120 #undef INSN
2121 
2122 #define INSN(NAME, op, width, mop, mew)                                                                  \
2123   void NAME(VectorRegister Vd, Register Rs1, VectorRegister Vs2, VectorMask vm = unmasked, Nf nf = g1) { \
2124     patch_VLdSt(op, Vd, width, Rs1, Vs2->encoding_nocheck(), vm, mop, mew, nf);                          \
2125   }
2126 
2127   // Vector unordered indexed load instructions
2128   INSN(vluxei8_v,  0b0000111, 0b000, 0b01, 0b0);
2129   INSN(vluxei16_v, 0b0000111, 0b101, 0b01, 0b0);
2130   INSN(vluxei32_v, 0b0000111, 0b110, 0b01, 0b0);
2131   INSN(vluxei64_v, 0b0000111, 0b111, 0b01, 0b0);
2132 
2133   // Vector ordered indexed load instructions
2134   INSN(vloxei8_v,  0b0000111, 0b000, 0b11, 0b0);
2135   INSN(vloxei16_v, 0b0000111, 0b101, 0b11, 0b0);
2136   INSN(vloxei32_v, 0b0000111, 0b110, 0b11, 0b0);
2137   INSN(vloxei64_v, 0b0000111, 0b111, 0b11, 0b0);
2138 #undef INSN
2139 
2140 #define INSN(NAME, op, width, mop, mew)                                                                  \
2141   void NAME(VectorRegister Vd, Register Rs1, Register Rs2, VectorMask vm = unmasked, Nf nf = g1) {       \
2142     patch_VLdSt(op, Vd, width, Rs1, Rs2->encoding_nocheck(), vm, mop, mew, nf);                          \
2143   }
2144 
2145   // Vector Strided Instructions
2146   INSN(vlse8_v,  0b0000111, 0b000, 0b10, 0b0);
2147   INSN(vlse16_v, 0b0000111, 0b101, 0b10, 0b0);
2148   INSN(vlse32_v, 0b0000111, 0b110, 0b10, 0b0);
2149   INSN(vlse64_v, 0b0000111, 0b111, 0b10, 0b0);
2150 
2151 #undef INSN
2152 
2153 #define INSN(NAME, op, width, mop, mew)                                                                   \
2154   void NAME(VectorRegister Vs3, Register Rs1, VectorRegister Vs2, VectorMask vm = unmasked, Nf nf = g1) { \
2155     patch_VLdSt(op, Vs3, width, Rs1, Vs2->encoding_nocheck(), vm, mop, mew, nf);                          \
2156   }
2157 
2158   // Vector unordered-indexed store instructions
2159   INSN(vsuxei8_v,  0b0100111, 0b000, 0b01, 0b0);
2160   INSN(vsuxei16_v, 0b0100111, 0b101, 0b01, 0b0);
2161   INSN(vsuxei32_v, 0b0100111, 0b110, 0b01, 0b0);
2162   INSN(vsuxei64_v, 0b0100111, 0b111, 0b01, 0b0);
2163 
2164   // Vector ordered indexed store instructions
2165   INSN(vsoxei8_v,  0b0100111, 0b000, 0b11, 0b0);
2166   INSN(vsoxei16_v, 0b0100111, 0b101, 0b11, 0b0);
2167   INSN(vsoxei32_v, 0b0100111, 0b110, 0b11, 0b0);
2168   INSN(vsoxei64_v, 0b0100111, 0b111, 0b11, 0b0);
2169 
2170 #undef INSN
2171 
2172 #define INSN(NAME, op, width, mop, mew)                                                                   \
2173   void NAME(VectorRegister Vs3, Register Rs1, Register Rs2, VectorMask vm = unmasked, Nf nf = g1) {       \
2174     patch_VLdSt(op, Vs3, width, Rs1, Rs2->encoding_nocheck(), vm, mop, mew, nf);                          \
2175   }
2176 
2177   INSN(vsse8_v,  0b0100111, 0b000, 0b10, 0b0);
2178   INSN(vsse16_v, 0b0100111, 0b101, 0b10, 0b0);
2179   INSN(vsse32_v, 0b0100111, 0b110, 0b10, 0b0);
2180   INSN(vsse64_v, 0b0100111, 0b111, 0b10, 0b0);
2181 
2182 #undef INSN
2183 #undef patch_VLdSt
2184 
2185   void bgt(Register Rs, Register Rt, const address &dest);
2186   void ble(Register Rs, Register Rt, const address &dest);
2187   void bgtu(Register Rs, Register Rt, const address &dest);
2188   void bleu(Register Rs, Register Rt, const address &dest);
2189   void bgt(Register Rs, Register Rt, Label &l, bool is_far = false);
2190   void ble(Register Rs, Register Rt, Label &l, bool is_far = false);
2191   void bgtu(Register Rs, Register Rt, Label &l, bool is_far = false);
2192   void bleu(Register Rs, Register Rt, Label &l, bool is_far = false);
2193 
2194   typedef void (Assembler::* jal_jalr_insn)(Register Rt, address dest);
2195   typedef void (Assembler::* load_insn_by_temp)(Register Rt, address dest, Register temp);
2196   typedef void (Assembler::* compare_and_branch_insn)(Register Rs1, Register Rs2, const address dest);
2197   typedef void (Assembler::* compare_and_branch_label_insn)(Register Rs1, Register Rs2, Label &L, bool is_far);
2198 
2199   void wrap_label(Register r1, Register r2, Label &L, compare_and_branch_insn insn,
2200                   compare_and_branch_label_insn neg_insn, bool is_far);
2201   void wrap_label(Register r, Label &L, Register t, load_insn_by_temp insn);
2202   void wrap_label(Register r, Label &L, jal_jalr_insn insn);
2203 
2204   // calculate pseudoinstruction
2205   void add(Register Rd, Register Rn, int64_t increment, Register temp = t0);
2206   void addw(Register Rd, Register Rn, int64_t increment, Register temp = t0);
2207   void sub(Register Rd, Register Rn, int64_t decrement, Register temp = t0);
2208   void subw(Register Rd, Register Rn, int64_t decrement, Register temp = t0);
2209 
2210   Assembler(CodeBuffer* code) : AbstractAssembler(code) {
2211   }
2212 
2213   virtual RegisterOrConstant delayed_value_impl(intptr_t* delayed_value_addr,
2214                                                 Register tmp,
2215                                                 int offset) {
2216     ShouldNotCallThis();
2217     return RegisterOrConstant();
2218   }
2219 
2220   // Stack overflow checking
2221   virtual void bang_stack_with_offset(int offset) { Unimplemented(); }
2222 
2223   static bool operand_valid_for_add_immediate(long imm) {
2224     return is_imm_in_range(imm, 12, 0);
2225   }
2226 
2227   // The maximum range of a branch is fixed for the riscv64
2228   // architecture.
2229   static const unsigned long branch_range = 1 * M;
2230 
2231   static bool reachable_from_branch_at(address branch, address target) {
2232     return uabs(target - branch) < branch_range;
2233   }
2234 
2235   static Assembler::SEW elemBytes_to_sew(int esize) {
2236     assert(esize > 0 && esize <= 64 && is_power_of_2(esize), "unsupported element size");
2237     return (Assembler::SEW) log2i_exact(esize);
2238   }
2239 
2240   virtual ~Assembler() {}
2241 
2242 };
2243 
2244 class BiasedLockingCounters;
2245 
2246 #endif // CPU_RISCV_ASSEMBLER_RISCV_HPP