1 /*
   2  * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2024, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #ifndef CPU_AARCH64_ASSEMBLER_AARCH64_HPP
  27 #define CPU_AARCH64_ASSEMBLER_AARCH64_HPP
  28 
  29 #include "asm/register.hpp"
  30 #include "metaprogramming/enableIf.hpp"
  31 #include "utilities/checkedCast.hpp"
  32 #include "utilities/debug.hpp"
  33 #include "utilities/globalDefinitions.hpp"
  34 #include "utilities/macros.hpp"
  35 #include <type_traits>
  36 
  37 #ifdef __GNUC__
  38 
  39 // __nop needs volatile so that compiler doesn't optimize it away
  40 #define NOP() asm volatile ("nop");
  41 
  42 #elif defined(_MSC_VER)
  43 
  44 // Use MSVC intrinsic: https://docs.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=vs-2019#I
  45 #define NOP() __nop();
  46 
  47 #endif
  48 
  49 
  50 // definitions of various symbolic names for machine registers
  51 
  52 // First intercalls between C and Java which use 8 general registers
  53 // and 8 floating registers
  54 
  55 // we also have to copy between x86 and ARM registers but that's a
  56 // secondary complication -- not all code employing C call convention
  57 // executes as x86 code though -- we generate some of it
  58 
  59 class Argument {
  60  public:
  61   enum {
  62     n_int_register_parameters_c   = 8,  // r0, r1, ... r7 (c_rarg0, c_rarg1, ...)
  63     n_float_register_parameters_c = 8,  // v0, v1, ... v7 (c_farg0, c_farg1, ... )
  64 
  65     n_int_register_parameters_j   = 8, // r1, ... r7, r0 (rj_rarg0, j_rarg1, ...
  66     n_float_register_parameters_j = 8  // v0, v1, ... v7 (j_farg0, j_farg1, ...
  67   };
  68 };
  69 
  70 constexpr Register c_rarg0 = r0;
  71 constexpr Register c_rarg1 = r1;
  72 constexpr Register c_rarg2 = r2;
  73 constexpr Register c_rarg3 = r3;
  74 constexpr Register c_rarg4 = r4;
  75 constexpr Register c_rarg5 = r5;
  76 constexpr Register c_rarg6 = r6;
  77 constexpr Register c_rarg7 = r7;
  78 
  79 constexpr FloatRegister c_farg0 = v0;
  80 constexpr FloatRegister c_farg1 = v1;
  81 constexpr FloatRegister c_farg2 = v2;
  82 constexpr FloatRegister c_farg3 = v3;
  83 constexpr FloatRegister c_farg4 = v4;
  84 constexpr FloatRegister c_farg5 = v5;
  85 constexpr FloatRegister c_farg6 = v6;
  86 constexpr FloatRegister c_farg7 = v7;
  87 
  88 // Symbolically name the register arguments used by the Java calling convention.
  89 // We have control over the convention for java so we can do what we please.
  90 // What pleases us is to offset the java calling convention so that when
  91 // we call a suitable jni method the arguments are lined up and we don't
  92 // have to do much shuffling. A suitable jni method is non-static and a
  93 // small number of arguments
  94 //
  95 //  |--------------------------------------------------------------------|
  96 //  | c_rarg0  c_rarg1  c_rarg2 c_rarg3 c_rarg4 c_rarg5 c_rarg6 c_rarg7  |
  97 //  |--------------------------------------------------------------------|
  98 //  | r0       r1       r2      r3      r4      r5      r6      r7       |
  99 //  |--------------------------------------------------------------------|
 100 //  | j_rarg7  j_rarg0  j_rarg1 j_rarg2 j_rarg3 j_rarg4 j_rarg5 j_rarg6  |
 101 //  |--------------------------------------------------------------------|
 102 
 103 
 104 constexpr Register j_rarg0 = c_rarg1;
 105 constexpr Register j_rarg1 = c_rarg2;
 106 constexpr Register j_rarg2 = c_rarg3;
 107 constexpr Register j_rarg3 = c_rarg4;
 108 constexpr Register j_rarg4 = c_rarg5;
 109 constexpr Register j_rarg5 = c_rarg6;
 110 constexpr Register j_rarg6 = c_rarg7;
 111 constexpr Register j_rarg7 = c_rarg0;
 112 
 113 // Java floating args are passed as per C
 114 
 115 constexpr FloatRegister j_farg0 = v0;
 116 constexpr FloatRegister j_farg1 = v1;
 117 constexpr FloatRegister j_farg2 = v2;
 118 constexpr FloatRegister j_farg3 = v3;
 119 constexpr FloatRegister j_farg4 = v4;
 120 constexpr FloatRegister j_farg5 = v5;
 121 constexpr FloatRegister j_farg6 = v6;
 122 constexpr FloatRegister j_farg7 = v7;
 123 
 124 // registers used to hold VM data either temporarily within a method
 125 // or across method calls
 126 
 127 // volatile (caller-save) registers
 128 
 129 // r8 is used for indirect result location return
 130 // we use it and r9 as scratch registers
 131 constexpr Register rscratch1 = r8;
 132 constexpr Register rscratch2 = r9;
 133 
 134 // current method -- must be in a call-clobbered register
 135 constexpr Register rmethod = r12;
 136 
 137 // non-volatile (callee-save) registers are r16-29
 138 // of which the following are dedicated global state
 139 
 140 constexpr Register lr            = r30; // link register
 141 constexpr Register rfp           = r29; // frame pointer
 142 constexpr Register rthread       = r28; // current thread
 143 constexpr Register rheapbase     = r27; // base of heap
 144 constexpr Register rcpool        = r26; // constant pool cache
 145 constexpr Register rlocals       = r24; // locals on stack
 146 constexpr Register rbcp          = r22; // bytecode pointer
 147 constexpr Register rdispatch     = r21; // dispatch table base
 148 constexpr Register esp           = r20; // Java expression stack pointer
 149 constexpr Register r19_sender_sp = r19; // sender's SP while in interpreter
 150 
 151 // Preserved predicate register with all elements set TRUE.
 152 constexpr PRegister ptrue = p7;
 153 
 154 #define assert_cond(ARG1) assert(ARG1, #ARG1)
 155 
 156 namespace asm_util {
 157   uint32_t encode_logical_immediate(bool is32, uint64_t imm);
 158   uint32_t encode_sve_logical_immediate(unsigned elembits, uint64_t imm);
 159   bool operand_valid_for_immediate_bits(int64_t imm, unsigned nbits);
 160 };
 161 
 162 using namespace asm_util;
 163 
 164 
 165 class Assembler;
 166 
 167 class Instruction_aarch64 {
 168   unsigned insn;
 169 #ifdef ASSERT
 170   unsigned bits;
 171 #endif
 172   Assembler *assem;
 173 
 174 public:
 175 
 176   Instruction_aarch64(class Assembler *as) {
 177 #ifdef ASSERT
 178     bits = 0;
 179 #endif
 180     insn = 0;
 181     assem = as;
 182   }
 183 
 184   inline ~Instruction_aarch64();
 185 
 186   unsigned &get_insn() { return insn; }
 187 #ifdef ASSERT
 188   unsigned &get_bits() { return bits; }
 189 #endif
 190 
 191   static inline int32_t extend(unsigned val, int hi = 31, int lo = 0) {
 192     union {
 193       unsigned u;
 194       int n;
 195     };
 196 
 197     u = val << (31 - hi);
 198     n = n >> (31 - hi + lo);
 199     return n;
 200   }
 201 
 202   static inline uint32_t extract(uint32_t val, int msb, int lsb) {
 203     int nbits = msb - lsb + 1;
 204     assert_cond(msb >= lsb);
 205     uint32_t mask = checked_cast<uint32_t>(right_n_bits(nbits));
 206     uint32_t result = val >> lsb;
 207     result &= mask;
 208     return result;
 209   }
 210 
 211   static inline int32_t sextract(uint32_t val, int msb, int lsb) {
 212     uint32_t uval = extract(val, msb, lsb);
 213     return extend(uval, msb - lsb);
 214   }
 215 
 216   static ALWAYSINLINE void patch(address a, int msb, int lsb, uint64_t val) {
 217     int nbits = msb - lsb + 1;
 218     guarantee(val < (1ULL << nbits), "Field too big for insn");
 219     assert_cond(msb >= lsb);
 220     unsigned mask = checked_cast<unsigned>(right_n_bits(nbits));
 221     val <<= lsb;
 222     mask <<= lsb;
 223     unsigned target = *(unsigned *)a;
 224     target &= ~mask;
 225     target |= (unsigned)val;
 226     *(unsigned *)a = target;
 227   }
 228 
 229   static void spatch(address a, int msb, int lsb, int64_t val) {
 230     int nbits = msb - lsb + 1;
 231     int64_t chk = val >> (nbits - 1);
 232     guarantee (chk == -1 || chk == 0, "Field too big for insn at " INTPTR_FORMAT, p2i(a));
 233     uint64_t uval = val;
 234     unsigned mask = checked_cast<unsigned>(right_n_bits(nbits));
 235     uval &= mask;
 236     uval <<= lsb;
 237     mask <<= lsb;
 238     unsigned target = *(unsigned *)a;
 239     target &= ~mask;
 240     target |= (unsigned)uval;
 241     *(unsigned *)a = target;
 242   }
 243 
 244   void f(unsigned val, int msb, int lsb) {
 245     int nbits = msb - lsb + 1;
 246     guarantee(val < (1ULL << nbits), "Field too big for insn");
 247     assert_cond(msb >= lsb);
 248     val <<= lsb;
 249     insn |= val;
 250 #ifdef ASSERT
 251     unsigned mask = checked_cast<unsigned>(right_n_bits(nbits));
 252     mask <<= lsb;
 253     assert_cond((bits & mask) == 0);
 254     bits |= mask;
 255 #endif
 256   }
 257 
 258   void f(unsigned val, int bit) {
 259     f(val, bit, bit);
 260   }
 261 
 262   void sf(int64_t val, int msb, int lsb) {
 263     int nbits = msb - lsb + 1;
 264     int64_t chk = val >> (nbits - 1);
 265     guarantee (chk == -1 || chk == 0, "Field too big for insn");
 266     uint64_t uval = val;
 267     unsigned mask = checked_cast<unsigned>(right_n_bits(nbits));
 268     uval &= mask;
 269     f((unsigned)uval, lsb + nbits - 1, lsb);
 270   }
 271 
 272   void rf(Register r, int lsb) {
 273     f(r->raw_encoding(), lsb + 4, lsb);
 274   }
 275 
 276   // reg|ZR
 277   void zrf(Register r, int lsb) {
 278     f(r->raw_encoding() - (r == zr), lsb + 4, lsb);
 279   }
 280 
 281   // reg|SP
 282   void srf(Register r, int lsb) {
 283     f(r == sp ? 31 : r->raw_encoding(), lsb + 4, lsb);
 284   }
 285 
 286   void rf(FloatRegister r, int lsb) {
 287     f(r->raw_encoding(), lsb + 4, lsb);
 288   }
 289 
 290   //<0-15>reg: As `rf(FloatRegister)`, but only the lower  16 FloatRegisters are allowed.
 291   void lrf(FloatRegister r, int lsb) {
 292     f(r->raw_encoding(), lsb + 3, lsb);
 293   }
 294 
 295   void prf(PRegister r, int lsb) {
 296     f(r->raw_encoding(), lsb + 3, lsb);
 297   }
 298 
 299   void pgrf(PRegister r, int lsb) {
 300     f(r->raw_encoding(), lsb + 2, lsb);
 301   }
 302 
 303   unsigned get(int msb = 31, int lsb = 0) {
 304     int nbits = msb - lsb + 1;
 305     unsigned mask = checked_cast<unsigned>(right_n_bits(nbits)) << lsb;
 306     assert_cond((bits & mask) == mask);
 307     return (insn & mask) >> lsb;
 308   }
 309 };
 310 
 311 #define starti Instruction_aarch64 current_insn(this);
 312 
 313 class PrePost {
 314   int _offset;
 315   Register _r;
 316 protected:
 317   PrePost(Register reg, int o) : _offset(o), _r(reg) { }
 318   ~PrePost() = default;
 319   PrePost(const PrePost&) = default;
 320   PrePost& operator=(const PrePost&) = default;
 321 public:
 322   int offset() const { return _offset; }
 323   Register reg() const { return _r; }
 324 };
 325 
 326 class Pre : public PrePost {
 327 public:
 328   Pre(Register reg, int o) : PrePost(reg, o) { }
 329 };
 330 
 331 class Post : public PrePost {
 332   Register _idx;
 333   bool _is_postreg;
 334 public:
 335   Post(Register reg, int o) : PrePost(reg, o), _idx(noreg), _is_postreg(false) {}
 336   Post(Register reg, Register idx) : PrePost(reg, 0), _idx(idx), _is_postreg(true) {}
 337   Register idx_reg() const { return _idx; }
 338   bool is_postreg() const { return _is_postreg; }
 339 };
 340 
 341 namespace ext
 342 {
 343   enum operation { uxtb, uxth, uxtw, uxtx, sxtb, sxth, sxtw, sxtx };
 344 };
 345 
 346 // Addressing modes
 347 class Address {
 348  public:
 349 
 350   enum mode { no_mode, base_plus_offset, pre, post, post_reg,
 351               base_plus_offset_reg, literal };
 352 
 353   // Shift and extend for base reg + reg offset addressing
 354   class extend {
 355     int _option, _shift;
 356     ext::operation _op;
 357   public:
 358     extend() { }
 359     extend(int s, int o, ext::operation op) : _option(o), _shift(s), _op(op) { }
 360     int option() const{ return _option; }
 361     int shift() const { return _shift; }
 362     ext::operation op() const { return _op; }
 363   };
 364 
 365   static extend uxtw(int shift = -1) { return extend(shift, 0b010, ext::uxtw); }
 366   static extend lsl(int shift = -1)  { return extend(shift, 0b011, ext::uxtx); }
 367   static extend sxtw(int shift = -1) { return extend(shift, 0b110, ext::sxtw); }
 368   static extend sxtx(int shift = -1) { return extend(shift, 0b111, ext::sxtx); }
 369 
 370  private:
 371   struct Nonliteral {
 372     Nonliteral(Register base, Register index, int64_t offset, extend ext = extend())
 373       : _base(base), _index(index), _offset(offset), _ext(ext) {}
 374     Register _base;
 375     Register _index;
 376     int64_t _offset;
 377     extend _ext;
 378   };
 379 
 380   struct Literal {
 381     Literal(address target, const RelocationHolder& rspec)
 382       : _target(target), _rspec(rspec) {}
 383 
 384     // If the target is far we'll need to load the ea of this to a
 385     // register to reach it. Otherwise if near we can do PC-relative
 386     // addressing.
 387     address _target;
 388 
 389     RelocationHolder _rspec;
 390   };
 391 
 392   void assert_is_nonliteral() const NOT_DEBUG_RETURN;
 393   void assert_is_literal() const NOT_DEBUG_RETURN;
 394 
 395   // Discriminated union, based on _mode.
 396   // - no_mode: uses dummy _nonliteral, for ease of copying.
 397   // - literal: only _literal is used.
 398   // - others: only _nonliteral is used.
 399   enum mode _mode;
 400   union {
 401     Nonliteral _nonliteral;
 402     Literal _literal;
 403   };
 404 
 405   // Helper for copy constructor and assignment operator.
 406   // Copy mode-relevant part of a into this.
 407   void copy_data(const Address& a) {
 408     assert(_mode == a._mode, "precondition");
 409     if (_mode == literal) {
 410       new (&_literal) Literal(a._literal);
 411     } else {
 412       // non-literal mode or no_mode.
 413       new (&_nonliteral) Nonliteral(a._nonliteral);
 414     }
 415   }
 416 
 417  public:
 418   // no_mode initializes _nonliteral for ease of copying.
 419   Address() :
 420     _mode(no_mode),
 421     _nonliteral(noreg, noreg, 0)
 422   {}
 423 
 424   Address(Register r) :
 425     _mode(base_plus_offset),
 426     _nonliteral(r, noreg, 0)
 427   {}
 428 
 429   template<typename T, ENABLE_IF(std::is_integral<T>::value)>
 430   Address(Register r, T o) :
 431     _mode(base_plus_offset),
 432     _nonliteral(r, noreg, o)
 433   {}
 434 
 435   Address(Register r, ByteSize disp) : Address(r, in_bytes(disp)) {}
 436 
 437   Address(Register r, Register r1, extend ext = lsl()) :
 438     _mode(base_plus_offset_reg),
 439     _nonliteral(r, r1, 0, ext)
 440   {}
 441 
 442   Address(Pre p) :
 443     _mode(pre),
 444     _nonliteral(p.reg(), noreg, p.offset())
 445   {}
 446 
 447   Address(Post p) :
 448     _mode(p.is_postreg() ? post_reg : post),
 449     _nonliteral(p.reg(), p.idx_reg(), p.offset())
 450   {}
 451 
 452   Address(address target, const RelocationHolder& rspec) :
 453     _mode(literal),
 454     _literal(target, rspec)
 455   {}
 456 
 457   Address(address target, relocInfo::relocType rtype = relocInfo::external_word_type);
 458 
 459   Address(Register base, RegisterOrConstant index, extend ext = lsl()) {
 460     if (index.is_register()) {
 461       _mode = base_plus_offset_reg;
 462       new (&_nonliteral) Nonliteral(base, index.as_register(), 0, ext);
 463     } else {
 464       guarantee(ext.option() == ext::uxtx, "should be");
 465       assert(index.is_constant(), "should be");
 466       _mode = base_plus_offset;
 467       new (&_nonliteral) Nonliteral(base,
 468                                     noreg,
 469                                     index.as_constant() << ext.shift());
 470     }
 471   }
 472 
 473   Address(const Address& a) : _mode(a._mode) { copy_data(a); }
 474 
 475   // Verify the value is trivially destructible regardless of mode, so our
 476   // destructor can also be trivial, and so our assignment operator doesn't
 477   // need to destruct the old value before copying over it.
 478   static_assert(std::is_trivially_destructible<Literal>::value, "must be");
 479   static_assert(std::is_trivially_destructible<Nonliteral>::value, "must be");
 480 
 481   Address& operator=(const Address& a) {
 482     _mode = a._mode;
 483     copy_data(a);
 484     return *this;
 485   }
 486 
 487   ~Address() = default;
 488 
 489   Register base() const {
 490     assert_is_nonliteral();
 491     return _nonliteral._base;
 492   }
 493 
 494   int64_t offset() const {
 495     assert_is_nonliteral();
 496     return _nonliteral._offset;
 497   }
 498 
 499   Register index() const {
 500     assert_is_nonliteral();
 501     return _nonliteral._index;
 502   }
 503 
 504   extend ext() const {
 505     assert_is_nonliteral();
 506     return _nonliteral._ext;
 507   }
 508 
 509   mode getMode() const {
 510     return _mode;
 511   }
 512 
 513   bool uses(Register reg) const {
 514     switch (_mode) {
 515     case literal:
 516     case no_mode:
 517       return false;
 518     case base_plus_offset:
 519     case base_plus_offset_reg:
 520     case pre:
 521     case post:
 522     case post_reg:
 523       return base() == reg || index() == reg;
 524     default:
 525       ShouldNotReachHere();
 526       return false;
 527     }
 528   }
 529 
 530   address target() const {
 531     assert_is_literal();
 532     return _literal._target;
 533   }
 534 
 535   const RelocationHolder& rspec() const {
 536     assert_is_literal();
 537     return _literal._rspec;
 538   }
 539 
 540   void encode(Instruction_aarch64 *i) const {
 541     i->f(0b111, 29, 27);
 542     i->srf(base(), 5);
 543 
 544     switch(_mode) {
 545     case base_plus_offset:
 546       {
 547         unsigned size = i->get(31, 30);
 548         if (i->get(26, 26) && i->get(23, 23)) {
 549           // SIMD Q Type - Size = 128 bits
 550           assert(size == 0, "bad size");
 551           size = 0b100;
 552         }
 553         assert(offset_ok_for_immed(offset(), size),
 554                "must be, was: " INT64_FORMAT ", %d", offset(), size);
 555         unsigned mask = (1 << size) - 1;
 556         if (offset() < 0 || offset() & mask) {
 557           i->f(0b00, 25, 24);
 558           i->f(0, 21), i->f(0b00, 11, 10);
 559           i->sf(offset(), 20, 12);
 560         } else {
 561           i->f(0b01, 25, 24);
 562           i->f(checked_cast<unsigned>(offset() >> size), 21, 10);
 563         }
 564       }
 565       break;
 566 
 567     case base_plus_offset_reg:
 568       {
 569         i->f(0b00, 25, 24);
 570         i->f(1, 21);
 571         i->rf(index(), 16);
 572         i->f(ext().option(), 15, 13);
 573         unsigned size = i->get(31, 30);
 574         if (i->get(26, 26) && i->get(23, 23)) {
 575           // SIMD Q Type - Size = 128 bits
 576           assert(size == 0, "bad size");
 577           size = 0b100;
 578         }
 579         if (size == 0) // It's a byte
 580           i->f(ext().shift() >= 0, 12);
 581         else {
 582           guarantee(ext().shift() <= 0 || ext().shift() == (int)size, "bad shift");
 583           i->f(ext().shift() > 0, 12);
 584         }
 585         i->f(0b10, 11, 10);
 586       }
 587       break;
 588 
 589     case pre:
 590       i->f(0b00, 25, 24);
 591       i->f(0, 21), i->f(0b11, 11, 10);
 592       i->sf(offset(), 20, 12);
 593       break;
 594 
 595     case post:
 596       i->f(0b00, 25, 24);
 597       i->f(0, 21), i->f(0b01, 11, 10);
 598       i->sf(offset(), 20, 12);
 599       break;
 600 
 601     default:
 602       ShouldNotReachHere();
 603     }
 604   }
 605 
 606   void encode_pair(Instruction_aarch64 *i) const {
 607     switch(_mode) {
 608     case base_plus_offset:
 609       i->f(0b010, 25, 23);
 610       break;
 611     case pre:
 612       i->f(0b011, 25, 23);
 613       break;
 614     case post:
 615       i->f(0b001, 25, 23);
 616       break;
 617     default:
 618       ShouldNotReachHere();
 619     }
 620 
 621     unsigned size; // Operand shift in 32-bit words
 622 
 623     if (i->get(26, 26)) { // float
 624       switch(i->get(31, 30)) {
 625       case 0b10:
 626         size = 2; break;
 627       case 0b01:
 628         size = 1; break;
 629       case 0b00:
 630         size = 0; break;
 631       default:
 632         ShouldNotReachHere();
 633         size = 0;  // unreachable
 634       }
 635     } else {
 636       size = i->get(31, 31);
 637     }
 638 
 639     size = 4 << size;
 640     guarantee(offset() % size == 0, "bad offset");
 641     i->sf(offset() / size, 21, 15);
 642     i->srf(base(), 5);
 643   }
 644 
 645   void encode_nontemporal_pair(Instruction_aarch64 *i) const {
 646     guarantee(_mode == base_plus_offset, "Bad addressing mode for nontemporal op");
 647     i->f(0b000, 25, 23);
 648     unsigned size = i->get(31, 31);
 649     size = 4 << size;
 650     guarantee(offset() % size == 0, "bad offset");
 651     i->sf(offset() / size, 21, 15);
 652     i->srf(base(), 5);
 653   }
 654 
 655   void lea(MacroAssembler *, Register) const;
 656 
 657   static bool offset_ok_for_immed(int64_t offset, uint shift);
 658 
 659   static bool offset_ok_for_sve_immed(int64_t offset, int shift, int vl /* sve vector length */) {
 660     if (offset % vl == 0) {
 661       // Convert address offset into sve imm offset (MUL VL).
 662       int64_t sve_offset = offset / vl;
 663       int32_t range = 1 << (shift - 1);
 664       if ((-range <= sve_offset) && (sve_offset < range)) {
 665         // sve_offset can be encoded
 666         return true;
 667       }
 668     }
 669     return false;
 670   }
 671 };
 672 
 673 // Convenience classes
 674 class RuntimeAddress: public Address {
 675 
 676   public:
 677 
 678   RuntimeAddress(address target) : Address(target, relocInfo::runtime_call_type) {}
 679 
 680 };
 681 
 682 class OopAddress: public Address {
 683 
 684   public:
 685 
 686   OopAddress(address target) : Address(target, relocInfo::oop_type){}
 687 
 688 };
 689 
 690 class ExternalAddress: public Address {
 691  private:
 692   static relocInfo::relocType reloc_for_target(address target) {
 693     // Sometimes ExternalAddress is used for values which aren't
 694     // exactly addresses, like the card table base.
 695     // external_word_type can't be used for values in the first page
 696     // so just skip the reloc in that case.
 697     return external_word_Relocation::can_be_relocated(target) ? relocInfo::external_word_type : relocInfo::none;
 698   }
 699 
 700  public:
 701 
 702   ExternalAddress(address target) : Address(target, reloc_for_target(target)) {}
 703 
 704 };
 705 
 706 class InternalAddress: public Address {
 707 
 708   public:
 709 
 710   InternalAddress(address target) : Address(target, relocInfo::internal_word_type) {}
 711 };
 712 
 713 const int FPUStateSizeInWords = FloatRegister::number_of_registers * FloatRegister::save_slots_per_register;
 714 
 715 typedef enum {
 716   PLDL1KEEP = 0b00000, PLDL1STRM, PLDL2KEEP, PLDL2STRM, PLDL3KEEP, PLDL3STRM,
 717   PSTL1KEEP = 0b10000, PSTL1STRM, PSTL2KEEP, PSTL2STRM, PSTL3KEEP, PSTL3STRM,
 718   PLIL1KEEP = 0b01000, PLIL1STRM, PLIL2KEEP, PLIL2STRM, PLIL3KEEP, PLIL3STRM
 719 } prfop;
 720 
 721 class Assembler : public AbstractAssembler {
 722 
 723 public:
 724 
 725 #ifndef PRODUCT
 726   static const uintptr_t asm_bp;
 727 
 728   void emit_int32(jint x) {
 729     if ((uintptr_t)pc() == asm_bp)
 730       NOP();
 731     AbstractAssembler::emit_int32(x);
 732   }
 733 #else
 734   void emit_int32(jint x) {
 735     AbstractAssembler::emit_int32(x);
 736   }
 737 #endif
 738 
 739   enum { instruction_size = 4 };
 740 
 741   //---<  calculate length of instruction  >---
 742   // We just use the values set above.
 743   // instruction must start at passed address
 744   static unsigned int instr_len(unsigned char *instr) { return instruction_size; }
 745 
 746   //---<  longest instructions  >---
 747   static unsigned int instr_maxlen() { return instruction_size; }
 748 
 749   Address adjust(Register base, int offset, bool preIncrement) {
 750     if (preIncrement)
 751       return Address(Pre(base, offset));
 752     else
 753       return Address(Post(base, offset));
 754   }
 755 
 756   Address pre(Register base, int offset) {
 757     return adjust(base, offset, true);
 758   }
 759 
 760   Address post(Register base, int offset) {
 761     return adjust(base, offset, false);
 762   }
 763 
 764   Address post(Register base, Register idx) {
 765     return Address(Post(base, idx));
 766   }
 767 
 768   static address locate_next_instruction(address inst);
 769 
 770 #define f current_insn.f
 771 #define sf current_insn.sf
 772 #define rf current_insn.rf
 773 #define lrf current_insn.lrf
 774 #define srf current_insn.srf
 775 #define zrf current_insn.zrf
 776 #define prf current_insn.prf
 777 #define pgrf current_insn.pgrf
 778 
 779   typedef void (Assembler::* uncond_branch_insn)(address dest);
 780   typedef void (Assembler::* compare_and_branch_insn)(Register Rt, address dest);
 781   typedef void (Assembler::* test_and_branch_insn)(Register Rt, int bitpos, address dest);
 782   typedef void (Assembler::* prefetch_insn)(address target, prfop);
 783 
 784   void wrap_label(Label &L, uncond_branch_insn insn);
 785   void wrap_label(Register r, Label &L, compare_and_branch_insn insn);
 786   void wrap_label(Register r, int bitpos, Label &L, test_and_branch_insn insn);
 787   void wrap_label(Label &L, prfop, prefetch_insn insn);
 788 
 789   // PC-rel. addressing
 790 
 791   void adr(Register Rd, address dest);
 792   void _adrp(Register Rd, address dest);
 793 
 794   void adr(Register Rd, const Address &dest);
 795   void _adrp(Register Rd, const Address &dest);
 796 
 797   void adr(Register Rd, Label &L) {
 798     wrap_label(Rd, L, &Assembler::Assembler::adr);
 799   }
 800   void _adrp(Register Rd, Label &L) {
 801     wrap_label(Rd, L, &Assembler::_adrp);
 802   }
 803 
 804   void adrp(Register Rd, const Address &dest, uint64_t &offset) = delete;
 805 
 806   void prfm(const Address &adr, prfop pfop = PLDL1KEEP);
 807 
 808 #undef INSN
 809 
 810   void add_sub_immediate(Instruction_aarch64 &current_insn, Register Rd, Register Rn,
 811                          unsigned uimm, int op, int negated_op);
 812 
 813   // Add/subtract (immediate)
 814 #define INSN(NAME, decode, negated)                                     \
 815   void NAME(Register Rd, Register Rn, unsigned imm, unsigned shift) {   \
 816     starti;                                                             \
 817     f(decode, 31, 29), f(0b10001, 28, 24), f(shift, 23, 22), f(imm, 21, 10); \
 818     zrf(Rd, 0), srf(Rn, 5);                                             \
 819   }                                                                     \
 820                                                                         \
 821   void NAME(Register Rd, Register Rn, unsigned imm) {                   \
 822     starti;                                                             \
 823     add_sub_immediate(current_insn, Rd, Rn, imm, decode, negated);      \
 824   }
 825 
 826   INSN(addsw, 0b001, 0b011);
 827   INSN(subsw, 0b011, 0b001);
 828   INSN(adds,  0b101, 0b111);
 829   INSN(subs,  0b111, 0b101);
 830 
 831 #undef INSN
 832 
 833 #define INSN(NAME, decode, negated)                     \
 834   void NAME(Register Rd, Register Rn, unsigned imm) {   \
 835     starti;                                             \
 836     add_sub_immediate(current_insn, Rd, Rn, imm, decode, negated);     \
 837   }
 838 
 839   INSN(addw, 0b000, 0b010);
 840   INSN(subw, 0b010, 0b000);
 841   INSN(add,  0b100, 0b110);
 842   INSN(sub,  0b110, 0b100);
 843 
 844 #undef INSN
 845 
 846  // Logical (immediate)
 847 #define INSN(NAME, decode, is32)                                \
 848   void NAME(Register Rd, Register Rn, uint64_t imm) {           \
 849     starti;                                                     \
 850     uint32_t val = encode_logical_immediate(is32, imm);         \
 851     f(decode, 31, 29), f(0b100100, 28, 23), f(val, 22, 10);     \
 852     srf(Rd, 0), zrf(Rn, 5);                                     \
 853   }
 854 
 855   INSN(andw, 0b000, true);
 856   INSN(orrw, 0b001, true);
 857   INSN(eorw, 0b010, true);
 858   INSN(andr, 0b100, false);
 859   INSN(orr,  0b101, false);
 860   INSN(eor,  0b110, false);
 861 
 862 #undef INSN
 863 
 864 #define INSN(NAME, decode, is32)                                \
 865   void NAME(Register Rd, Register Rn, uint64_t imm) {           \
 866     starti;                                                     \
 867     uint32_t val = encode_logical_immediate(is32, imm);         \
 868     f(decode, 31, 29), f(0b100100, 28, 23), f(val, 22, 10);     \
 869     zrf(Rd, 0), zrf(Rn, 5);                                     \
 870   }
 871 
 872   INSN(ands, 0b111, false);
 873   INSN(andsw, 0b011, true);
 874 
 875 #undef INSN
 876 
 877   // Move wide (immediate)
 878 #define INSN(NAME, opcode)                                              \
 879   void NAME(Register Rd, unsigned imm, unsigned shift = 0) {            \
 880     assert_cond((shift/16)*16 == shift);                                \
 881     starti;                                                             \
 882     f(opcode, 31, 29), f(0b100101, 28, 23), f(shift/16, 22, 21),        \
 883       f(imm, 20, 5);                                                    \
 884     zrf(Rd, 0);                                                         \
 885   }
 886 
 887   INSN(movnw, 0b000);
 888   INSN(movzw, 0b010);
 889   INSN(movkw, 0b011);
 890   INSN(movn,  0b100);
 891   INSN(movz,  0b110);
 892   INSN(movk,  0b111);
 893 
 894 #undef INSN
 895 
 896   // Bitfield
 897 #define INSN(NAME, opcode, size)                                        \
 898   void NAME(Register Rd, Register Rn, unsigned immr, unsigned imms) {   \
 899     starti;                                                             \
 900     guarantee(size == 1 || (immr < 32 && imms < 32), "incorrect immr/imms");\
 901     f(opcode, 31, 22), f(immr, 21, 16), f(imms, 15, 10);                \
 902     zrf(Rn, 5), rf(Rd, 0);                                              \
 903   }
 904 
 905   INSN(sbfmw, 0b0001001100, 0);
 906   INSN(bfmw,  0b0011001100, 0);
 907   INSN(ubfmw, 0b0101001100, 0);
 908   INSN(sbfm,  0b1001001101, 1);
 909   INSN(bfm,   0b1011001101, 1);
 910   INSN(ubfm,  0b1101001101, 1);
 911 
 912 #undef INSN
 913 
 914   // Extract
 915 #define INSN(NAME, opcode, size)                                        \
 916   void NAME(Register Rd, Register Rn, Register Rm, unsigned imms) {     \
 917     starti;                                                             \
 918     guarantee(size == 1 || imms < 32, "incorrect imms");                \
 919     f(opcode, 31, 21), f(imms, 15, 10);                                 \
 920     zrf(Rm, 16), zrf(Rn, 5), zrf(Rd, 0);                                \
 921   }
 922 
 923   INSN(extrw, 0b00010011100, 0);
 924   INSN(extr,  0b10010011110, 1);
 925 
 926 #undef INSN
 927 
 928   // The maximum range of a branch is fixed for the AArch64
 929   // architecture.  In debug mode we shrink it in order to test
 930   // trampolines, but not so small that branches in the interpreter
 931   // are out of range.
 932   static const uint64_t branch_range = NOT_DEBUG(128 * M) DEBUG_ONLY(2 * M);
 933 
 934   static bool reachable_from_branch_at(address branch, address target) {
 935     return uabs(target - branch) < branch_range;
 936   }
 937 
 938   // Unconditional branch (immediate)
 939 #define INSN(NAME, opcode)                                              \
 940   void NAME(address dest) {                                             \
 941     starti;                                                             \
 942     int64_t offset = (dest - pc()) >> 2;                                \
 943     DEBUG_ONLY(assert(reachable_from_branch_at(pc(), dest), "debug only")); \
 944     f(opcode, 31), f(0b00101, 30, 26), sf(offset, 25, 0);               \
 945   }                                                                     \
 946   void NAME(Label &L) {                                                 \
 947     wrap_label(L, &Assembler::NAME);                                    \
 948   }                                                                     \
 949   void NAME(const Address &dest);
 950 
 951   INSN(b, 0);
 952   INSN(bl, 1);
 953 
 954 #undef INSN
 955 
 956   // Compare & branch (immediate)
 957 #define INSN(NAME, opcode)                              \
 958   void NAME(Register Rt, address dest) {                \
 959     int64_t offset = (dest - pc()) >> 2;                \
 960     starti;                                             \
 961     f(opcode, 31, 24), sf(offset, 23, 5), rf(Rt, 0);    \
 962   }                                                     \
 963   void NAME(Register Rt, Label &L) {                    \
 964     wrap_label(Rt, L, &Assembler::NAME);                \
 965   }
 966 
 967   INSN(cbzw,  0b00110100);
 968   INSN(cbnzw, 0b00110101);
 969   INSN(cbz,   0b10110100);
 970   INSN(cbnz,  0b10110101);
 971 
 972 #undef INSN
 973 
 974   // Test & branch (immediate)
 975 #define INSN(NAME, opcode)                                              \
 976   void NAME(Register Rt, int bitpos, address dest) {                    \
 977     int64_t offset = (dest - pc()) >> 2;                                \
 978     int b5 = bitpos >> 5;                                               \
 979     bitpos &= 0x1f;                                                     \
 980     starti;                                                             \
 981     f(b5, 31), f(opcode, 30, 24), f(bitpos, 23, 19), sf(offset, 18, 5); \
 982     rf(Rt, 0);                                                          \
 983   }                                                                     \
 984   void NAME(Register Rt, int bitpos, Label &L) {                        \
 985     wrap_label(Rt, bitpos, L, &Assembler::NAME);                        \
 986   }
 987 
 988   INSN(tbz,  0b0110110);
 989   INSN(tbnz, 0b0110111);
 990 
 991 #undef INSN
 992 
 993   // Conditional branch (immediate)
 994   enum Condition
 995     {EQ, NE, HS, CS=HS, LO, CC=LO, MI, PL, VS, VC, HI, LS, GE, LT, GT, LE, AL, NV};
 996 
 997   void br(Condition  cond, address dest) {
 998     int64_t offset = (dest - pc()) >> 2;
 999     starti;
1000     f(0b0101010, 31, 25), f(0, 24), sf(offset, 23, 5), f(0, 4), f(cond, 3, 0);
1001   }
1002 
1003 #define INSN(NAME, cond)                        \
1004   void NAME(address dest) {                     \
1005     br(cond, dest);                             \
1006   }
1007 
1008   INSN(beq, EQ);
1009   INSN(bne, NE);
1010   INSN(bhs, HS);
1011   INSN(bcs, CS);
1012   INSN(blo, LO);
1013   INSN(bcc, CC);
1014   INSN(bmi, MI);
1015   INSN(bpl, PL);
1016   INSN(bvs, VS);
1017   INSN(bvc, VC);
1018   INSN(bhi, HI);
1019   INSN(bls, LS);
1020   INSN(bge, GE);
1021   INSN(blt, LT);
1022   INSN(bgt, GT);
1023   INSN(ble, LE);
1024   INSN(bal, AL);
1025   INSN(bnv, NV);
1026 
1027   void br(Condition cc, Label &L);
1028 
1029 #undef INSN
1030 
1031   // Exception generation
1032   void generate_exception(int opc, int op2, int LL, unsigned imm) {
1033     starti;
1034     f(0b11010100, 31, 24);
1035     f(opc, 23, 21), f(imm, 20, 5), f(op2, 4, 2), f(LL, 1, 0);
1036   }
1037 
1038 #define INSN(NAME, opc, op2, LL)                \
1039   void NAME(unsigned imm) {                     \
1040     generate_exception(opc, op2, LL, imm);      \
1041   }
1042 
1043   INSN(svc, 0b000, 0, 0b01);
1044   INSN(hvc, 0b000, 0, 0b10);
1045   INSN(smc, 0b000, 0, 0b11);
1046   INSN(brk, 0b001, 0, 0b00);
1047   INSN(hlt, 0b010, 0, 0b00);
1048   INSN(dcps1, 0b101, 0, 0b01);
1049   INSN(dcps2, 0b101, 0, 0b10);
1050   INSN(dcps3, 0b101, 0, 0b11);
1051 
1052 #undef INSN
1053 
1054   // System
1055   void system(int op0, int op1, int CRn, int CRm, int op2,
1056               Register rt = dummy_reg)
1057   {
1058     starti;
1059     f(0b11010101000, 31, 21);
1060     f(op0, 20, 19);
1061     f(op1, 18, 16);
1062     f(CRn, 15, 12);
1063     f(CRm, 11, 8);
1064     f(op2, 7, 5);
1065     rf(rt, 0);
1066   }
1067 
1068   // Hint instructions
1069 
1070 #define INSN(NAME, crm, op2)               \
1071   void NAME() {                            \
1072     system(0b00, 0b011, 0b0010, crm, op2); \
1073   }
1074 
1075   INSN(nop,   0b000, 0b0000);
1076   INSN(yield, 0b000, 0b0001);
1077   INSN(wfe,   0b000, 0b0010);
1078   INSN(wfi,   0b000, 0b0011);
1079   INSN(sev,   0b000, 0b0100);
1080   INSN(sevl,  0b000, 0b0101);
1081 
1082   INSN(autia1716, 0b0001, 0b100);
1083   INSN(autiasp,   0b0011, 0b101);
1084   INSN(autiaz,    0b0011, 0b100);
1085   INSN(autib1716, 0b0001, 0b110);
1086   INSN(autibsp,   0b0011, 0b111);
1087   INSN(autibz,    0b0011, 0b110);
1088   INSN(pacia1716, 0b0001, 0b000);
1089   INSN(paciasp,   0b0011, 0b001);
1090   INSN(paciaz,    0b0011, 0b000);
1091   INSN(pacib1716, 0b0001, 0b010);
1092   INSN(pacibsp,   0b0011, 0b011);
1093   INSN(pacibz,    0b0011, 0b010);
1094   INSN(xpaclri,   0b0000, 0b111);
1095 
1096 #undef INSN
1097 
1098   // we only provide mrs and msr for the special purpose system
1099   // registers where op1 (instr[20:19]) == 11
1100   // n.b msr has L (instr[21]) == 0 mrs has L == 1
1101 
1102   void msr(int op1, int CRn, int CRm, int op2, Register rt) {
1103     starti;
1104     f(0b1101010100011, 31, 19);
1105     f(op1, 18, 16);
1106     f(CRn, 15, 12);
1107     f(CRm, 11, 8);
1108     f(op2, 7, 5);
1109     // writing zr is ok
1110     zrf(rt, 0);
1111   }
1112 
1113   void mrs(int op1, int CRn, int CRm, int op2, Register rt) {
1114     starti;
1115     f(0b1101010100111, 31, 19);
1116     f(op1, 18, 16);
1117     f(CRn, 15, 12);
1118     f(CRm, 11, 8);
1119     f(op2, 7, 5);
1120     // reading to zr is a mistake
1121     rf(rt, 0);
1122   }
1123 
1124   enum barrier {OSHLD = 0b0001, OSHST, OSH, NSHLD=0b0101, NSHST, NSH,
1125                 ISHLD = 0b1001, ISHST, ISH, LD=0b1101, ST, SY};
1126 
1127   void dsb(barrier imm) {
1128     system(0b00, 0b011, 0b00011, imm, 0b100);
1129   }
1130 
1131   void dmb(barrier imm) {
1132     system(0b00, 0b011, 0b00011, imm, 0b101);
1133   }
1134 
1135   void isb() {
1136     system(0b00, 0b011, 0b00011, SY, 0b110);
1137   }
1138 
1139   void sys(int op1, int CRn, int CRm, int op2,
1140            Register rt = as_Register(0b11111)) {
1141     system(0b01, op1, CRn, CRm, op2, rt);
1142   }
1143 
1144   // Only implement operations accessible from EL0 or higher, i.e.,
1145   //            op1    CRn    CRm    op2
1146   // IC IVAU     3      7      5      1
1147   // DC CVAC     3      7      10     1
1148   // DC CVAP     3      7      12     1
1149   // DC CVAU     3      7      11     1
1150   // DC CIVAC    3      7      14     1
1151   // DC ZVA      3      7      4      1
1152   // So only deal with the CRm field.
1153   enum icache_maintenance {IVAU = 0b0101};
1154   enum dcache_maintenance {CVAC = 0b1010, CVAP = 0b1100, CVAU = 0b1011, CIVAC = 0b1110, ZVA = 0b100};
1155 
1156   void dc(dcache_maintenance cm, Register Rt) {
1157     sys(0b011, 0b0111, cm, 0b001, Rt);
1158   }
1159 
1160   void ic(icache_maintenance cm, Register Rt) {
1161     sys(0b011, 0b0111, cm, 0b001, Rt);
1162   }
1163 
1164   // A more convenient access to dmb for our purposes
1165   enum Membar_mask_bits {
1166     // We can use ISH for a barrier because the Arm ARM says "This
1167     // architecture assumes that all Processing Elements that use the
1168     // same operating system or hypervisor are in the same Inner
1169     // Shareable shareability domain."
1170     StoreStore = ISHST,
1171     LoadStore  = ISHLD,
1172     LoadLoad   = ISHLD,
1173     StoreLoad  = ISH,
1174     AnyAny     = ISH
1175   };
1176 
1177   void membar(Membar_mask_bits order_constraint) {
1178     dmb(Assembler::barrier(order_constraint));
1179   }
1180 
1181   // Unconditional branch (register)
1182 
1183   void branch_reg(int OP, int A, int M, Register RN, Register RM) {
1184     starti;
1185     f(0b1101011, 31, 25);
1186     f(OP, 24, 21);
1187     f(0b111110000, 20, 12);
1188     f(A, 11, 11);
1189     f(M, 10, 10);
1190     rf(RN, 5);
1191     rf(RM, 0);
1192   }
1193 
1194 #define INSN(NAME, opc)                         \
1195   void NAME(Register RN) {                      \
1196     branch_reg(opc, 0, 0, RN, r0);              \
1197   }
1198 
1199   INSN(br,  0b0000);
1200   INSN(blr, 0b0001);
1201   INSN(ret, 0b0010);
1202 
1203   void ret(void *p); // This forces a compile-time error for ret(0)
1204 
1205 #undef INSN
1206 
1207 #define INSN(NAME, opc)                         \
1208   void NAME() {                                 \
1209     branch_reg(opc, 0, 0, dummy_reg, r0);       \
1210   }
1211 
1212   INSN(eret, 0b0100);
1213   INSN(drps, 0b0101);
1214 
1215 #undef INSN
1216 
1217 #define INSN(NAME, M)                                  \
1218   void NAME() {                                        \
1219     branch_reg(0b0010, 1, M, dummy_reg, dummy_reg);    \
1220   }
1221 
1222   INSN(retaa, 0);
1223   INSN(retab, 1);
1224 
1225 #undef INSN
1226 
1227 #define INSN(NAME, OP, M)                   \
1228   void NAME(Register rn) {                  \
1229     branch_reg(OP, 1, M, rn, dummy_reg);    \
1230   }
1231 
1232   INSN(braaz,  0b0000, 0);
1233   INSN(brabz,  0b0000, 1);
1234   INSN(blraaz, 0b0001, 0);
1235   INSN(blrabz, 0b0001, 1);
1236 
1237 #undef INSN
1238 
1239 #define INSN(NAME, OP, M)                  \
1240   void NAME(Register rn, Register rm) {    \
1241     branch_reg(OP, 1, M, rn, rm);          \
1242   }
1243 
1244   INSN(braa,  0b1000, 0);
1245   INSN(brab,  0b1000, 1);
1246   INSN(blraa, 0b1001, 0);
1247   INSN(blrab, 0b1001, 1);
1248 
1249 #undef INSN
1250 
1251   // Load/store exclusive
1252   enum operand_size { byte, halfword, word, xword };
1253 
1254   void load_store_exclusive(Register Rs, Register Rt1, Register Rt2,
1255     Register Rn, enum operand_size sz, int op, bool ordered) {
1256     starti;
1257     f(sz, 31, 30), f(0b001000, 29, 24), f(op, 23, 21);
1258     rf(Rs, 16), f(ordered, 15), zrf(Rt2, 10), srf(Rn, 5), zrf(Rt1, 0);
1259   }
1260 
1261   void load_exclusive(Register dst, Register addr,
1262                       enum operand_size sz, bool ordered) {
1263     load_store_exclusive(dummy_reg, dst, dummy_reg, addr,
1264                          sz, 0b010, ordered);
1265   }
1266 
1267   void store_exclusive(Register status, Register new_val, Register addr,
1268                        enum operand_size sz, bool ordered) {
1269     load_store_exclusive(status, new_val, dummy_reg, addr,
1270                          sz, 0b000, ordered);
1271   }
1272 
1273 #define INSN4(NAME, sz, op, o0) /* Four registers */                    \
1274   void NAME(Register Rs, Register Rt1, Register Rt2, Register Rn) {     \
1275     guarantee(Rs != Rn && Rs != Rt1 && Rs != Rt2, "unpredictable instruction"); \
1276     load_store_exclusive(Rs, Rt1, Rt2, Rn, sz, op, o0);                 \
1277   }
1278 
1279 #define INSN3(NAME, sz, op, o0) /* Three registers */                   \
1280   void NAME(Register Rs, Register Rt, Register Rn) {                    \
1281     guarantee(Rs != Rn && Rs != Rt, "unpredictable instruction");       \
1282     load_store_exclusive(Rs, Rt, dummy_reg, Rn, sz, op, o0); \
1283   }
1284 
1285 #define INSN2(NAME, sz, op, o0) /* Two registers */                     \
1286   void NAME(Register Rt, Register Rn) {                                 \
1287     load_store_exclusive(dummy_reg, Rt, dummy_reg, \
1288                          Rn, sz, op, o0);                               \
1289   }
1290 
1291 #define INSN_FOO(NAME, sz, op, o0) /* Three registers, encoded differently */ \
1292   void NAME(Register Rt1, Register Rt2, Register Rn) {                  \
1293     guarantee(Rt1 != Rt2, "unpredictable instruction");                 \
1294     load_store_exclusive(dummy_reg, Rt1, Rt2, Rn, sz, op, o0);          \
1295   }
1296 
1297   // bytes
1298   INSN3(stxrb,  byte, 0b000, 0);
1299   INSN3(stlxrb, byte, 0b000, 1);
1300   INSN2(ldxrb,  byte, 0b010, 0);
1301   INSN2(ldaxrb, byte, 0b010, 1);
1302   INSN2(stlrb,  byte, 0b100, 1);
1303   INSN2(ldarb,  byte, 0b110, 1);
1304 
1305   // halfwords
1306   INSN3(stxrh,  halfword, 0b000, 0);
1307   INSN3(stlxrh, halfword, 0b000, 1);
1308   INSN2(ldxrh,  halfword, 0b010, 0);
1309   INSN2(ldaxrh, halfword, 0b010, 1);
1310   INSN2(stlrh,  halfword, 0b100, 1);
1311   INSN2(ldarh,  halfword, 0b110, 1);
1312 
1313   // words
1314   INSN3(stxrw,  word, 0b000, 0);
1315   INSN3(stlxrw, word, 0b000, 1);
1316   INSN4(stxpw,  word, 0b001, 0);
1317   INSN4(stlxpw, word, 0b001, 1);
1318   INSN2(ldxrw,  word, 0b010, 0);
1319   INSN2(ldaxrw, word, 0b010, 1);
1320   INSN2(stlrw,  word, 0b100, 1);
1321   INSN2(ldarw,  word, 0b110, 1);
1322   // pairs of words
1323   INSN_FOO(ldxpw,  word, 0b011, 0);
1324   INSN_FOO(ldaxpw, word, 0b011, 1);
1325 
1326   // xwords
1327   INSN3(stxr,  xword, 0b000, 0);
1328   INSN3(stlxr, xword, 0b000, 1);
1329   INSN4(stxp,  xword, 0b001, 0);
1330   INSN4(stlxp, xword, 0b001, 1);
1331   INSN2(ldxr,  xword, 0b010, 0);
1332   INSN2(ldaxr, xword, 0b010, 1);
1333   INSN2(stlr,  xword, 0b100, 1);
1334   INSN2(ldar,  xword, 0b110, 1);
1335   // pairs of xwords
1336   INSN_FOO(ldxp,  xword, 0b011, 0);
1337   INSN_FOO(ldaxp, xword, 0b011, 1);
1338 
1339 #undef INSN2
1340 #undef INSN3
1341 #undef INSN4
1342 #undef INSN_FOO
1343 
1344   // 8.1 Compare and swap extensions
1345   void lse_cas(Register Rs, Register Rt, Register Rn,
1346                         enum operand_size sz, bool a, bool r, bool not_pair) {
1347     starti;
1348     if (! not_pair) { // Pair
1349       assert(sz == word || sz == xword, "invalid size");
1350       /* The size bit is in bit 30, not 31 */
1351       sz = (operand_size)(sz == word ? 0b00:0b01);
1352     }
1353     f(sz, 31, 30), f(0b001000, 29, 24), f(not_pair ? 1 : 0, 23), f(a, 22), f(1, 21);
1354     zrf(Rs, 16), f(r, 15), f(0b11111, 14, 10), srf(Rn, 5), zrf(Rt, 0);
1355   }
1356 
1357   // CAS
1358 #define INSN(NAME, a, r)                                                \
1359   void NAME(operand_size sz, Register Rs, Register Rt, Register Rn) {   \
1360     assert(Rs != Rn && Rs != Rt, "unpredictable instruction");          \
1361     lse_cas(Rs, Rt, Rn, sz, a, r, true);                                \
1362   }
1363   INSN(cas,   false, false)
1364   INSN(casa,  true,  false)
1365   INSN(casl,  false, true)
1366   INSN(casal, true,  true)
1367 #undef INSN
1368 
1369   // CASP
1370 #define INSN(NAME, a, r)                                                \
1371   void NAME(operand_size sz, Register Rs, Register Rs1,                 \
1372             Register Rt, Register Rt1, Register Rn) {                   \
1373     assert((Rs->encoding() & 1) == 0 && (Rt->encoding() & 1) == 0 &&    \
1374            Rs->successor() == Rs1 && Rt->successor() == Rt1 &&          \
1375            Rs != Rn && Rs1 != Rn && Rs != Rt, "invalid registers");     \
1376     lse_cas(Rs, Rt, Rn, sz, a, r, false);                               \
1377   }
1378   INSN(casp,   false, false)
1379   INSN(caspa,  true,  false)
1380   INSN(caspl,  false, true)
1381   INSN(caspal, true,  true)
1382 #undef INSN
1383 
1384   // 8.1 Atomic operations
1385   void lse_atomic(Register Rs, Register Rt, Register Rn,
1386                   enum operand_size sz, int op1, int op2, bool a, bool r) {
1387     starti;
1388     f(sz, 31, 30), f(0b111000, 29, 24), f(a, 23), f(r, 22), f(1, 21);
1389     zrf(Rs, 16), f(op1, 15), f(op2, 14, 12), f(0, 11, 10), srf(Rn, 5), zrf(Rt, 0);
1390   }
1391 
1392 #define INSN(NAME, NAME_A, NAME_L, NAME_AL, op1, op2)                   \
1393   void NAME(operand_size sz, Register Rs, Register Rt, Register Rn) {   \
1394     lse_atomic(Rs, Rt, Rn, sz, op1, op2, false, false);                 \
1395   }                                                                     \
1396   void NAME_A(operand_size sz, Register Rs, Register Rt, Register Rn) { \
1397     lse_atomic(Rs, Rt, Rn, sz, op1, op2, true, false);                  \
1398   }                                                                     \
1399   void NAME_L(operand_size sz, Register Rs, Register Rt, Register Rn) { \
1400     lse_atomic(Rs, Rt, Rn, sz, op1, op2, false, true);                  \
1401   }                                                                     \
1402   void NAME_AL(operand_size sz, Register Rs, Register Rt, Register Rn) {\
1403     lse_atomic(Rs, Rt, Rn, sz, op1, op2, true, true);                   \
1404   }
1405   INSN(ldadd,  ldadda,  ldaddl,  ldaddal,  0, 0b000);
1406   INSN(ldbic,  ldbica,  ldbicl,  ldbical,  0, 0b001);
1407   INSN(ldeor,  ldeora,  ldeorl,  ldeoral,  0, 0b010);
1408   INSN(ldorr,  ldorra,  ldorrl,  ldorral,  0, 0b011);
1409   INSN(ldsmax, ldsmaxa, ldsmaxl, ldsmaxal, 0, 0b100);
1410   INSN(ldsmin, ldsmina, ldsminl, ldsminal, 0, 0b101);
1411   INSN(ldumax, ldumaxa, ldumaxl, ldumaxal, 0, 0b110);
1412   INSN(ldumin, ldumina, lduminl, lduminal, 0, 0b111);
1413   INSN(swp,    swpa,    swpl,    swpal,    1, 0b000);
1414 #undef INSN
1415 
1416   // Load register (literal)
1417 #define INSN(NAME, opc, V)                                              \
1418   void NAME(Register Rt, address dest) {                                \
1419     int64_t offset = (dest - pc()) >> 2;                                \
1420     starti;                                                             \
1421     f(opc, 31, 30), f(0b011, 29, 27), f(V, 26), f(0b00, 25, 24),        \
1422       sf(offset, 23, 5);                                                \
1423     rf(Rt, 0);                                                          \
1424   }                                                                     \
1425   void NAME(Register Rt, address dest, relocInfo::relocType rtype) {    \
1426     InstructionMark im(this);                                           \
1427     guarantee(rtype == relocInfo::internal_word_type,                   \
1428               "only internal_word_type relocs make sense here");        \
1429     code_section()->relocate(inst_mark(), InternalAddress(dest).rspec()); \
1430     NAME(Rt, dest);                                                     \
1431   }                                                                     \
1432   void NAME(Register Rt, Label &L) {                                    \
1433     wrap_label(Rt, L, &Assembler::NAME);                                \
1434   }
1435 
1436   INSN(ldrw, 0b00, 0);
1437   INSN(ldr, 0b01, 0);
1438   INSN(ldrsw, 0b10, 0);
1439 
1440 #undef INSN
1441 
1442 #define INSN(NAME, opc, V)                                              \
1443   void NAME(FloatRegister Rt, address dest) {                           \
1444     int64_t offset = (dest - pc()) >> 2;                                \
1445     starti;                                                             \
1446     f(opc, 31, 30), f(0b011, 29, 27), f(V, 26), f(0b00, 25, 24),        \
1447       sf(offset, 23, 5);                                                \
1448     rf(as_Register(Rt), 0);                                             \
1449   }
1450 
1451   INSN(ldrs, 0b00, 1);
1452   INSN(ldrd, 0b01, 1);
1453   INSN(ldrq, 0b10, 1);
1454 
1455 #undef INSN
1456 
1457 #define INSN(NAME, size, opc)                                           \
1458   void NAME(FloatRegister Rt, Register Rn) {                            \
1459     starti;                                                             \
1460     f(size, 31, 30), f(0b111100, 29, 24), f(opc, 23, 22), f(0, 21);     \
1461     f(0, 20, 12), f(0b01, 11, 10);                                      \
1462     rf(Rn, 5), rf(as_Register(Rt), 0);                                  \
1463   }
1464 
1465   INSN(ldrs, 0b10, 0b01);
1466   INSN(ldrd, 0b11, 0b01);
1467   INSN(ldrq, 0b00, 0b11);
1468 
1469 #undef INSN
1470 
1471 
1472 #define INSN(NAME, opc, V)                                              \
1473   void NAME(address dest, prfop op = PLDL1KEEP) {                       \
1474     int64_t offset = (dest - pc()) >> 2;                                \
1475     starti;                                                             \
1476     f(opc, 31, 30), f(0b011, 29, 27), f(V, 26), f(0b00, 25, 24),        \
1477       sf(offset, 23, 5);                                                \
1478     f(op, 4, 0);                                                        \
1479   }                                                                     \
1480   void NAME(Label &L, prfop op = PLDL1KEEP) {                           \
1481     wrap_label(L, op, &Assembler::NAME);                                \
1482   }
1483 
1484   INSN(prfm, 0b11, 0);
1485 
1486 #undef INSN
1487 
1488   // Load/store
1489   void ld_st1(int opc, int p1, int V, int L,
1490               Register Rt1, Register Rt2, Address adr, bool no_allocate) {
1491     starti;
1492     f(opc, 31, 30), f(p1, 29, 27), f(V, 26), f(L, 22);
1493     zrf(Rt2, 10), zrf(Rt1, 0);
1494     if (no_allocate) {
1495       adr.encode_nontemporal_pair(&current_insn);
1496     } else {
1497       adr.encode_pair(&current_insn);
1498     }
1499   }
1500 
1501   // Load/store register pair (offset)
1502 #define INSN(NAME, size, p1, V, L, no_allocate)         \
1503   void NAME(Register Rt1, Register Rt2, Address adr) {  \
1504     ld_st1(size, p1, V, L, Rt1, Rt2, adr, no_allocate); \
1505    }
1506 
1507   INSN(stpw,  0b00, 0b101, 0, 0, false);
1508   INSN(ldpw,  0b00, 0b101, 0, 1, false);
1509   INSN(ldpsw, 0b01, 0b101, 0, 1, false);
1510   INSN(stp,   0b10, 0b101, 0, 0, false);
1511   INSN(ldp,   0b10, 0b101, 0, 1, false);
1512 
1513   // Load/store no-allocate pair (offset)
1514   INSN(stnpw, 0b00, 0b101, 0, 0, true);
1515   INSN(ldnpw, 0b00, 0b101, 0, 1, true);
1516   INSN(stnp,  0b10, 0b101, 0, 0, true);
1517   INSN(ldnp,  0b10, 0b101, 0, 1, true);
1518 
1519 #undef INSN
1520 
1521 #define INSN(NAME, size, p1, V, L, no_allocate)                         \
1522   void NAME(FloatRegister Rt1, FloatRegister Rt2, Address adr) {        \
1523     ld_st1(size, p1, V, L,                                              \
1524            as_Register(Rt1), as_Register(Rt2), adr, no_allocate);       \
1525    }
1526 
1527   INSN(stps, 0b00, 0b101, 1, 0, false);
1528   INSN(ldps, 0b00, 0b101, 1, 1, false);
1529   INSN(stpd, 0b01, 0b101, 1, 0, false);
1530   INSN(ldpd, 0b01, 0b101, 1, 1, false);
1531   INSN(stpq, 0b10, 0b101, 1, 0, false);
1532   INSN(ldpq, 0b10, 0b101, 1, 1, false);
1533 
1534 #undef INSN
1535 
1536   // Load/store register (all modes)
1537   void ld_st2(Register Rt, const Address &adr, int size, int op, int V = 0) {
1538     starti;
1539 
1540     f(V, 26); // general reg?
1541     zrf(Rt, 0);
1542 
1543     // Encoding for literal loads is done here (rather than pushed
1544     // down into Address::encode) because the encoding of this
1545     // instruction is too different from all of the other forms to
1546     // make it worth sharing.
1547     if (adr.getMode() == Address::literal) {
1548       assert(size == 0b10 || size == 0b11, "bad operand size in ldr");
1549       assert(op == 0b01, "literal form can only be used with loads");
1550       f(size & 0b01, 31, 30), f(0b011, 29, 27), f(0b00, 25, 24);
1551       int64_t offset = (adr.target() - pc()) >> 2;
1552       sf(offset, 23, 5);
1553       code_section()->relocate(pc(), adr.rspec());
1554       return;
1555     }
1556 
1557     f(size, 31, 30);
1558     f(op, 23, 22); // str
1559     adr.encode(&current_insn);
1560   }
1561 
1562 #define INSN(NAME, size, op)                            \
1563   void NAME(Register Rt, const Address &adr) {          \
1564     ld_st2(Rt, adr, size, op);                          \
1565   }                                                     \
1566 
1567   INSN(str,  0b11, 0b00);
1568   INSN(strw, 0b10, 0b00);
1569   INSN(strb, 0b00, 0b00);
1570   INSN(strh, 0b01, 0b00);
1571 
1572   INSN(ldr,  0b11, 0b01);
1573   INSN(ldrw, 0b10, 0b01);
1574   INSN(ldrb, 0b00, 0b01);
1575   INSN(ldrh, 0b01, 0b01);
1576 
1577   INSN(ldrsb,  0b00, 0b10);
1578   INSN(ldrsbw, 0b00, 0b11);
1579   INSN(ldrsh,  0b01, 0b10);
1580   INSN(ldrshw, 0b01, 0b11);
1581   INSN(ldrsw,  0b10, 0b10);
1582 
1583 #undef INSN
1584 
1585 #define INSN(NAME, size, op)                            \
1586   void NAME(FloatRegister Rt, const Address &adr) {     \
1587     ld_st2(as_Register(Rt), adr, size, op, 1);          \
1588   }
1589 
1590   INSN(strd, 0b11, 0b00);
1591   INSN(strs, 0b10, 0b00);
1592   INSN(ldrd, 0b11, 0b01);
1593   INSN(ldrs, 0b10, 0b01);
1594   INSN(strq, 0b00, 0b10);
1595   INSN(ldrq, 0x00, 0b11);
1596 
1597 #undef INSN
1598 
1599   // Load/store a register, but with a BasicType parameter. Loaded signed integer values are
1600   // extended to 64 bits.
1601   void load(Register Rt, const Address &adr, BasicType bt) {
1602     int op = (is_signed_subword_type(bt) || bt == T_INT) ? 0b10 : 0b01;
1603     ld_st2(Rt, adr, exact_log2(type2aelembytes(bt)), op);
1604   }
1605   void store(Register Rt, const Address &adr, BasicType bt) {
1606     ld_st2(Rt, adr, exact_log2(type2aelembytes(bt)), 0b00);
1607   }
1608 
1609 /* SIMD extensions
1610  *
1611  * We just use FloatRegister in the following. They are exactly the same
1612  * as SIMD registers.
1613  */
1614 public:
1615 
1616   enum SIMD_Arrangement {
1617     T8B, T16B, T4H, T8H, T2S, T4S, T1D, T2D, T1Q, INVALID_ARRANGEMENT
1618   };
1619 
1620   enum SIMD_RegVariant {
1621       B, H, S, D, Q, INVALID
1622   };
1623 
1624 private:
1625 
1626   static SIMD_Arrangement _esize2arrangement_table[9][2];
1627   static SIMD_RegVariant _esize2regvariant[9];
1628 
1629 public:
1630 
1631   static SIMD_Arrangement esize2arrangement(unsigned esize, bool isQ);
1632   static SIMD_RegVariant elemType_to_regVariant(BasicType bt);
1633   static SIMD_RegVariant elemBytes_to_regVariant(unsigned esize);
1634   // Return the corresponding bits for different SIMD_RegVariant value.
1635   static unsigned regVariant_to_elemBits(SIMD_RegVariant T);
1636 
1637   enum shift_kind { LSL, LSR, ASR, ROR };
1638 
1639   void op_shifted_reg(Instruction_aarch64 &current_insn, unsigned decode,
1640                       enum shift_kind kind, unsigned shift,
1641                       unsigned size, unsigned op) {
1642     f(size, 31);
1643     f(op, 30, 29);
1644     f(decode, 28, 24);
1645     f(shift, 15, 10);
1646     f(kind, 23, 22);
1647   }
1648 
1649   // Logical (shifted register)
1650 #define INSN(NAME, size, op, N)                                         \
1651   void NAME(Register Rd, Register Rn, Register Rm,                      \
1652             enum shift_kind kind = LSL, unsigned shift = 0) {           \
1653     starti;                                                             \
1654     guarantee(size == 1 || shift < 32, "incorrect shift");              \
1655     f(N, 21);                                                           \
1656     zrf(Rm, 16), zrf(Rn, 5), zrf(Rd, 0);                                \
1657     op_shifted_reg(current_insn, 0b01010, kind, shift, size, op);       \
1658   }
1659 
1660   INSN(andr,  1, 0b00, 0);
1661   INSN(orr,   1, 0b01, 0);
1662   INSN(eor,   1, 0b10, 0);
1663   INSN(ands,  1, 0b11, 0);
1664   INSN(andw,  0, 0b00, 0);
1665   INSN(orrw,  0, 0b01, 0);
1666   INSN(eorw,  0, 0b10, 0);
1667   INSN(andsw, 0, 0b11, 0);
1668 
1669 #undef INSN
1670 
1671 #define INSN(NAME, size, op, N)                                         \
1672   void NAME(Register Rd, Register Rn, Register Rm,                      \
1673             enum shift_kind kind = LSL, unsigned shift = 0) {           \
1674     starti;                                                             \
1675     f(N, 21);                                                           \
1676     zrf(Rm, 16), zrf(Rn, 5), zrf(Rd, 0);                                \
1677     op_shifted_reg(current_insn, 0b01010, kind, shift, size, op);       \
1678   }                                                                     \
1679                                                                         \
1680   /* These instructions have no immediate form. Provide an overload so  \
1681      that if anyone does try to use an immediate operand -- this has    \
1682      happened! -- we'll get a compile-time error. */                    \
1683   void NAME(Register Rd, Register Rn, unsigned imm,                     \
1684             enum shift_kind kind = LSL, unsigned shift = 0) {           \
1685     assert(false, " can't be used with immediate operand");             \
1686   }
1687 
1688   INSN(bic,   1, 0b00, 1);
1689   INSN(orn,   1, 0b01, 1);
1690   INSN(eon,   1, 0b10, 1);
1691   INSN(bics,  1, 0b11, 1);
1692   INSN(bicw,  0, 0b00, 1);
1693   INSN(ornw,  0, 0b01, 1);
1694   INSN(eonw,  0, 0b10, 1);
1695   INSN(bicsw, 0, 0b11, 1);
1696 
1697 #undef INSN
1698 
1699 #ifdef _WIN64
1700 // In MSVC, `mvn` is defined as a macro and it affects compilation
1701 #undef mvn
1702 #endif
1703 
1704   // Aliases for short forms of orn
1705 void mvn(Register Rd, Register Rm,
1706             enum shift_kind kind = LSL, unsigned shift = 0) {
1707   orn(Rd, zr, Rm, kind, shift);
1708 }
1709 
1710 void mvnw(Register Rd, Register Rm,
1711             enum shift_kind kind = LSL, unsigned shift = 0) {
1712   ornw(Rd, zr, Rm, kind, shift);
1713 }
1714 
1715   // Add/subtract (shifted register)
1716 #define INSN(NAME, size, op)                            \
1717   void NAME(Register Rd, Register Rn, Register Rm,      \
1718             enum shift_kind kind, unsigned shift = 0) { \
1719     starti;                                             \
1720     f(0, 21);                                           \
1721     assert_cond(kind != ROR);                           \
1722     guarantee(size == 1 || shift < 32, "incorrect shift");\
1723     zrf(Rd, 0), zrf(Rn, 5), zrf(Rm, 16);                \
1724     op_shifted_reg(current_insn, 0b01011, kind, shift, size, op);      \
1725   }
1726 
1727   INSN(add,  1, 0b000);
1728   INSN(sub,  1, 0b10);
1729   INSN(addw, 0, 0b000);
1730   INSN(subw, 0, 0b10);
1731 
1732   INSN(adds,  1, 0b001);
1733   INSN(subs,  1, 0b11);
1734   INSN(addsw, 0, 0b001);
1735   INSN(subsw, 0, 0b11);
1736 
1737 #undef INSN
1738 
1739   // Add/subtract (extended register)
1740 #define INSN(NAME, op)                                                  \
1741   void NAME(Register Rd, Register Rn, Register Rm,                      \
1742            ext::operation option, int amount = 0) {                     \
1743     starti;                                                             \
1744     zrf(Rm, 16), srf(Rn, 5), srf(Rd, 0);                                \
1745     add_sub_extended_reg(current_insn, op, 0b01011, Rd, Rn, Rm, 0b00, option, amount); \
1746   }
1747 
1748   void add_sub_extended_reg(Instruction_aarch64 &current_insn, unsigned op, unsigned decode,
1749     Register Rd, Register Rn, Register Rm,
1750     unsigned opt, ext::operation option, unsigned imm) {
1751     guarantee(imm <= 4, "shift amount must be <= 4");
1752     f(op, 31, 29), f(decode, 28, 24), f(opt, 23, 22), f(1, 21);
1753     f(option, 15, 13), f(imm, 12, 10);
1754   }
1755 
1756   INSN(addw, 0b000);
1757   INSN(subw, 0b010);
1758   INSN(add,  0b100);
1759   INSN(sub,  0b110);
1760 
1761 #undef INSN
1762 
1763 #define INSN(NAME, op)                                                  \
1764   void NAME(Register Rd, Register Rn, Register Rm,                      \
1765            ext::operation option, int amount = 0) {                     \
1766     starti;                                                             \
1767     zrf(Rm, 16), srf(Rn, 5), zrf(Rd, 0);                                \
1768     add_sub_extended_reg(current_insn, op, 0b01011, Rd, Rn, Rm, 0b00, option, amount); \
1769   }
1770 
1771   INSN(addsw, 0b001);
1772   INSN(subsw, 0b011);
1773   INSN(adds,  0b101);
1774   INSN(subs,  0b111);
1775 
1776 #undef INSN
1777 
1778   // Aliases for short forms of add and sub
1779 #define INSN(NAME)                                      \
1780   void NAME(Register Rd, Register Rn, Register Rm) {    \
1781     if (Rd == sp || Rn == sp)                           \
1782       NAME(Rd, Rn, Rm, ext::uxtx);                      \
1783     else                                                \
1784       NAME(Rd, Rn, Rm, LSL);                            \
1785   }
1786 
1787   INSN(addw);
1788   INSN(subw);
1789   INSN(add);
1790   INSN(sub);
1791 
1792   INSN(addsw);
1793   INSN(subsw);
1794   INSN(adds);
1795   INSN(subs);
1796 
1797 #undef INSN
1798 
1799   // Add/subtract (with carry)
1800   void add_sub_carry(unsigned op, Register Rd, Register Rn, Register Rm) {
1801     starti;
1802     f(op, 31, 29);
1803     f(0b11010000, 28, 21);
1804     f(0b000000, 15, 10);
1805     zrf(Rm, 16), zrf(Rn, 5), zrf(Rd, 0);
1806   }
1807 
1808   #define INSN(NAME, op)                                \
1809     void NAME(Register Rd, Register Rn, Register Rm) {  \
1810       add_sub_carry(op, Rd, Rn, Rm);                    \
1811     }
1812 
1813   INSN(adcw,  0b000);
1814   INSN(adcsw, 0b001);
1815   INSN(sbcw,  0b010);
1816   INSN(sbcsw, 0b011);
1817   INSN(adc,   0b100);
1818   INSN(adcs,  0b101);
1819   INSN(sbc,   0b110);
1820   INSN(sbcs,  0b111);
1821 
1822 #undef INSN
1823 
1824   // Conditional compare (both kinds)
1825   void conditional_compare(unsigned op, int o1, int o2, int o3,
1826                            Register Rn, unsigned imm5, unsigned nzcv,
1827                            unsigned cond) {
1828     starti;
1829     f(op, 31, 29);
1830     f(0b11010010, 28, 21);
1831     f(cond, 15, 12);
1832     f(o1, 11);
1833     f(o2, 10);
1834     f(o3, 4);
1835     f(nzcv, 3, 0);
1836     f(imm5, 20, 16), zrf(Rn, 5);
1837   }
1838 
1839 #define INSN(NAME, op)                                                  \
1840   void NAME(Register Rn, Register Rm, int imm, Condition cond) {        \
1841     int regNumber = (Rm == zr ? 31 : Rm->encoding());                   \
1842     conditional_compare(op, 0, 0, 0, Rn, regNumber, imm, cond);         \
1843   }                                                                     \
1844                                                                         \
1845   void NAME(Register Rn, int imm5, int imm, Condition cond) {           \
1846     conditional_compare(op, 1, 0, 0, Rn, imm5, imm, cond);              \
1847   }
1848 
1849   INSN(ccmnw, 0b001);
1850   INSN(ccmpw, 0b011);
1851   INSN(ccmn, 0b101);
1852   INSN(ccmp, 0b111);
1853 
1854 #undef INSN
1855 
1856   // Conditional select
1857   void conditional_select(unsigned op, unsigned op2,
1858                           Register Rd, Register Rn, Register Rm,
1859                           unsigned cond) {
1860     starti;
1861     f(op, 31, 29);
1862     f(0b11010100, 28, 21);
1863     f(cond, 15, 12);
1864     f(op2, 11, 10);
1865     zrf(Rm, 16), zrf(Rn, 5), rf(Rd, 0);
1866   }
1867 
1868 #define INSN(NAME, op, op2)                                             \
1869   void NAME(Register Rd, Register Rn, Register Rm, Condition cond) {    \
1870     conditional_select(op, op2, Rd, Rn, Rm, cond);                      \
1871   }
1872 
1873   INSN(cselw,  0b000, 0b00);
1874   INSN(csincw, 0b000, 0b01);
1875   INSN(csinvw, 0b010, 0b00);
1876   INSN(csnegw, 0b010, 0b01);
1877   INSN(csel,   0b100, 0b00);
1878   INSN(csinc,  0b100, 0b01);
1879   INSN(csinv,  0b110, 0b00);
1880   INSN(csneg,  0b110, 0b01);
1881 
1882 #undef INSN
1883 
1884   // Data processing
1885   void data_processing(Instruction_aarch64 &current_insn, unsigned op29, unsigned opcode,
1886                        Register Rd, Register Rn) {
1887     f(op29, 31, 29), f(0b11010110, 28, 21);
1888     f(opcode, 15, 10);
1889     rf(Rn, 5), rf(Rd, 0);
1890   }
1891 
1892   // (1 source)
1893 #define INSN(NAME, op29, opcode2, opcode)                       \
1894   void NAME(Register Rd, Register Rn) {                         \
1895     starti;                                                     \
1896     f(opcode2, 20, 16);                                         \
1897     data_processing(current_insn, op29, opcode, Rd, Rn);        \
1898   }
1899 
1900   INSN(rbitw,  0b010, 0b00000, 0b00000);
1901   INSN(rev16w, 0b010, 0b00000, 0b00001);
1902   INSN(revw,   0b010, 0b00000, 0b00010);
1903   INSN(clzw,   0b010, 0b00000, 0b00100);
1904   INSN(clsw,   0b010, 0b00000, 0b00101);
1905 
1906   INSN(rbit,   0b110, 0b00000, 0b00000);
1907   INSN(rev16,  0b110, 0b00000, 0b00001);
1908   INSN(rev32,  0b110, 0b00000, 0b00010);
1909   INSN(rev,    0b110, 0b00000, 0b00011);
1910   INSN(clz,    0b110, 0b00000, 0b00100);
1911   INSN(cls,    0b110, 0b00000, 0b00101);
1912 
1913   // PAC instructions
1914   INSN(pacia,  0b110, 0b00001, 0b00000);
1915   INSN(pacib,  0b110, 0b00001, 0b00001);
1916   INSN(pacda,  0b110, 0b00001, 0b00010);
1917   INSN(pacdb,  0b110, 0b00001, 0b00011);
1918   INSN(autia,  0b110, 0b00001, 0b00100);
1919   INSN(autib,  0b110, 0b00001, 0b00101);
1920   INSN(autda,  0b110, 0b00001, 0b00110);
1921   INSN(autdb,  0b110, 0b00001, 0b00111);
1922 
1923 #undef INSN
1924 
1925 #define INSN(NAME, op29, opcode2, opcode)                       \
1926   void NAME(Register Rd) {                                      \
1927     starti;                                                     \
1928     f(opcode2, 20, 16);                                         \
1929     data_processing(current_insn, op29, opcode, Rd, dummy_reg); \
1930   }
1931 
1932   // PAC instructions (with zero modifier)
1933   INSN(paciza,  0b110, 0b00001, 0b01000);
1934   INSN(pacizb,  0b110, 0b00001, 0b01001);
1935   INSN(pacdza,  0b110, 0b00001, 0b01010);
1936   INSN(pacdzb,  0b110, 0b00001, 0b01011);
1937   INSN(autiza,  0b110, 0b00001, 0b01100);
1938   INSN(autizb,  0b110, 0b00001, 0b01101);
1939   INSN(autdza,  0b110, 0b00001, 0b01110);
1940   INSN(autdzb,  0b110, 0b00001, 0b01111);
1941   INSN(xpaci,   0b110, 0b00001, 0b10000);
1942   INSN(xpacd,   0b110, 0b00001, 0b10001);
1943 
1944 #undef INSN
1945 
1946   // Data-processing (2 source)
1947 #define INSN(NAME, op29, opcode)                                \
1948   void NAME(Register Rd, Register Rn, Register Rm) {            \
1949     starti;                                                     \
1950     rf(Rm, 16);                                                 \
1951     data_processing(current_insn, op29, opcode, Rd, Rn);        \
1952   }
1953 
1954   INSN(udivw, 0b000, 0b000010);
1955   INSN(sdivw, 0b000, 0b000011);
1956   INSN(lslvw, 0b000, 0b001000);
1957   INSN(lsrvw, 0b000, 0b001001);
1958   INSN(asrvw, 0b000, 0b001010);
1959   INSN(rorvw, 0b000, 0b001011);
1960 
1961   INSN(udiv, 0b100, 0b000010);
1962   INSN(sdiv, 0b100, 0b000011);
1963   INSN(lslv, 0b100, 0b001000);
1964   INSN(lsrv, 0b100, 0b001001);
1965   INSN(asrv, 0b100, 0b001010);
1966   INSN(rorv, 0b100, 0b001011);
1967 
1968 #undef INSN
1969 
1970   // Data-processing (3 source)
1971   void data_processing(unsigned op54, unsigned op31, unsigned o0,
1972                        Register Rd, Register Rn, Register Rm,
1973                        Register Ra) {
1974     starti;
1975     f(op54, 31, 29), f(0b11011, 28, 24);
1976     f(op31, 23, 21), f(o0, 15);
1977     zrf(Rm, 16), zrf(Ra, 10), zrf(Rn, 5), zrf(Rd, 0);
1978   }
1979 
1980 #define INSN(NAME, op54, op31, o0)                                      \
1981   void NAME(Register Rd, Register Rn, Register Rm, Register Ra) {       \
1982     data_processing(op54, op31, o0, Rd, Rn, Rm, Ra);                    \
1983   }
1984 
1985   INSN(maddw,  0b000, 0b000, 0);
1986   INSN(msubw,  0b000, 0b000, 1);
1987   INSN(madd,   0b100, 0b000, 0);
1988   INSN(msub,   0b100, 0b000, 1);
1989   INSN(smaddl, 0b100, 0b001, 0);
1990   INSN(smsubl, 0b100, 0b001, 1);
1991   INSN(umaddl, 0b100, 0b101, 0);
1992   INSN(umsubl, 0b100, 0b101, 1);
1993 
1994 #undef INSN
1995 
1996 #define INSN(NAME, op54, op31, o0)                                      \
1997   void NAME(Register Rd, Register Rn, Register Rm) {                    \
1998     data_processing(op54, op31, o0, Rd, Rn, Rm, as_Register(31));       \
1999   }
2000 
2001   INSN(smulh, 0b100, 0b010, 0);
2002   INSN(umulh, 0b100, 0b110, 0);
2003 
2004 #undef INSN
2005 
2006   // Floating-point data-processing (1 source)
2007   void data_processing(unsigned type, unsigned opcode,
2008                        FloatRegister Vd, FloatRegister Vn) {
2009     starti;
2010     f(0b000, 31, 29);
2011     f(0b11110, 28, 24);
2012     f(type, 23, 22), f(1, 21), f(opcode, 20, 15), f(0b10000, 14, 10);
2013     rf(Vn, 5), rf(Vd, 0);
2014   }
2015 
2016 #define INSN(NAME, type, opcode)                        \
2017   void NAME(FloatRegister Vd, FloatRegister Vn) {       \
2018     data_processing(type, opcode, Vd, Vn);              \
2019   }
2020 
2021   INSN(fmovs,  0b00, 0b000000);
2022   INSN(fabss,  0b00, 0b000001);
2023   INSN(fnegs,  0b00, 0b000010);
2024   INSN(fsqrts, 0b00, 0b000011);
2025   INSN(fcvts,  0b00, 0b000101);   // Single-precision to double-precision
2026   INSN(fcvths, 0b11, 0b000100);   // Half-precision to single-precision
2027   INSN(fcvtsh, 0b00, 0b000111);   // Single-precision to half-precision
2028 
2029   INSN(fmovd,  0b01, 0b000000);
2030   INSN(fabsd,  0b01, 0b000001);
2031   INSN(fnegd,  0b01, 0b000010);
2032   INSN(fsqrtd, 0b01, 0b000011);
2033   INSN(fcvtd,  0b01, 0b000100);   // Double-precision to single-precision
2034 
2035 private:
2036   void _fcvt_narrow_extend(FloatRegister Vd, SIMD_Arrangement Ta,
2037                            FloatRegister Vn, SIMD_Arrangement Tb, bool do_extend) {
2038     assert((do_extend && (Tb >> 1) + 1 == (Ta >> 1))
2039            || (!do_extend && (Ta >> 1) + 1 == (Tb >> 1)), "Incompatible arrangement");
2040     starti;
2041     int op30 = (do_extend ? Tb : Ta) & 1;
2042     int op22 = ((do_extend ? Ta : Tb) >> 1) & 1;
2043     f(0, 31), f(op30, 30), f(0b0011100, 29, 23), f(op22, 22);
2044     f(0b100001011, 21, 13), f(do_extend ? 1 : 0, 12), f(0b10, 11, 10);
2045     rf(Vn, 5), rf(Vd, 0);
2046   }
2047 
2048 public:
2049   void fcvtl(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn,  SIMD_Arrangement Tb) {
2050     assert(Tb == T4H || Tb == T8H|| Tb == T2S || Tb == T4S, "invalid arrangement");
2051     _fcvt_narrow_extend(Vd, Ta, Vn, Tb, true);
2052   }
2053 
2054   void fcvtn(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn,  SIMD_Arrangement Tb) {
2055     assert(Ta == T4H || Ta == T8H|| Ta == T2S || Ta == T4S, "invalid arrangement");
2056     _fcvt_narrow_extend(Vd, Ta, Vn, Tb, false);
2057   }
2058 
2059 #undef INSN
2060 
2061   // Floating-point data-processing (2 source)
2062   void data_processing(unsigned op31, unsigned type, unsigned opcode,
2063                        FloatRegister Vd, FloatRegister Vn, FloatRegister Vm) {
2064     starti;
2065     f(op31, 31, 29);
2066     f(0b11110, 28, 24);
2067     f(type, 23, 22), f(1, 21), f(opcode, 15, 10);
2068     rf(Vm, 16), rf(Vn, 5), rf(Vd, 0);
2069   }
2070 
2071 #define INSN(NAME, op31, type, opcode)                  \
2072   void NAME(FloatRegister Vd, FloatRegister Vn, FloatRegister Vm) {     \
2073     data_processing(op31, type, opcode, Vd, Vn, Vm);    \
2074   }
2075 
2076   INSN(fabds,  0b011, 0b10, 0b110101);
2077   INSN(fmuls,  0b000, 0b00, 0b000010);
2078   INSN(fdivs,  0b000, 0b00, 0b000110);
2079   INSN(fadds,  0b000, 0b00, 0b001010);
2080   INSN(fsubs,  0b000, 0b00, 0b001110);
2081   INSN(fmaxs,  0b000, 0b00, 0b010010);
2082   INSN(fmins,  0b000, 0b00, 0b010110);
2083   INSN(fnmuls, 0b000, 0b00, 0b100010);
2084 
2085   INSN(fabdd,  0b011, 0b11, 0b110101);
2086   INSN(fmuld,  0b000, 0b01, 0b000010);
2087   INSN(fdivd,  0b000, 0b01, 0b000110);
2088   INSN(faddd,  0b000, 0b01, 0b001010);
2089   INSN(fsubd,  0b000, 0b01, 0b001110);
2090   INSN(fmaxd,  0b000, 0b01, 0b010010);
2091   INSN(fmind,  0b000, 0b01, 0b010110);
2092   INSN(fnmuld, 0b000, 0b01, 0b100010);
2093 
2094 #undef INSN
2095 
2096    // Floating-point data-processing (3 source)
2097   void data_processing(unsigned op31, unsigned type, unsigned o1, unsigned o0,
2098                        FloatRegister Vd, FloatRegister Vn, FloatRegister Vm,
2099                        FloatRegister Va) {
2100     starti;
2101     f(op31, 31, 29);
2102     f(0b11111, 28, 24);
2103     f(type, 23, 22), f(o1, 21), f(o0, 15);
2104     rf(Vm, 16), rf(Va, 10), rf(Vn, 5), rf(Vd, 0);
2105   }
2106 
2107 #define INSN(NAME, op31, type, o1, o0)                                  \
2108   void NAME(FloatRegister Vd, FloatRegister Vn, FloatRegister Vm,       \
2109             FloatRegister Va) {                                         \
2110     data_processing(op31, type, o1, o0, Vd, Vn, Vm, Va);                \
2111   }
2112 
2113   INSN(fmadds,  0b000, 0b00, 0, 0);
2114   INSN(fmsubs,  0b000, 0b00, 0, 1);
2115   INSN(fnmadds, 0b000, 0b00, 1, 0);
2116   INSN(fnmsubs, 0b000, 0b00, 1, 1);
2117 
2118   INSN(fmaddd,  0b000, 0b01, 0, 0);
2119   INSN(fmsubd,  0b000, 0b01, 0, 1);
2120   INSN(fnmaddd, 0b000, 0b01, 1, 0);
2121   INSN(fnmsub,  0b000, 0b01, 1, 1);
2122 
2123 #undef INSN
2124 
2125    // Floating-point conditional select
2126   void fp_conditional_select(unsigned op31, unsigned type,
2127                              unsigned op1, unsigned op2,
2128                              Condition cond, FloatRegister Vd,
2129                              FloatRegister Vn, FloatRegister Vm) {
2130     starti;
2131     f(op31, 31, 29);
2132     f(0b11110, 28, 24);
2133     f(type, 23, 22);
2134     f(op1, 21, 21);
2135     f(op2, 11, 10);
2136     f(cond, 15, 12);
2137     rf(Vm, 16), rf(Vn, 5), rf(Vd, 0);
2138   }
2139 
2140 #define INSN(NAME, op31, type, op1, op2)                                \
2141   void NAME(FloatRegister Vd, FloatRegister Vn,                         \
2142             FloatRegister Vm, Condition cond) {                         \
2143     fp_conditional_select(op31, type, op1, op2, cond, Vd, Vn, Vm);      \
2144   }
2145 
2146   INSN(fcsels, 0b000, 0b00, 0b1, 0b11);
2147   INSN(fcseld, 0b000, 0b01, 0b1, 0b11);
2148 
2149 #undef INSN
2150 
2151   // Conversion between floating-point and integer
2152   void float_int_convert(unsigned sflag, unsigned ftype,
2153                          unsigned rmode, unsigned opcode,
2154                          Register Rd, Register Rn) {
2155     starti;
2156     f(sflag, 31);
2157     f(0b00, 30, 29);
2158     f(0b11110, 28, 24);
2159     f(ftype, 23, 22), f(1, 21), f(rmode, 20, 19);
2160     f(opcode, 18, 16), f(0b000000, 15, 10);
2161     zrf(Rn, 5), zrf(Rd, 0);
2162   }
2163 
2164 #define INSN(NAME, sflag, ftype, rmode, opcode)                          \
2165   void NAME(Register Rd, FloatRegister Vn) {                             \
2166     float_int_convert(sflag, ftype, rmode, opcode, Rd, as_Register(Vn)); \
2167   }
2168 
2169   INSN(fcvtzsw, 0b0, 0b00, 0b11, 0b000);
2170   INSN(fcvtzs,  0b1, 0b00, 0b11, 0b000);
2171   INSN(fcvtzdw, 0b0, 0b01, 0b11, 0b000);
2172   INSN(fcvtzd,  0b1, 0b01, 0b11, 0b000);
2173 
2174   // RoundToNearestTiesAway
2175   INSN(fcvtassw, 0b0, 0b00, 0b00, 0b100);  // float -> signed word
2176   INSN(fcvtasd,  0b1, 0b01, 0b00, 0b100);  // double -> signed xword
2177 
2178   // RoundTowardsNegative
2179   INSN(fcvtmssw, 0b0, 0b00, 0b10, 0b000);  // float -> signed word
2180   INSN(fcvtmsd,  0b1, 0b01, 0b10, 0b000);  // double -> signed xword
2181 
2182   INSN(fmovs, 0b0, 0b00, 0b00, 0b110);
2183   INSN(fmovd, 0b1, 0b01, 0b00, 0b110);
2184 
2185   INSN(fmovhid, 0b1, 0b10, 0b01, 0b110);
2186 
2187 #undef INSN
2188 
2189 #define INSN(NAME, sflag, type, rmode, opcode)                          \
2190   void NAME(FloatRegister Vd, Register Rn) {                            \
2191     float_int_convert(sflag, type, rmode, opcode, as_Register(Vd), Rn); \
2192   }
2193 
2194   INSN(fmovs, 0b0, 0b00, 0b00, 0b111);
2195   INSN(fmovd, 0b1, 0b01, 0b00, 0b111);
2196 
2197   INSN(scvtfws, 0b0, 0b00, 0b00, 0b010);
2198   INSN(scvtfs,  0b1, 0b00, 0b00, 0b010);
2199   INSN(scvtfwd, 0b0, 0b01, 0b00, 0b010);
2200   INSN(scvtfd,  0b1, 0b01, 0b00, 0b010);
2201 
2202   // INSN(fmovhid, 0b100, 0b10, 0b01, 0b111);
2203 
2204 #undef INSN
2205 
2206 private:
2207   void _xcvtf_vector_integer(bool is_unsigned, SIMD_Arrangement T,
2208                              FloatRegister Rd, FloatRegister Rn) {
2209     assert(T == T2S || T == T4S || T == T2D, "invalid arrangement");
2210     starti;
2211     f(0, 31), f(T & 1, 30), f(is_unsigned ? 1 : 0, 29);
2212     f(0b011100, 28, 23), f((T >> 1) & 1, 22), f(0b100001110110, 21, 10);
2213     rf(Rn, 5), rf(Rd, 0);
2214   }
2215 
2216 public:
2217 
2218   void scvtfv(SIMD_Arrangement T, FloatRegister Rd, FloatRegister Rn) {
2219     _xcvtf_vector_integer(/* is_unsigned */ false, T, Rd, Rn);
2220   }
2221 
2222   // Floating-point compare
2223   void float_compare(unsigned op31, unsigned type,
2224                      unsigned op, unsigned op2,
2225                      FloatRegister Vn, FloatRegister Vm = as_FloatRegister(0)) {
2226     starti;
2227     f(op31, 31, 29);
2228     f(0b11110, 28, 24);
2229     f(type, 23, 22), f(1, 21);
2230     f(op, 15, 14), f(0b1000, 13, 10), f(op2, 4, 0);
2231     rf(Vn, 5), rf(Vm, 16);
2232   }
2233 
2234 
2235 #define INSN(NAME, op31, type, op, op2)                 \
2236   void NAME(FloatRegister Vn, FloatRegister Vm) {       \
2237     float_compare(op31, type, op, op2, Vn, Vm);         \
2238   }
2239 
2240 #define INSN1(NAME, op31, type, op, op2)        \
2241   void NAME(FloatRegister Vn, double d) {       \
2242     assert_cond(d == 0.0);                      \
2243     float_compare(op31, type, op, op2, Vn);     \
2244   }
2245 
2246   INSN(fcmps, 0b000, 0b00, 0b00, 0b00000);
2247   INSN1(fcmps, 0b000, 0b00, 0b00, 0b01000);
2248   // INSN(fcmpes, 0b000, 0b00, 0b00, 0b10000);
2249   // INSN1(fcmpes, 0b000, 0b00, 0b00, 0b11000);
2250 
2251   INSN(fcmpd, 0b000,   0b01, 0b00, 0b00000);
2252   INSN1(fcmpd, 0b000,  0b01, 0b00, 0b01000);
2253   // INSN(fcmped, 0b000,  0b01, 0b00, 0b10000);
2254   // INSN1(fcmped, 0b000, 0b01, 0b00, 0b11000);
2255 
2256 #undef INSN
2257 #undef INSN1
2258 
2259 // Floating-point compare. 3-registers versions (scalar).
2260 #define INSN(NAME, sz, e)                                             \
2261   void NAME(FloatRegister Vd, FloatRegister Vn, FloatRegister Vm) {   \
2262     starti;                                                           \
2263     f(0b01111110, 31, 24), f(e, 23), f(sz, 22), f(1, 21), rf(Vm, 16); \
2264     f(0b111011, 15, 10), rf(Vn, 5), rf(Vd, 0);                        \
2265   }                                                                   \
2266 
2267   INSN(facged, 1, 0); // facge-double
2268   INSN(facges, 0, 0); // facge-single
2269   INSN(facgtd, 1, 1); // facgt-double
2270   INSN(facgts, 0, 1); // facgt-single
2271 
2272 #undef INSN
2273 
2274   // Floating-point Move (immediate)
2275 private:
2276   unsigned pack(double value);
2277 
2278   void fmov_imm(FloatRegister Vn, double value, unsigned size) {
2279     starti;
2280     f(0b00011110, 31, 24), f(size, 23, 22), f(1, 21);
2281     f(pack(value), 20, 13), f(0b10000000, 12, 5);
2282     rf(Vn, 0);
2283   }
2284 
2285 public:
2286 
2287   void fmovs(FloatRegister Vn, double value) {
2288     if (value)
2289       fmov_imm(Vn, value, 0b00);
2290     else
2291       movi(Vn, T2S, 0);
2292   }
2293   void fmovd(FloatRegister Vn, double value) {
2294     if (value)
2295       fmov_imm(Vn, value, 0b01);
2296     else
2297       movi(Vn, T1D, 0);
2298   }
2299 
2300   // Floating-point data-processing (1 source)
2301 
2302    // Floating-point rounding
2303    // type: half-precision = 11
2304    //       single         = 00
2305    //       double         = 01
2306    // rmode: A = Away     = 100
2307    //        I = current  = 111
2308    //        M = MinusInf = 010
2309    //        N = eveN     = 000
2310    //        P = PlusInf  = 001
2311    //        X = eXact    = 110
2312    //        Z = Zero     = 011
2313   void float_round(unsigned type, unsigned rmode, FloatRegister Rd, FloatRegister Rn) {
2314     starti;
2315     f(0b00011110, 31, 24);
2316     f(type, 23, 22);
2317     f(0b1001, 21, 18);
2318     f(rmode, 17, 15);
2319     f(0b10000, 14, 10);
2320     rf(Rn, 5), rf(Rd, 0);
2321   }
2322 #define INSN(NAME, type, rmode)                   \
2323   void NAME(FloatRegister Vd, FloatRegister Vn) { \
2324     float_round(type, rmode, Vd, Vn);             \
2325   }
2326 
2327 public:
2328   INSN(frintah, 0b11, 0b100);
2329   INSN(frintih, 0b11, 0b111);
2330   INSN(frintmh, 0b11, 0b010);
2331   INSN(frintnh, 0b11, 0b000);
2332   INSN(frintph, 0b11, 0b001);
2333   INSN(frintxh, 0b11, 0b110);
2334   INSN(frintzh, 0b11, 0b011);
2335 
2336   INSN(frintas, 0b00, 0b100);
2337   INSN(frintis, 0b00, 0b111);
2338   INSN(frintms, 0b00, 0b010);
2339   INSN(frintns, 0b00, 0b000);
2340   INSN(frintps, 0b00, 0b001);
2341   INSN(frintxs, 0b00, 0b110);
2342   INSN(frintzs, 0b00, 0b011);
2343 
2344   INSN(frintad, 0b01, 0b100);
2345   INSN(frintid, 0b01, 0b111);
2346   INSN(frintmd, 0b01, 0b010);
2347   INSN(frintnd, 0b01, 0b000);
2348   INSN(frintpd, 0b01, 0b001);
2349   INSN(frintxd, 0b01, 0b110);
2350   INSN(frintzd, 0b01, 0b011);
2351 #undef INSN
2352 
2353 private:
2354   static short SIMD_Size_in_bytes[];
2355 
2356 public:
2357 #define INSN(NAME, op)                                                  \
2358   void NAME(FloatRegister Rt, SIMD_RegVariant T, const Address &adr) {  \
2359     ld_st2(as_Register(Rt), adr, (int)T & 3, op + ((T==Q) ? 0b10:0b00), 1); \
2360   }
2361 
2362   INSN(ldr, 1);
2363   INSN(str, 0);
2364 
2365 #undef INSN
2366 
2367  private:
2368 
2369   void ld_st(FloatRegister Vt, SIMD_Arrangement T, Register Xn, int op1, int op2) {
2370     starti;
2371     f(0,31), f((int)T & 1, 30);
2372     f(op1, 29, 21), f(0, 20, 16), f(op2, 15, 12);
2373     f((int)T >> 1, 11, 10), srf(Xn, 5), rf(Vt, 0);
2374   }
2375   void ld_st(FloatRegister Vt, SIMD_Arrangement T, Register Xn,
2376              int imm, int op1, int op2, int regs) {
2377 
2378     bool replicate = op2 >> 2 == 3;
2379     // post-index value (imm) is formed differently for replicate/non-replicate ld* instructions
2380     int expectedImmediate = replicate ? regs * (1 << (T >> 1)) : SIMD_Size_in_bytes[T] * regs;
2381     guarantee(T < T1Q , "incorrect arrangement");
2382     guarantee(imm == expectedImmediate, "bad offset");
2383     starti;
2384     f(0,31), f((int)T & 1, 30);
2385     f(op1 | 0b100, 29, 21), f(0b11111, 20, 16), f(op2, 15, 12);
2386     f((int)T >> 1, 11, 10), srf(Xn, 5), rf(Vt, 0);
2387   }
2388   void ld_st(FloatRegister Vt, SIMD_Arrangement T, Register Xn,
2389              Register Xm, int op1, int op2) {
2390     starti;
2391     f(0,31), f((int)T & 1, 30);
2392     f(op1 | 0b100, 29, 21), rf(Xm, 16), f(op2, 15, 12);
2393     f((int)T >> 1, 11, 10), srf(Xn, 5), rf(Vt, 0);
2394   }
2395 
2396   void ld_st(FloatRegister Vt, SIMD_Arrangement T, Address a, int op1, int op2, int regs) {
2397     switch (a.getMode()) {
2398     case Address::base_plus_offset:
2399       guarantee(a.offset() == 0, "no offset allowed here");
2400       ld_st(Vt, T, a.base(), op1, op2);
2401       break;
2402     case Address::post:
2403       ld_st(Vt, T, a.base(), checked_cast<int>(a.offset()), op1, op2, regs);
2404       break;
2405     case Address::post_reg:
2406       ld_st(Vt, T, a.base(), a.index(), op1, op2);
2407       break;
2408     default:
2409       ShouldNotReachHere();
2410     }
2411   }
2412 
2413   // Single-structure load/store method (all addressing variants)
2414   void ld_st(FloatRegister Vt, SIMD_RegVariant T, int index, Address a,
2415              int op1, int op2, int regs) {
2416     int expectedImmediate = (regVariant_to_elemBits(T) >> 3) * regs;
2417     int sVal = (T < D) ? (index >> (2 - T)) & 0x01 : 0;
2418     int opcode = (T < D) ? (T << 2) : ((T & 0x02) << 2);
2419     int size = (T < D) ? (index & (0x3 << T)) : 1;  // only care about low 2b
2420     Register Xn = a.base();
2421     int Rm;
2422 
2423     switch (a.getMode()) {
2424     case Address::base_plus_offset:
2425       guarantee(a.offset() == 0, "no offset allowed here");
2426       Rm = 0;
2427       break;
2428     case Address::post:
2429       guarantee(a.offset() == expectedImmediate, "bad offset");
2430       op1 |= 0b100;
2431       Rm = 0b11111;
2432       break;
2433     case Address::post_reg:
2434       op1 |= 0b100;
2435       Rm = a.index()->encoding();
2436       break;
2437     default:
2438       ShouldNotReachHere();
2439       Rm = 0;  // unreachable
2440     }
2441 
2442     starti;
2443     f(0,31), f((index >> (3 - T)), 30);
2444     f(op1, 29, 21), f(Rm, 20, 16), f(op2 | opcode | sVal, 15, 12);
2445     f(size, 11, 10), srf(Xn, 5), rf(Vt, 0);
2446   }
2447 
2448  public:
2449 
2450 #define INSN1(NAME, op1, op2)                                           \
2451   void NAME(FloatRegister Vt, SIMD_Arrangement T, const Address &a) {   \
2452     ld_st(Vt, T, a, op1, op2, 1);                                       \
2453  }
2454 
2455 #define INSN2(NAME, op1, op2)                                           \
2456   void NAME(FloatRegister Vt, FloatRegister Vt2, SIMD_Arrangement T, const Address &a) { \
2457     assert(Vt->successor() == Vt2, "Registers must be ordered");        \
2458     ld_st(Vt, T, a, op1, op2, 2);                                       \
2459   }
2460 
2461 #define INSN3(NAME, op1, op2)                                           \
2462   void NAME(FloatRegister Vt, FloatRegister Vt2, FloatRegister Vt3,     \
2463             SIMD_Arrangement T, const Address &a) {                     \
2464     assert(Vt->successor() == Vt2 && Vt2->successor() == Vt3,           \
2465            "Registers must be ordered");                                \
2466     ld_st(Vt, T, a, op1, op2, 3);                                       \
2467   }
2468 
2469 #define INSN4(NAME, op1, op2)                                           \
2470   void NAME(FloatRegister Vt, FloatRegister Vt2, FloatRegister Vt3,     \
2471             FloatRegister Vt4, SIMD_Arrangement T, const Address &a) {  \
2472     assert(Vt->successor() == Vt2 && Vt2->successor() == Vt3 &&         \
2473            Vt3->successor() == Vt4, "Registers must be ordered");       \
2474     ld_st(Vt, T, a, op1, op2, 4);                                       \
2475   }
2476 
2477   INSN1(ld1,  0b001100010, 0b0111);
2478   INSN2(ld1,  0b001100010, 0b1010);
2479   INSN3(ld1,  0b001100010, 0b0110);
2480   INSN4(ld1,  0b001100010, 0b0010);
2481 
2482   INSN2(ld2,  0b001100010, 0b1000);
2483   INSN3(ld3,  0b001100010, 0b0100);
2484   INSN4(ld4,  0b001100010, 0b0000);
2485 
2486   INSN1(st1,  0b001100000, 0b0111);
2487   INSN2(st1,  0b001100000, 0b1010);
2488   INSN3(st1,  0b001100000, 0b0110);
2489   INSN4(st1,  0b001100000, 0b0010);
2490 
2491   INSN2(st2,  0b001100000, 0b1000);
2492   INSN3(st3,  0b001100000, 0b0100);
2493   INSN4(st4,  0b001100000, 0b0000);
2494 
2495   INSN1(ld1r, 0b001101010, 0b1100);
2496   INSN2(ld2r, 0b001101011, 0b1100);
2497   INSN3(ld3r, 0b001101010, 0b1110);
2498   INSN4(ld4r, 0b001101011, 0b1110);
2499 
2500 #undef INSN1
2501 #undef INSN2
2502 #undef INSN3
2503 #undef INSN4
2504 
2505 // Handle common single-structure ld/st parameter sanity checks
2506 // for all variations (1 to 4) of SIMD reigster inputs.  This
2507 // method will call the routine that generates the opcode.
2508 template<typename R, typename... Rx>
2509   void ldst_sstr(SIMD_RegVariant T, int index, const Address &a,
2510             int op1, int op2, R firstReg, Rx... otherRegs) {
2511     const FloatRegister vtSet[] = { firstReg, otherRegs... };
2512     const int regCount = sizeof...(otherRegs) + 1;
2513     assert(index >= 0 && (T <= D) && ((T == B && index <= 15) ||
2514               (T == H && index <= 7) || (T == S && index <= 3) ||
2515               (T == D && index <= 1)), "invalid index");
2516     assert(regCount >= 1 && regCount <= 4, "illegal register count");
2517 
2518     // Check to make sure when multiple SIMD registers are used
2519     // that they are in successive order.
2520     for (int i = 0; i < regCount - 1; i++) {
2521       assert(vtSet[i]->successor() == vtSet[i + 1],
2522              "Registers must be ordered");
2523     }
2524 
2525     ld_st(firstReg, T, index, a, op1, op2, regCount);
2526   }
2527 
2528 // Define a set of INSN1/2/3/4 macros to handle single-structure
2529 // load/store instructions.
2530 #define INSN1(NAME, op1, op2)                                           \
2531   void NAME(FloatRegister Vt, SIMD_RegVariant T, int index,             \
2532             const Address &a) {                                         \
2533     ldst_sstr(T, index, a, op1, op2, Vt);                               \
2534  }
2535 
2536 #define INSN2(NAME, op1, op2)                                           \
2537   void NAME(FloatRegister Vt, FloatRegister Vt2, SIMD_RegVariant T,     \
2538             int index, const Address &a) {                              \
2539     ldst_sstr(T, index, a, op1, op2, Vt, Vt2);                          \
2540   }
2541 
2542 #define INSN3(NAME, op1, op2)                                           \
2543   void NAME(FloatRegister Vt, FloatRegister Vt2, FloatRegister Vt3,     \
2544             SIMD_RegVariant T, int index, const Address &a) {           \
2545     ldst_sstr(T, index, a, op1, op2, Vt, Vt2, Vt3);                     \
2546   }
2547 
2548 #define INSN4(NAME, op1, op2)                                           \
2549   void NAME(FloatRegister Vt, FloatRegister Vt2, FloatRegister Vt3,     \
2550             FloatRegister Vt4, SIMD_RegVariant T, int index,            \
2551             const Address &a) {                                         \
2552     ldst_sstr(T, index, a, op1, op2, Vt, Vt2, Vt3, Vt4);                \
2553   }
2554 
2555   INSN1(ld1, 0b001101010, 0b0000);
2556   INSN2(ld2, 0b001101011, 0b0000);
2557   INSN3(ld3, 0b001101010, 0b0010);
2558   INSN4(ld4, 0b001101011, 0b0010);
2559 
2560   INSN1(st1, 0b001101000, 0b0000);
2561   INSN2(st2, 0b001101001, 0b0000);
2562   INSN3(st3, 0b001101000, 0b0010);
2563   INSN4(st4, 0b001101001, 0b0010);
2564 
2565 #undef INSN1
2566 #undef INSN2
2567 #undef INSN3
2568 #undef INSN4
2569 
2570 #define INSN(NAME, opc)                                                                 \
2571   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \
2572     starti;                                                                             \
2573     assert(T == T8B || T == T16B, "must be T8B or T16B");                               \
2574     f(0, 31), f((int)T & 1, 30), f(opc, 29, 21);                                        \
2575     rf(Vm, 16), f(0b000111, 15, 10), rf(Vn, 5), rf(Vd, 0);                              \
2576   }
2577 
2578   INSN(eor,  0b101110001);
2579   INSN(orr,  0b001110101);
2580   INSN(andr, 0b001110001);
2581   INSN(bic,  0b001110011);
2582   INSN(bif,  0b101110111);
2583   INSN(bit,  0b101110101);
2584   INSN(bsl,  0b101110011);
2585   INSN(orn,  0b001110111);
2586 
2587 #undef INSN
2588 
2589   // Advanced SIMD three different
2590 #define INSN(NAME, opc, opc2, acceptT2D)                                                \
2591   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \
2592     guarantee(T != T1Q && T != T1D, "incorrect arrangement");                           \
2593     if (!acceptT2D) guarantee(T != T2D, "incorrect arrangement");                       \
2594     if (opc2 ==  0b101101) guarantee(T != T8B && T != T16B, "incorrect arrangement");   \
2595     starti;                                                                             \
2596     f(0, 31), f((int)T & 1, 30), f(opc, 29), f(0b01110, 28, 24);                        \
2597     f((int)T >> 1, 23, 22), f(1, 21), rf(Vm, 16), f(opc2, 15, 10);                      \
2598     rf(Vn, 5), rf(Vd, 0);                                                               \
2599   }
2600 
2601   INSN(addv,   0, 0b100001, true);  // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2602   INSN(subv,   1, 0b100001, true);  // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2603   INSN(sqaddv, 0, 0b000011, true);  // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2604   INSN(sqsubv, 0, 0b001011, true);  // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2605   INSN(uqaddv, 1, 0b000011, true);  // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2606   INSN(uqsubv, 1, 0b001011, true);  // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2607   INSN(mulv,   0, 0b100111, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2608   INSN(mlav,   0, 0b100101, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2609   INSN(mlsv,   1, 0b100101, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2610   INSN(sshl,   0, 0b010001, true);  // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2611   INSN(ushl,   1, 0b010001, true);  // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2612   INSN(addpv,  0, 0b101111, true);  // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2613   INSN(smullv, 0, 0b110000, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2614   INSN(umullv, 1, 0b110000, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2615   INSN(smlalv, 0, 0b100000, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2616   INSN(umlalv, 1, 0b100000, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2617   INSN(maxv,   0, 0b011001, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2618   INSN(minv,   0, 0b011011, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2619   INSN(umaxv,  1, 0b011001, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2620   INSN(uminv,  1, 0b011011, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2621   INSN(smaxp,  0, 0b101001, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2622   INSN(sminp,  0, 0b101011, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2623   INSN(sqdmulh,0, 0b101101, false); // accepted arrangements: T4H, T8H, T2S, T4S
2624   INSN(shsubv, 0, 0b001001, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2625 
2626 #undef INSN
2627 
2628   // Advanced SIMD across lanes
2629 #define INSN(NAME, opc, opc2, accepted) \
2630   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {                   \
2631     guarantee(T != T1Q && T != T1D, "incorrect arrangement");                           \
2632     if (accepted < 3) guarantee(T != T2D, "incorrect arrangement");                     \
2633     if (accepted < 2) guarantee(T != T2S, "incorrect arrangement");                     \
2634     if (accepted < 1) guarantee(T == T8B || T == T16B, "incorrect arrangement");        \
2635     starti;                                                                             \
2636     f(0, 31), f((int)T & 1, 30), f(opc, 29), f(0b01110, 28, 24);                        \
2637     f((int)T >> 1, 23, 22), f(opc2, 21, 10);                                            \
2638     rf(Vn, 5), rf(Vd, 0);                                                               \
2639   }
2640 
2641   INSN(absr,   0, 0b100000101110, 3); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2642   INSN(negr,   1, 0b100000101110, 3); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2643   INSN(notr,   1, 0b100000010110, 0); // accepted arrangements: T8B, T16B
2644   INSN(addv,   0, 0b110001101110, 1); // accepted arrangements: T8B, T16B, T4H, T8H,      T4S
2645   INSN(smaxv,  0, 0b110000101010, 1); // accepted arrangements: T8B, T16B, T4H, T8H,      T4S
2646   INSN(umaxv,  1, 0b110000101010, 1); // accepted arrangements: T8B, T16B, T4H, T8H,      T4S
2647   INSN(sminv,  0, 0b110001101010, 1); // accepted arrangements: T8B, T16B, T4H, T8H,      T4S
2648   INSN(uminv,  1, 0b110001101010, 1); // accepted arrangements: T8B, T16B, T4H, T8H,      T4S
2649   INSN(cls,    0, 0b100000010010, 2); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2650   INSN(clz,    1, 0b100000010010, 2); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2651   INSN(cnt,    0, 0b100000010110, 0); // accepted arrangements: T8B, T16B
2652   INSN(uaddlp, 1, 0b100000001010, 2); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2653   INSN(uaddlv, 1, 0b110000001110, 1); // accepted arrangements: T8B, T16B, T4H, T8H,      T4S
2654 
2655 #undef INSN
2656 
2657 #define INSN(NAME, opc) \
2658   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {                  \
2659     starti;                                                                            \
2660     assert(T == T4S, "arrangement must be T4S");                                       \
2661     f(0, 31), f((int)T & 1, 30), f(0b101110, 29, 24), f(opc, 23),                      \
2662     f(T == T4S ? 0 : 1, 22), f(0b110000111110, 21, 10); rf(Vn, 5), rf(Vd, 0);          \
2663   }
2664 
2665   INSN(fmaxv, 0);
2666   INSN(fminv, 1);
2667 
2668 #undef INSN
2669 
2670 // Advanced SIMD modified immediate
2671 #define INSN(NAME, op0, cmode0) \
2672   void NAME(FloatRegister Vd, SIMD_Arrangement T, unsigned imm8, unsigned lsl = 0) {   \
2673     unsigned cmode = cmode0;                                                           \
2674     unsigned op = op0;                                                                 \
2675     starti;                                                                            \
2676     assert(lsl == 0 ||                                                                 \
2677            ((T == T4H || T == T8H) && lsl == 8) ||                                     \
2678            ((T == T2S || T == T4S) && ((lsl >> 3) < 4) && ((lsl & 7) == 0)), "invalid shift");\
2679     cmode |= lsl >> 2;                                                                 \
2680     if (T == T4H || T == T8H) cmode |= 0b1000;                                         \
2681     if (!(T == T4H || T == T8H || T == T2S || T == T4S)) {                             \
2682       assert(op == 0 && cmode0 == 0, "must be MOVI");                                  \
2683       cmode = 0b1110;                                                                  \
2684       if (T == T1D || T == T2D) op = 1;                                                \
2685     }                                                                                  \
2686     f(0, 31), f((int)T & 1, 30), f(op, 29), f(0b0111100000, 28, 19);                   \
2687     f(imm8 >> 5, 18, 16), f(cmode, 15, 12), f(0x01, 11, 10), f(imm8 & 0b11111, 9, 5);  \
2688     rf(Vd, 0);                                                                         \
2689   }
2690 
2691   INSN(movi, 0, 0);
2692   INSN(orri, 0, 1);
2693   INSN(mvni, 1, 0);
2694   INSN(bici, 1, 1);
2695 
2696 #undef INSN
2697 
2698 #define INSN(NAME, op, cmode)                                           \
2699   void NAME(FloatRegister Vd, SIMD_Arrangement T, double imm) {         \
2700     unsigned imm8 = pack(imm);                                          \
2701     starti;                                                             \
2702     f(0, 31), f((int)T & 1, 30), f(op, 29), f(0b0111100000, 28, 19);    \
2703     f(imm8 >> 5, 18, 16), f(cmode, 15, 12), f(0x01, 11, 10), f(imm8 & 0b11111, 9, 5); \
2704     rf(Vd, 0);                                                          \
2705   }
2706 
2707   INSN(fmovs, 0, 0b1111);
2708   INSN(fmovd, 1, 0b1111);
2709 
2710 #undef INSN
2711 
2712 // Advanced SIMD three same
2713 #define INSN(NAME, op1, op2, op3)                                                       \
2714   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \
2715     starti;                                                                             \
2716     assert(T == T2S || T == T4S || T == T2D, "invalid arrangement");                    \
2717     f(0, 31), f((int)T & 1, 30), f(op1, 29), f(0b01110, 28, 24), f(op2, 23);            \
2718     f(T==T2D ? 1:0, 22); f(1, 21), rf(Vm, 16), f(op3, 15, 10), rf(Vn, 5), rf(Vd, 0);    \
2719   }
2720 
2721   INSN(fabd, 1, 1, 0b110101);
2722   INSN(fadd, 0, 0, 0b110101);
2723   INSN(fdiv, 1, 0, 0b111111);
2724   INSN(faddp, 1, 0, 0b110101);
2725   INSN(fmul, 1, 0, 0b110111);
2726   INSN(fsub, 0, 1, 0b110101);
2727   INSN(fmla, 0, 0, 0b110011);
2728   INSN(fmls, 0, 1, 0b110011);
2729   INSN(fmax, 0, 0, 0b111101);
2730   INSN(fmin, 0, 1, 0b111101);
2731   INSN(facgt, 1, 1, 0b111011);
2732 
2733 #undef INSN
2734 
2735   // AdvSIMD vector compare
2736   void cm(Condition cond, FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) {
2737     starti;
2738     assert(T != T1Q && T != T1D, "incorrect arrangement");
2739     int cond_op;
2740     switch (cond) {
2741       case EQ: cond_op = 0b110001; break;
2742       case GT: cond_op = 0b000110; break;
2743       case GE: cond_op = 0b000111; break;
2744       case HI: cond_op = 0b100110; break;
2745       case HS: cond_op = 0b100111; break;
2746       default:
2747         ShouldNotReachHere();
2748         break;
2749     }
2750 
2751     f(0, 31), f((int)T & 1, 30), f((cond_op >> 5) & 1, 29);
2752     f(0b01110, 28, 24), f((int)T >> 1, 23, 22), f(1, 21), rf(Vm, 16);
2753     f(cond_op & 0b11111, 15, 11), f(1, 10), rf(Vn, 5), rf(Vd, 0);
2754   }
2755 
2756   // AdvSIMD Floating-point vector compare
2757   void fcm(Condition cond, FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) {
2758     starti;
2759     assert(T == T2S || T == T4S || T == T2D, "invalid arrangement");
2760     int cond_op;
2761     switch (cond) {
2762       case EQ: cond_op = 0b00; break;
2763       case GT: cond_op = 0b11; break;
2764       case GE: cond_op = 0b10; break;
2765       default:
2766         ShouldNotReachHere();
2767         break;
2768     }
2769 
2770     f(0, 31), f((int)T & 1, 30), f((cond_op >> 1) & 1, 29);
2771     f(0b01110, 28, 24), f(cond_op & 1, 23), f(T == T2D ? 1 : 0, 22);
2772     f(1, 21), rf(Vm, 16), f(0b111001, 15, 10), rf(Vn, 5), rf(Vd, 0);
2773   }
2774 
2775 #define INSN(NAME, opc)                                                                 \
2776   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \
2777     starti;                                                                             \
2778     assert(T == T4S, "arrangement must be T4S");                                        \
2779     f(0b01011110000, 31, 21), rf(Vm, 16), f(opc, 15, 10), rf(Vn, 5), rf(Vd, 0);         \
2780   }
2781 
2782   INSN(sha1c,     0b000000);
2783   INSN(sha1m,     0b001000);
2784   INSN(sha1p,     0b000100);
2785   INSN(sha1su0,   0b001100);
2786   INSN(sha256h2,  0b010100);
2787   INSN(sha256h,   0b010000);
2788   INSN(sha256su1, 0b011000);
2789 
2790 #undef INSN
2791 
2792 #define INSN(NAME, opc)                                                                 \
2793   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {                   \
2794     starti;                                                                             \
2795     assert(T == T4S, "arrangement must be T4S");                                        \
2796     f(0b0101111000101000, 31, 16), f(opc, 15, 10), rf(Vn, 5), rf(Vd, 0);                \
2797   }
2798 
2799   INSN(sha1h,     0b000010);
2800   INSN(sha1su1,   0b000110);
2801   INSN(sha256su0, 0b001010);
2802 
2803 #undef INSN
2804 
2805 #define INSN(NAME, opc)                                                                 \
2806   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \
2807     starti;                                                                             \
2808     assert(T == T2D, "arrangement must be T2D");                                        \
2809     f(0b11001110011, 31, 21), rf(Vm, 16), f(opc, 15, 10), rf(Vn, 5), rf(Vd, 0);         \
2810   }
2811 
2812   INSN(sha512h,   0b100000);
2813   INSN(sha512h2,  0b100001);
2814   INSN(sha512su1, 0b100010);
2815 
2816 #undef INSN
2817 
2818 #define INSN(NAME, opc)                                                                 \
2819   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {                   \
2820     starti;                                                                             \
2821     assert(T == T2D, "arrangement must be T2D");                                        \
2822     f(opc, 31, 10), rf(Vn, 5), rf(Vd, 0);                                               \
2823   }
2824 
2825   INSN(sha512su0, 0b1100111011000000100000);
2826 
2827 #undef INSN
2828 
2829 #define INSN(NAME, opc)                                                                                   \
2830   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm, FloatRegister Va) { \
2831     starti;                                                                                               \
2832     assert(T == T16B, "arrangement must be T16B");                                                        \
2833     f(0b11001110, 31, 24), f(opc, 23, 21), rf(Vm, 16), f(0b0, 15, 15), rf(Va, 10), rf(Vn, 5), rf(Vd, 0);  \
2834   }
2835 
2836   INSN(eor3, 0b000);
2837   INSN(bcax, 0b001);
2838 
2839 #undef INSN
2840 
2841 #define INSN(NAME, opc)                                                                               \
2842   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm, unsigned imm) { \
2843     starti;                                                                                           \
2844     assert(T == T2D, "arrangement must be T2D");                                                      \
2845     f(0b11001110, 31, 24), f(opc, 23, 21), rf(Vm, 16), f(imm, 15, 10), rf(Vn, 5), rf(Vd, 0);          \
2846   }
2847 
2848   INSN(xar, 0b100);
2849 
2850 #undef INSN
2851 
2852 #define INSN(NAME, opc)                                                                           \
2853   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) {           \
2854     starti;                                                                                       \
2855     assert(T == T2D, "arrangement must be T2D");                                                  \
2856     f(0b11001110, 31, 24), f(opc, 23, 21), rf(Vm, 16), f(0b100011, 15, 10), rf(Vn, 5), rf(Vd, 0); \
2857   }
2858 
2859   INSN(rax1, 0b011);
2860 
2861 #undef INSN
2862 
2863 #define INSN(NAME, opc)                           \
2864   void NAME(FloatRegister Vd, FloatRegister Vn) { \
2865     starti;                                       \
2866     f(opc, 31, 10), rf(Vn, 5), rf(Vd, 0);         \
2867   }
2868 
2869   INSN(aese,   0b0100111000101000010010);
2870   INSN(aesd,   0b0100111000101000010110);
2871   INSN(aesmc,  0b0100111000101000011010);
2872   INSN(aesimc, 0b0100111000101000011110);
2873 
2874 #undef INSN
2875 
2876 #define INSN(NAME, op1, op2) \
2877   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm, int index = 0) { \
2878     starti;                                                                                            \
2879     assert(T == T2S || T == T4S || T == T2D, "invalid arrangement");                                   \
2880     assert(index >= 0 && ((T == T2D && index <= 1) || (T != T2D && index <= 3)), "invalid index");     \
2881     f(0, 31), f((int)T & 1, 30), f(op1, 29); f(0b011111, 28, 23);                                      \
2882     f(T == T2D ? 1 : 0, 22), f(T == T2D ? 0 : index & 1, 21), rf(Vm, 16);                              \
2883     f(op2, 15, 12), f(T == T2D ? index : (index >> 1), 11), f(0, 10);                                  \
2884     rf(Vn, 5), rf(Vd, 0);                                                                              \
2885   }
2886 
2887   // FMLA/FMLS - Vector - Scalar
2888   INSN(fmlavs, 0, 0b0001);
2889   INSN(fmlsvs, 0, 0b0101);
2890   // FMULX - Vector - Scalar
2891   INSN(fmulxvs, 1, 0b1001);
2892 
2893 #undef INSN
2894 
2895 #define INSN(NAME, op1, op2)                                                                       \
2896   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm, int index) { \
2897     starti;                                                                                        \
2898     assert(T == T4H || T == T8H || T == T2S || T == T4S, "invalid arrangement");                   \
2899     assert(index >= 0 &&                                                                           \
2900                ((T == T2S && index <= 1) || (T != T2S && index <= 3) || (T == T8H && index <= 7)), \
2901            "invalid index");                                                                       \
2902     assert((T != T4H && T != T8H) || Vm->encoding() < 16, "invalid source SIMD&FP register");      \
2903     f(0, 31), f((int)T & 1, 30), f(op1, 29), f(0b01111, 28, 24);                                   \
2904     if (T == T4H || T == T8H) {                                                                    \
2905       f(0b01, 23, 22), f(index & 0b11, 21, 20), lrf(Vm, 16), f(index >> 2 & 1, 11);                \
2906     } else {                                                                                       \
2907       f(0b10, 23, 22), f(index & 1, 21), rf(Vm, 16), f(index >> 1, 11);                            \
2908     }                                                                                              \
2909     f(op2, 15, 12), f(0, 10), rf(Vn, 5), rf(Vd, 0);                                                \
2910   }
2911 
2912   // MUL - Vector - Scalar
2913   INSN(mulvs, 0, 0b1000);
2914 
2915 #undef INSN
2916 
2917   // Floating-point Reciprocal Estimate
2918   void frecpe(FloatRegister Vd, FloatRegister Vn, SIMD_RegVariant type) {
2919     assert(type == D || type == S, "Wrong type for frecpe");
2920     starti;
2921     f(0b010111101, 31, 23);
2922     f(type == D ? 1 : 0, 22);
2923     f(0b100001110110, 21, 10);
2924     rf(Vn, 5), rf(Vd, 0);
2925   }
2926 
2927   // (long) {a, b} -> (a + b)
2928   void addpd(FloatRegister Vd, FloatRegister Vn) {
2929     starti;
2930     f(0b0101111011110001101110, 31, 10);
2931     rf(Vn, 5), rf(Vd, 0);
2932   }
2933 
2934   // Floating-point AdvSIMD scalar pairwise
2935 #define INSN(NAME, op1, op2) \
2936   void NAME(FloatRegister Vd, FloatRegister Vn, SIMD_RegVariant type) {                 \
2937     starti;                                                                             \
2938     assert(type == D || type == S, "Wrong type for faddp/fmaxp/fminp");                 \
2939     f(0b0111111, 31, 25), f(op1, 24, 23),                                               \
2940     f(type == S ? 0 : 1, 22), f(0b11000, 21, 17), f(op2, 16, 10), rf(Vn, 5), rf(Vd, 0); \
2941   }
2942 
2943   INSN(faddp, 0b00, 0b0110110);
2944   INSN(fmaxp, 0b00, 0b0111110);
2945   INSN(fminp, 0b01, 0b0111110);
2946 
2947 #undef INSN
2948 
2949   void ins(FloatRegister Vd, SIMD_RegVariant T, FloatRegister Vn, int didx, int sidx) {
2950     starti;
2951     assert(T != Q, "invalid register variant");
2952     f(0b01101110000, 31, 21), f(((didx<<1)|1)<<(int)T, 20, 16), f(0, 15);
2953     f(sidx<<(int)T, 14, 11), f(1, 10), rf(Vn, 5), rf(Vd, 0);
2954   }
2955 
2956 #define INSN(NAME, cond, op1, op2)                                                      \
2957   void NAME(Register Rd, FloatRegister Vn, SIMD_RegVariant T, int idx) {                \
2958     starti;                                                                             \
2959     assert(cond, "invalid register variant");                                           \
2960     f(0, 31), f(op1, 30), f(0b001110000, 29, 21);                                       \
2961     f(((idx << 1) | 1) << (int)T, 20, 16), f(op2, 15, 10);                              \
2962     rf(Vn, 5), rf(Rd, 0);                                                               \
2963   }
2964 
2965   INSN(umov, (T != Q), (T == D ? 1 : 0), 0b001111);
2966   INSN(smov, (T < D),  1,                0b001011);
2967 
2968 #undef INSN
2969 
2970 #define INSN(NAME, opc, opc2, isSHR)                                    \
2971   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, int shift){ \
2972     starti;                                                             \
2973     /* The encodings for the immh:immb fields (bits 22:16) in *SHR are  \
2974      *   0001 xxx       8B/16B, shift = 16  - UInt(immh:immb)           \
2975      *   001x xxx       4H/8H,  shift = 32  - UInt(immh:immb)           \
2976      *   01xx xxx       2S/4S,  shift = 64  - UInt(immh:immb)           \
2977      *   1xxx xxx       1D/2D,  shift = 128 - UInt(immh:immb)           \
2978      *   (1D is RESERVED)                                               \
2979      * for SHL shift is calculated as:                                  \
2980      *   0001 xxx       8B/16B, shift = UInt(immh:immb) - 8             \
2981      *   001x xxx       4H/8H,  shift = UInt(immh:immb) - 16            \
2982      *   01xx xxx       2S/4S,  shift = UInt(immh:immb) - 32            \
2983      *   1xxx xxx       1D/2D,  shift = UInt(immh:immb) - 64            \
2984      *   (1D is RESERVED)                                               \
2985      */                                                                 \
2986     guarantee(!isSHR || (isSHR && (shift != 0)), "impossible encoding");\
2987     assert((1 << ((T>>1)+3)) > shift, "Invalid Shift value");           \
2988     int cVal = (1 << (((T >> 1) + 3) + (isSHR ? 1 : 0)));               \
2989     int encodedShift = isSHR ? cVal - shift : cVal + shift;             \
2990     f(0, 31), f(T & 1, 30), f(opc, 29), f(0b011110, 28, 23),            \
2991     f(encodedShift, 22, 16); f(opc2, 15, 10), rf(Vn, 5), rf(Vd, 0);     \
2992   }
2993 
2994   INSN(shl,  0, 0b010101, /* isSHR = */ false);
2995   INSN(sshr, 0, 0b000001, /* isSHR = */ true);
2996   INSN(ushr, 1, 0b000001, /* isSHR = */ true);
2997   INSN(usra, 1, 0b000101, /* isSHR = */ true);
2998   INSN(ssra, 0, 0b000101, /* isSHR = */ true);
2999   INSN(sli,  1, 0b010101, /* isSHR = */ false);
3000 
3001 #undef INSN
3002 
3003 #define INSN(NAME, opc, opc2, isSHR)                                    \
3004   void NAME(FloatRegister Vd, FloatRegister Vn, int shift){             \
3005     starti;                                                             \
3006     int encodedShift = isSHR ? 128 - shift : 64 + shift;                \
3007     f(0b01, 31, 30), f(opc, 29), f(0b111110, 28, 23),                   \
3008     f(encodedShift, 22, 16); f(opc2, 15, 10), rf(Vn, 5), rf(Vd, 0);     \
3009   }
3010 
3011   INSN(shld,  0, 0b010101, /* isSHR = */ false);
3012   INSN(sshrd, 0, 0b000001, /* isSHR = */ true);
3013   INSN(ushrd, 1, 0b000001, /* isSHR = */ true);
3014 
3015 #undef INSN
3016 
3017 protected:
3018   void _xshll(bool is_unsigned, FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn, SIMD_Arrangement Tb, int shift) {
3019     starti;
3020     /* The encodings for the immh:immb fields (bits 22:16) are
3021      *   0001 xxx       8H, 8B/16B shift = xxx
3022      *   001x xxx       4S, 4H/8H  shift = xxxx
3023      *   01xx xxx       2D, 2S/4S  shift = xxxxx
3024      *   1xxx xxx       RESERVED
3025      */
3026     assert((Tb >> 1) + 1 == (Ta >> 1), "Incompatible arrangement");
3027     assert((1 << ((Tb>>1)+3)) > shift, "Invalid shift value");
3028     f(0, 31), f(Tb & 1, 30), f(is_unsigned ? 1 : 0, 29), f(0b011110, 28, 23);
3029     f((1 << ((Tb>>1)+3))|shift, 22, 16);
3030     f(0b101001, 15, 10), rf(Vn, 5), rf(Vd, 0);
3031   }
3032 
3033 public:
3034   void ushll(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn,  SIMD_Arrangement Tb, int shift) {
3035     assert(Tb == T8B || Tb == T4H || Tb == T2S, "invalid arrangement");
3036     _xshll(/* is_unsigned */ true, Vd, Ta, Vn, Tb, shift);
3037   }
3038 
3039   void ushll2(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn,  SIMD_Arrangement Tb, int shift) {
3040     assert(Tb == T16B || Tb == T8H || Tb == T4S, "invalid arrangement");
3041     _xshll(/* is_unsigned */ true, Vd, Ta, Vn, Tb, shift);
3042   }
3043 
3044   void uxtl(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn,  SIMD_Arrangement Tb) {
3045     ushll(Vd, Ta, Vn, Tb, 0);
3046   }
3047 
3048   void sshll(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn,  SIMD_Arrangement Tb, int shift) {
3049     assert(Tb == T8B || Tb == T4H || Tb == T2S, "invalid arrangement");
3050     _xshll(/* is_unsigned */ false, Vd, Ta, Vn, Tb, shift);
3051   }
3052 
3053   void sshll2(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn,  SIMD_Arrangement Tb, int shift) {
3054     assert(Tb == T16B || Tb == T8H || Tb == T4S, "invalid arrangement");
3055     _xshll(/* is_unsigned */ false, Vd, Ta, Vn, Tb, shift);
3056   }
3057 
3058   void sxtl(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn,  SIMD_Arrangement Tb) {
3059     sshll(Vd, Ta, Vn, Tb, 0);
3060   }
3061 
3062   // Move from general purpose register
3063   //   mov  Vd.T[index], Rn
3064   void mov(FloatRegister Vd, SIMD_RegVariant T, int index, Register Xn) {
3065     guarantee(T != Q, "invalid register variant");
3066     starti;
3067     f(0b01001110000, 31, 21), f(((1 << T) | (index << (T + 1))), 20, 16);
3068     f(0b000111, 15, 10), zrf(Xn, 5), rf(Vd, 0);
3069   }
3070 
3071   // Move to general purpose register
3072   //   mov  Rd, Vn.T[index]
3073   void mov(Register Xd, FloatRegister Vn, SIMD_RegVariant T, int index) {
3074     guarantee(T == S || T == D, "invalid register variant");
3075     umov(Xd, Vn, T, index);
3076   }
3077 
3078  protected:
3079   void _xaddwv(bool is_unsigned, FloatRegister Vd, FloatRegister Vn, SIMD_Arrangement Ta,
3080                FloatRegister Vm, SIMD_Arrangement Tb) {
3081     starti;
3082     assert((Tb >> 1) + 1 == (Ta >> 1), "Incompatible arrangement");
3083     f(0, 31), f((int)Tb & 1, 30), f(is_unsigned ? 1 : 0, 29), f(0b01110, 28, 24);
3084     f((int)(Ta >> 1) - 1, 23, 22), f(1, 21), rf(Vm, 16), f(0b000100, 15, 10), rf(Vn, 5), rf(Vd, 0);
3085   }
3086 
3087  public:
3088 #define INSN(NAME, assertion, is_unsigned)                              \
3089   void NAME(FloatRegister Vd, FloatRegister Vn, SIMD_Arrangement Ta, FloatRegister Vm, \
3090               SIMD_Arrangement Tb) {                                    \
3091     assert((assertion), "invalid arrangement");                         \
3092     _xaddwv(is_unsigned, Vd, Vn, Ta, Vm, Tb);                           \
3093   }
3094 
3095 public:
3096 
3097   INSN(uaddwv,  Tb == T8B || Tb == T4H || Tb == T2S,  /*is_unsigned*/true)
3098   INSN(uaddwv2, Tb == T16B || Tb == T8H || Tb == T4S, /*is_unsigned*/true)
3099   INSN(saddwv,  Tb == T8B || Tb == T4H || Tb == T2S,  /*is_unsigned*/false)
3100   INSN(saddwv2, Tb == T16B || Tb == T8H || Tb == T4S, /*is_unsigned*/false)
3101 
3102 #undef INSN
3103 
3104 
3105 private:
3106   void _pmull(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn, FloatRegister Vm, SIMD_Arrangement Tb) {
3107     starti;
3108     assert((Ta == T1Q && (Tb == T1D || Tb == T2D)) ||
3109            (Ta == T8H && (Tb == T8B || Tb == T16B)), "Invalid Size specifier");
3110     int size = (Ta == T1Q) ? 0b11 : 0b00;
3111     f(0, 31), f(Tb & 1, 30), f(0b001110, 29, 24), f(size, 23, 22);
3112     f(1, 21), rf(Vm, 16), f(0b111000, 15, 10), rf(Vn, 5), rf(Vd, 0);
3113   }
3114 
3115 public:
3116   void pmull(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn, FloatRegister Vm, SIMD_Arrangement Tb) {
3117     assert(Tb == T1D || Tb == T8B, "pmull assumes T1D or T8B as the second size specifier");
3118     _pmull(Vd, Ta, Vn, Vm, Tb);
3119   }
3120 
3121   void pmull2(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn, FloatRegister Vm, SIMD_Arrangement Tb) {
3122     assert(Tb == T2D || Tb == T16B, "pmull2 assumes T2D or T16B as the second size specifier");
3123     _pmull(Vd, Ta, Vn, Vm, Tb);
3124   }
3125 
3126   void uqxtn(FloatRegister Vd, SIMD_Arrangement Tb, FloatRegister Vn, SIMD_Arrangement Ta) {
3127     starti;
3128     int size_b = (int)Tb >> 1;
3129     int size_a = (int)Ta >> 1;
3130     assert(size_b < 3 && size_b == size_a - 1, "Invalid size specifier");
3131     f(0, 31), f(Tb & 1, 30), f(0b101110, 29, 24), f(size_b, 23, 22);
3132     f(0b100001010010, 21, 10), rf(Vn, 5), rf(Vd, 0);
3133   }
3134 
3135   void xtn(FloatRegister Vd, SIMD_Arrangement Tb, FloatRegister Vn, SIMD_Arrangement Ta) {
3136     starti;
3137     int size_b = (int)Tb >> 1;
3138     int size_a = (int)Ta >> 1;
3139     assert(size_b < 3 && size_b == size_a - 1, "Invalid size specifier");
3140     f(0, 31), f(Tb & 1, 30), f(0b001110, 29, 24), f(size_b, 23, 22);
3141     f(0b100001001010, 21, 10), rf(Vn, 5), rf(Vd, 0);
3142   }
3143 
3144   void dup(FloatRegister Vd, SIMD_Arrangement T, Register Xs)
3145   {
3146     starti;
3147     assert(T != T1D, "reserved encoding");
3148     f(0,31), f((int)T & 1, 30), f(0b001110000, 29, 21);
3149     f((1 << (T >> 1)), 20, 16), f(0b000011, 15, 10), zrf(Xs, 5), rf(Vd, 0);
3150   }
3151 
3152   void dup(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, int index = 0)
3153   {
3154     starti;
3155     assert(T != T1D, "reserved encoding");
3156     f(0, 31), f((int)T & 1, 30), f(0b001110000, 29, 21);
3157     f(((1 << (T >> 1)) | (index << ((T >> 1) + 1))), 20, 16);
3158     f(0b000001, 15, 10), rf(Vn, 5), rf(Vd, 0);
3159   }
3160 
3161   // Advanced SIMD scalar copy
3162   void dup(FloatRegister Vd, SIMD_RegVariant T, FloatRegister Vn, int index = 0)
3163   {
3164     starti;
3165     assert(T != Q, "invalid size");
3166     f(0b01011110000, 31, 21);
3167     f((1 << T) | (index << (T + 1)), 20, 16);
3168     f(0b000001, 15, 10), rf(Vn, 5), rf(Vd, 0);
3169   }
3170 
3171   // AdvSIMD ZIP/UZP/TRN
3172 #define INSN(NAME, opcode)                                              \
3173   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \
3174     guarantee(T != T1D && T != T1Q, "invalid arrangement");             \
3175     starti;                                                             \
3176     f(0, 31), f(0b001110, 29, 24), f(0, 21), f(0, 15);                  \
3177     f(opcode, 14, 12), f(0b10, 11, 10);                                 \
3178     rf(Vm, 16), rf(Vn, 5), rf(Vd, 0);                                   \
3179     f(T & 1, 30), f(T >> 1, 23, 22);                                    \
3180   }
3181 
3182   INSN(uzp1, 0b001);
3183   INSN(trn1, 0b010);
3184   INSN(zip1, 0b011);
3185   INSN(uzp2, 0b101);
3186   INSN(trn2, 0b110);
3187   INSN(zip2, 0b111);
3188 
3189 #undef INSN
3190 
3191   // CRC32 instructions
3192 #define INSN(NAME, c, sf, sz)                                             \
3193   void NAME(Register Rd, Register Rn, Register Rm) {                      \
3194     starti;                                                               \
3195     f(sf, 31), f(0b0011010110, 30, 21), f(0b010, 15, 13), f(c, 12);       \
3196     f(sz, 11, 10), rf(Rm, 16), rf(Rn, 5), rf(Rd, 0);                      \
3197   }
3198 
3199   INSN(crc32b,  0, 0, 0b00);
3200   INSN(crc32h,  0, 0, 0b01);
3201   INSN(crc32w,  0, 0, 0b10);
3202   INSN(crc32x,  0, 1, 0b11);
3203   INSN(crc32cb, 1, 0, 0b00);
3204   INSN(crc32ch, 1, 0, 0b01);
3205   INSN(crc32cw, 1, 0, 0b10);
3206   INSN(crc32cx, 1, 1, 0b11);
3207 
3208 #undef INSN
3209 
3210   // Table vector lookup
3211 #define INSN(NAME, op)                                                  \
3212   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, unsigned registers, FloatRegister Vm) { \
3213     starti;                                                             \
3214     assert(T == T8B || T == T16B, "invalid arrangement");               \
3215     assert(0 < registers && registers <= 4, "invalid number of registers"); \
3216     f(0, 31), f((int)T & 1, 30), f(0b001110000, 29, 21), rf(Vm, 16), f(0, 15); \
3217     f(registers - 1, 14, 13), f(op, 12),f(0b00, 11, 10), rf(Vn, 5), rf(Vd, 0); \
3218   }
3219 
3220   INSN(tbl, 0);
3221   INSN(tbx, 1);
3222 
3223 #undef INSN
3224 
3225   // AdvSIMD two-reg misc
3226   // In this instruction group, the 2 bits in the size field ([23:22]) may be
3227   // fixed or determined by the "SIMD_Arrangement T", or both. The additional
3228   // parameter "tmask" is a 2-bit mask used to indicate which bits in the size
3229   // field are determined by the SIMD_Arrangement. The bit of "tmask" should be
3230   // set to 1 if corresponding bit marked as "x" in the ArmARM.
3231 #define INSN(NAME, U, size, tmask, opcode)                                          \
3232   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {               \
3233        starti;                                                                      \
3234        assert((ASSERTION), MSG);                                                    \
3235        f(0, 31), f((int)T & 1, 30), f(U, 29), f(0b01110, 28, 24);                   \
3236        f(size | ((int)(T >> 1) & tmask), 23, 22), f(0b10000, 21, 17);               \
3237        f(opcode, 16, 12), f(0b10, 11, 10), rf(Vn, 5), rf(Vd, 0);                    \
3238  }
3239 
3240 #define MSG "invalid arrangement"
3241 
3242 #define ASSERTION (T == T2S || T == T4S || T == T2D)
3243   INSN(fsqrt,  1, 0b10, 0b01, 0b11111);
3244   INSN(fabs,   0, 0b10, 0b01, 0b01111);
3245   INSN(fneg,   1, 0b10, 0b01, 0b01111);
3246   INSN(frintn, 0, 0b00, 0b01, 0b11000);
3247   INSN(frintm, 0, 0b00, 0b01, 0b11001);
3248   INSN(frintp, 0, 0b10, 0b01, 0b11000);
3249   INSN(fcvtas, 0, 0b00, 0b01, 0b11100);
3250   INSN(fcvtzs, 0, 0b10, 0b01, 0b11011);
3251   INSN(fcvtms, 0, 0b00, 0b01, 0b11011);
3252 #undef ASSERTION
3253 
3254 #define ASSERTION (T == T8B || T == T16B || T == T4H || T == T8H || T == T2S || T == T4S)
3255   INSN(rev64, 0, 0b00, 0b11, 0b00000);
3256 #undef ASSERTION
3257 
3258 #define ASSERTION (T == T8B || T == T16B || T == T4H || T == T8H)
3259   INSN(rev32, 1, 0b00, 0b11, 0b00000);
3260 #undef ASSERTION
3261 
3262 #define ASSERTION (T == T8B || T == T16B)
3263   INSN(rev16, 0, 0b00, 0b11, 0b00001);
3264   INSN(rbit,  1, 0b01, 0b00, 0b00101);
3265 #undef ASSERTION
3266 
3267 #undef MSG
3268 
3269 #undef INSN
3270 
3271   // AdvSIMD compare with zero (vector)
3272   void cm(Condition cond, FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {
3273     starti;
3274     assert(T != T1Q && T != T1D, "invalid arrangement");
3275     int cond_op;
3276     switch (cond) {
3277       case EQ: cond_op = 0b001; break;
3278       case GE: cond_op = 0b100; break;
3279       case GT: cond_op = 0b000; break;
3280       case LE: cond_op = 0b101; break;
3281       case LT: cond_op = 0b010; break;
3282       default:
3283         ShouldNotReachHere();
3284         break;
3285     }
3286 
3287     f(0, 31), f((int)T & 1, 30), f((cond_op >> 2) & 1, 29);
3288     f(0b01110, 28, 24), f((int)T >> 1, 23, 22), f(0b10000010, 21, 14);
3289     f(cond_op & 0b11, 13, 12), f(0b10, 11, 10), rf(Vn, 5), rf(Vd, 0);
3290   }
3291 
3292   // AdvSIMD Floating-point compare with zero (vector)
3293   void fcm(Condition cond, FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {
3294     starti;
3295     assert(T == T2S || T == T4S || T == T2D, "invalid arrangement");
3296     int cond_op;
3297     switch (cond) {
3298       case EQ: cond_op = 0b010; break;
3299       case GT: cond_op = 0b000; break;
3300       case GE: cond_op = 0b001; break;
3301       case LE: cond_op = 0b011; break;
3302       case LT: cond_op = 0b100; break;
3303       default:
3304         ShouldNotReachHere();
3305         break;
3306     }
3307 
3308     f(0, 31), f((int)T & 1, 30), f(cond_op & 1, 29), f(0b011101, 28, 23);
3309     f(((int)(T >> 1) & 1), 22), f(0b10000011, 21, 14);
3310     f((cond_op >> 1) & 0b11, 13, 12), f(0b10, 11, 10), rf(Vn, 5), rf(Vd, 0);
3311   }
3312 
3313   void ext(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm, int index)
3314   {
3315     starti;
3316     assert(T == T8B || T == T16B, "invalid arrangement");
3317     assert((T == T8B && index <= 0b0111) || (T == T16B && index <= 0b1111), "Invalid index value");
3318     f(0, 31), f((int)T & 1, 30), f(0b101110000, 29, 21);
3319     rf(Vm, 16), f(0, 15), f(index, 14, 11);
3320     f(0, 10), rf(Vn, 5), rf(Vd, 0);
3321   }
3322 
3323 // SVE arithmetic - unpredicated
3324 #define INSN(NAME, opcode)                                                             \
3325   void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, FloatRegister Zm) { \
3326     starti;                                                                            \
3327     assert(T != Q, "invalid register variant");                                        \
3328     f(0b00000100, 31, 24), f(T, 23, 22), f(1, 21),                                     \
3329     rf(Zm, 16), f(0, 15, 13), f(opcode, 12, 10), rf(Zn, 5), rf(Zd, 0);                 \
3330   }
3331   INSN(sve_add,   0b000);
3332   INSN(sve_sub,   0b001);
3333   INSN(sve_sqadd, 0b100);
3334   INSN(sve_sqsub, 0b110);
3335   INSN(sve_uqadd, 0b101);
3336   INSN(sve_uqsub, 0b111);
3337 #undef INSN
3338 
3339 // SVE integer add/subtract immediate (unpredicated)
3340 #define INSN(NAME, op)                                                  \
3341   void NAME(FloatRegister Zd, SIMD_RegVariant T, unsigned imm8) {       \
3342     starti;                                                             \
3343     /* The immediate is an unsigned value in the range 0 to 255, and    \
3344      * for element width of 16 bits or higher it may also be a          \
3345      * positive multiple of 256 in the range 256 to 65280.              \
3346      */                                                                 \
3347     assert(T != Q, "invalid size");                                     \
3348     int sh = 0;                                                         \
3349     if (imm8 <= 0xff) {                                                 \
3350       sh = 0;                                                           \
3351     } else if (T != B && imm8 <= 0xff00 && (imm8 & 0xff) == 0) {        \
3352       sh = 1;                                                           \
3353       imm8 = (imm8 >> 8);                                               \
3354     } else {                                                            \
3355       guarantee(false, "invalid immediate");                            \
3356     }                                                                   \
3357     f(0b00100101, 31, 24), f(T, 23, 22), f(0b10000, 21, 17);            \
3358     f(op, 16, 14), f(sh, 13), f(imm8, 12, 5), rf(Zd, 0);                \
3359   }
3360 
3361   INSN(sve_add, 0b011);
3362   INSN(sve_sub, 0b111);
3363 #undef INSN
3364 
3365 // SVE floating-point arithmetic - unpredicated
3366 #define INSN(NAME, opcode)                                                             \
3367   void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, FloatRegister Zm) { \
3368     starti;                                                                            \
3369     assert(T == S || T == D, "invalid register variant");                              \
3370     f(0b01100101, 31, 24), f(T, 23, 22), f(0, 21),                                     \
3371     rf(Zm, 16), f(0, 15, 13), f(opcode, 12, 10), rf(Zn, 5), rf(Zd, 0);                 \
3372   }
3373 
3374   INSN(sve_fadd, 0b000);
3375   INSN(sve_fmul, 0b010);
3376   INSN(sve_fsub, 0b001);
3377 #undef INSN
3378 
3379 private:
3380   void sve_predicate_reg_insn(unsigned op24, unsigned op13,
3381                               FloatRegister Zd_or_Vd, SIMD_RegVariant T,
3382                               PRegister Pg, FloatRegister Zn_or_Vn) {
3383     starti;
3384     f(op24, 31, 24), f(T, 23, 22), f(op13, 21, 13);
3385     pgrf(Pg, 10), rf(Zn_or_Vn, 5), rf(Zd_or_Vd, 0);
3386   }
3387 
3388   void sve_shift_imm_encoding(SIMD_RegVariant T, int shift, bool isSHR,
3389                               int& tszh, int& tszl_imm) {
3390     /* The encodings for the tszh:tszl:imm3 fields
3391      * for shift right is calculated as:
3392      *   0001 xxx       B, shift = 16  - UInt(tszh:tszl:imm3)
3393      *   001x xxx       H, shift = 32  - UInt(tszh:tszl:imm3)
3394      *   01xx xxx       S, shift = 64  - UInt(tszh:tszl:imm3)
3395      *   1xxx xxx       D, shift = 128 - UInt(tszh:tszl:imm3)
3396      * for shift left is calculated as:
3397      *   0001 xxx       B, shift = UInt(tszh:tszl:imm3) - 8
3398      *   001x xxx       H, shift = UInt(tszh:tszl:imm3) - 16
3399      *   01xx xxx       S, shift = UInt(tszh:tszl:imm3) - 32
3400      *   1xxx xxx       D, shift = UInt(tszh:tszl:imm3) - 64
3401      */
3402     assert(T != Q, "Invalid register variant");
3403     if (isSHR) {
3404       assert(((1 << (T + 3)) >= shift) && (shift > 0) , "Invalid shift value");
3405     } else {
3406       assert(((1 << (T + 3)) > shift) && (shift >= 0) , "Invalid shift value");
3407     }
3408     int cVal = (1 << ((T + 3) + (isSHR ? 1 : 0)));
3409     int encodedShift = isSHR ? cVal - shift : cVal + shift;
3410     tszh = encodedShift >> 5;
3411     tszl_imm = encodedShift & 0x1f;
3412   }
3413 
3414 public:
3415 
3416 // SVE integer arithmetic - predicate
3417 #define INSN(NAME, op1, op2)                                                                            \
3418   void NAME(FloatRegister Zdn_or_Zd_or_Vd, SIMD_RegVariant T, PRegister Pg, FloatRegister Znm_or_Vn) {  \
3419     assert(T != Q, "invalid register variant");                                                         \
3420     sve_predicate_reg_insn(op1, op2, Zdn_or_Zd_or_Vd, T, Pg, Znm_or_Vn);                                \
3421   }
3422 
3423   INSN(sve_abs,   0b00000100, 0b010110101); // vector abs, unary
3424   INSN(sve_add,   0b00000100, 0b000000000); // vector add
3425   INSN(sve_and,   0b00000100, 0b011010000); // vector and
3426   INSN(sve_andv,  0b00000100, 0b011010001); // bitwise and reduction to scalar
3427   INSN(sve_asr,   0b00000100, 0b010000100); // vector arithmetic shift right
3428   INSN(sve_bic,   0b00000100, 0b011011000); // vector bitwise clear
3429   INSN(sve_clz,   0b00000100, 0b011001101); // vector count leading zero bits
3430   INSN(sve_cnt,   0b00000100, 0b011010101); // count non-zero bits
3431   INSN(sve_cpy,   0b00000101, 0b100000100); // copy scalar to each active vector element
3432   INSN(sve_eor,   0b00000100, 0b011001000); // vector eor
3433   INSN(sve_eorv,  0b00000100, 0b011001001); // bitwise xor reduction to scalar
3434   INSN(sve_lsl,   0b00000100, 0b010011100); // vector logical shift left
3435   INSN(sve_lsr,   0b00000100, 0b010001100); // vector logical shift right
3436   INSN(sve_mul,   0b00000100, 0b010000000); // vector mul
3437   INSN(sve_neg,   0b00000100, 0b010111101); // vector neg, unary
3438   INSN(sve_not,   0b00000100, 0b011110101); // bitwise invert vector, unary
3439   INSN(sve_orr,   0b00000100, 0b011000000); // vector or
3440   INSN(sve_orv,   0b00000100, 0b011000001); // bitwise or reduction to scalar
3441   INSN(sve_smax,  0b00000100, 0b001000000); // signed maximum vectors
3442   INSN(sve_smaxv, 0b00000100, 0b001000001); // signed maximum reduction to scalar
3443   INSN(sve_smin,  0b00000100, 0b001010000); // signed minimum vectors
3444   INSN(sve_sminv, 0b00000100, 0b001010001); // signed minimum reduction to scalar
3445   INSN(sve_sub,   0b00000100, 0b000001000); // vector sub
3446   INSN(sve_uaddv, 0b00000100, 0b000001001); // unsigned add reduction to scalar
3447   INSN(sve_umax,  0b00000100, 0b001001000); // unsigned maximum vectors
3448   INSN(sve_umin,  0b00000100, 0b001011000); // unsigned minimum vectors
3449 #undef INSN
3450 
3451 // SVE floating-point arithmetic - predicate
3452 #define INSN(NAME, op1, op2)                                                                          \
3453   void NAME(FloatRegister Zd_or_Zdn_or_Vd, SIMD_RegVariant T, PRegister Pg, FloatRegister Zn_or_Zm) { \
3454     assert(T == S || T == D, "invalid register variant");                                             \
3455     sve_predicate_reg_insn(op1, op2, Zd_or_Zdn_or_Vd, T, Pg, Zn_or_Zm);                               \
3456   }
3457 
3458   INSN(sve_fabd,   0b01100101, 0b001000100); // floating-point absolute difference
3459   INSN(sve_fabs,   0b00000100, 0b011100101);
3460   INSN(sve_fadd,   0b01100101, 0b000000100);
3461   INSN(sve_fadda,  0b01100101, 0b011000001); // add strictly-ordered reduction to scalar Vd
3462   INSN(sve_fdiv,   0b01100101, 0b001101100);
3463   INSN(sve_fmax,   0b01100101, 0b000110100); // floating-point maximum
3464   INSN(sve_fmaxv,  0b01100101, 0b000110001); // floating-point maximum recursive reduction to scalar
3465   INSN(sve_fmin,   0b01100101, 0b000111100); // floating-point minimum
3466   INSN(sve_fminv,  0b01100101, 0b000111001); // floating-point minimum recursive reduction to scalar
3467   INSN(sve_fmul,   0b01100101, 0b000010100);
3468   INSN(sve_fneg,   0b00000100, 0b011101101);
3469   INSN(sve_frintm, 0b01100101, 0b000010101); // floating-point round to integral value, toward minus infinity
3470   INSN(sve_frintn, 0b01100101, 0b000000101); // floating-point round to integral value, nearest with ties to even
3471   INSN(sve_frinta, 0b01100101, 0b000100101); // floating-point round to integral value, nearest with ties to away
3472   INSN(sve_frintp, 0b01100101, 0b000001101); // floating-point round to integral value, toward plus infinity
3473   INSN(sve_fsqrt,  0b01100101, 0b001101101);
3474   INSN(sve_fsub,   0b01100101, 0b000001100);
3475 #undef INSN
3476 
3477   // SVE multiple-add/sub - predicated
3478 #define INSN(NAME, op0, op1, op2)                                                                     \
3479   void NAME(FloatRegister Zda, SIMD_RegVariant T, PRegister Pg, FloatRegister Zn, FloatRegister Zm) { \
3480     starti;                                                                                           \
3481     assert(T != Q, "invalid size");                                                                   \
3482     f(op0, 31, 24), f(T, 23, 22), f(op1, 21), rf(Zm, 16);                                             \
3483     f(op2, 15, 13), pgrf(Pg, 10), rf(Zn, 5), rf(Zda, 0);                                              \
3484   }
3485 
3486   INSN(sve_fmla,  0b01100101, 1, 0b000); // floating-point fused multiply-add, writing addend: Zda = Zda + Zn * Zm
3487   INSN(sve_fmls,  0b01100101, 1, 0b001); // floating-point fused multiply-subtract: Zda = Zda + -Zn * Zm
3488   INSN(sve_fnmla, 0b01100101, 1, 0b010); // floating-point negated fused multiply-add: Zda = -Zda + -Zn * Zm
3489   INSN(sve_fnmls, 0b01100101, 1, 0b011); // floating-point negated fused multiply-subtract: Zda = -Zda + Zn * Zm
3490   INSN(sve_fmad,  0b01100101, 1, 0b100); // floating-point fused multiply-add, writing multiplicand: Zda = Zm + Zda * Zn
3491   INSN(sve_fmsb,  0b01100101, 1, 0b101); // floating-point fused multiply-subtract, writing multiplicand: Zda = Zm + -Zda * Zn
3492   INSN(sve_fnmad, 0b01100101, 1, 0b110); // floating-point negated fused multiply-add, writing multiplicand: Zda = -Zm + -Zda * Zn
3493   INSN(sve_fnmsb, 0b01100101, 1, 0b111); // floating-point negated fused multiply-subtract, writing multiplicand: Zda = -Zm + Zda * Zn
3494   INSN(sve_mla,   0b00000100, 0, 0b010); // multiply-add, writing addend: Zda = Zda + Zn*Zm
3495   INSN(sve_mls,   0b00000100, 0, 0b011); // multiply-subtract, writing addend: Zda = Zda + -Zn*Zm
3496 #undef INSN
3497 
3498 // SVE bitwise logical - unpredicated
3499 #define INSN(NAME, opc)                                              \
3500   void NAME(FloatRegister Zd, FloatRegister Zn, FloatRegister Zm) {  \
3501     starti;                                                          \
3502     f(0b00000100, 31, 24), f(opc, 23, 22), f(1, 21),                 \
3503     rf(Zm, 16), f(0b001100, 15, 10), rf(Zn, 5), rf(Zd, 0);           \
3504   }
3505   INSN(sve_and, 0b00);
3506   INSN(sve_eor, 0b10);
3507   INSN(sve_orr, 0b01);
3508   INSN(sve_bic, 0b11);
3509 #undef INSN
3510 
3511 // SVE bitwise logical with immediate (unpredicated)
3512 #define INSN(NAME, opc)                                                      \
3513   void NAME(FloatRegister Zd, SIMD_RegVariant T, uint64_t imm) {             \
3514     starti;                                                                  \
3515     unsigned elembits = regVariant_to_elemBits(T);                           \
3516     uint32_t val = encode_sve_logical_immediate(elembits, imm);              \
3517     f(0b00000101, 31, 24), f(opc, 23, 22), f(0b0000, 21, 18);                \
3518     f(val, 17, 5), rf(Zd, 0);                                                \
3519   }
3520   INSN(sve_and, 0b10);
3521   INSN(sve_eor, 0b01);
3522   INSN(sve_orr, 0b00);
3523 #undef INSN
3524 
3525 // SVE shift immediate - unpredicated
3526 #define INSN(NAME, opc, isSHR)                                                  \
3527   void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, int shift) { \
3528     starti;                                                                     \
3529     int tszh, tszl_imm;                                                         \
3530     sve_shift_imm_encoding(T, shift, isSHR, tszh, tszl_imm);                    \
3531     f(0b00000100, 31, 24);                                                      \
3532     f(tszh, 23, 22), f(1,21), f(tszl_imm, 20, 16);                              \
3533     f(0b100, 15, 13), f(opc, 12, 10), rf(Zn, 5), rf(Zd, 0);                     \
3534   }
3535 
3536   INSN(sve_asr, 0b100, /* isSHR = */ true);
3537   INSN(sve_lsl, 0b111, /* isSHR = */ false);
3538   INSN(sve_lsr, 0b101, /* isSHR = */ true);
3539 #undef INSN
3540 
3541 // SVE bitwise shift by immediate (predicated)
3542 #define INSN(NAME, opc, isSHR)                                                  \
3543   void NAME(FloatRegister Zdn, SIMD_RegVariant T, PRegister Pg, int shift) {    \
3544     starti;                                                                     \
3545     int tszh, tszl_imm;                                                         \
3546     sve_shift_imm_encoding(T, shift, isSHR, tszh, tszl_imm);                    \
3547     f(0b00000100, 31, 24), f(tszh, 23, 22), f(0b00, 21, 20), f(opc, 19, 16);    \
3548     f(0b100, 15, 13), pgrf(Pg, 10), f(tszl_imm, 9, 5), rf(Zdn, 0);              \
3549   }
3550 
3551   INSN(sve_asr, 0b0000, /* isSHR = */ true);
3552   INSN(sve_lsl, 0b0011, /* isSHR = */ false);
3553   INSN(sve_lsr, 0b0001, /* isSHR = */ true);
3554 #undef INSN
3555 
3556 private:
3557 
3558   // Scalar base + immediate index
3559   void sve_ld_st1(FloatRegister Zt, Register Xn, int imm, PRegister Pg,
3560               SIMD_RegVariant T, int op1, int type, int op2) {
3561     starti;
3562     assert_cond(T >= type);
3563     f(op1, 31, 25), f(type, 24, 23), f(T, 22, 21);
3564     f(0, 20), sf(imm, 19, 16), f(op2, 15, 13);
3565     pgrf(Pg, 10), srf(Xn, 5), rf(Zt, 0);
3566   }
3567 
3568   // Scalar base + scalar index
3569   void sve_ld_st1(FloatRegister Zt, Register Xn, Register Xm, PRegister Pg,
3570               SIMD_RegVariant T, int op1, int type, int op2) {
3571     starti;
3572     assert_cond(T >= type);
3573     f(op1, 31, 25), f(type, 24, 23), f(T, 22, 21);
3574     rf(Xm, 16), f(op2, 15, 13);
3575     pgrf(Pg, 10), srf(Xn, 5), rf(Zt, 0);
3576   }
3577 
3578   void sve_ld_st1(FloatRegister Zt, PRegister Pg,
3579               SIMD_RegVariant T, const Address &a,
3580               int op1, int type, int imm_op2, int scalar_op2) {
3581     switch (a.getMode()) {
3582     case Address::base_plus_offset:
3583       sve_ld_st1(Zt, a.base(), checked_cast<int>(a.offset()), Pg, T, op1, type, imm_op2);
3584       break;
3585     case Address::base_plus_offset_reg:
3586       sve_ld_st1(Zt, a.base(), a.index(), Pg, T, op1, type, scalar_op2);
3587       break;
3588     default:
3589       ShouldNotReachHere();
3590     }
3591   }
3592 
3593 public:
3594 
3595 // SVE contiguous load/store
3596 #define INSN(NAME, op1, type, imm_op2, scalar_op2)                                   \
3597   void NAME(FloatRegister Zt, SIMD_RegVariant T, PRegister Pg, const Address &a) {   \
3598     assert(T != Q, "invalid register variant");                                      \
3599     sve_ld_st1(Zt, Pg, T, a, op1, type, imm_op2, scalar_op2);                        \
3600   }
3601 
3602   INSN(sve_ld1b, 0b1010010, 0b00, 0b101, 0b010);
3603   INSN(sve_st1b, 0b1110010, 0b00, 0b111, 0b010);
3604   INSN(sve_ld1h, 0b1010010, 0b01, 0b101, 0b010);
3605   INSN(sve_st1h, 0b1110010, 0b01, 0b111, 0b010);
3606   INSN(sve_ld1w, 0b1010010, 0b10, 0b101, 0b010);
3607   INSN(sve_st1w, 0b1110010, 0b10, 0b111, 0b010);
3608   INSN(sve_ld1d, 0b1010010, 0b11, 0b101, 0b010);
3609   INSN(sve_st1d, 0b1110010, 0b11, 0b111, 0b010);
3610 #undef INSN
3611 
3612 // Gather/scatter load/store (SVE) - scalar plus vector
3613 #define INSN(NAME, op1, type, op2, op3)                                         \
3614   void NAME(FloatRegister Zt, PRegister Pg, Register Xn, FloatRegister Zm) {    \
3615     starti;                                                                     \
3616     f(op1, 31, 25), f(type, 24, 23), f(op2, 22, 21), rf(Zm, 16);                \
3617     f(op3, 15, 13), pgrf(Pg, 10), srf(Xn, 5), rf(Zt, 0);                        \
3618   }
3619   // SVE 32-bit gather load words (scalar plus 32-bit scaled offsets)
3620   INSN(sve_ld1w_gather,  0b1000010, 0b10, 0b01, 0b010);
3621   // SVE 64-bit gather load (scalar plus 32-bit unpacked scaled offsets)
3622   INSN(sve_ld1d_gather,  0b1100010, 0b11, 0b01, 0b010);
3623   // SVE 32-bit scatter store (scalar plus 32-bit scaled offsets)
3624   INSN(sve_st1w_scatter, 0b1110010, 0b10, 0b11, 0b100);
3625   // SVE 64-bit scatter store (scalar plus unpacked 32-bit scaled offsets)
3626   INSN(sve_st1d_scatter, 0b1110010, 0b11, 0b01, 0b100);
3627 #undef INSN
3628 
3629 // SVE load/store - unpredicated
3630 #define INSN(NAME, op1)                                                         \
3631   void NAME(FloatRegister Zt, const Address &a)  {                              \
3632     starti;                                                                     \
3633     assert(a.index() == noreg, "invalid address variant");                      \
3634     f(op1, 31, 29), f(0b0010110, 28, 22), sf(a.offset() >> 3, 21, 16),          \
3635     f(0b010, 15, 13), f(a.offset() & 0x7, 12, 10), srf(a.base(), 5), rf(Zt, 0); \
3636   }
3637 
3638   INSN(sve_ldr, 0b100); // LDR (vector)
3639   INSN(sve_str, 0b111); // STR (vector)
3640 #undef INSN
3641 
3642 // SVE stack frame adjustment
3643 #define INSN(NAME, op) \
3644   void NAME(Register Xd, Register Xn, int imm6) {                 \
3645     starti;                                                       \
3646     f(0b000001000, 31, 23), f(op, 22, 21);                        \
3647     srf(Xn, 16), f(0b01010, 15, 11), sf(imm6, 10, 5), srf(Xd, 0); \
3648   }
3649 
3650   INSN(sve_addvl, 0b01); // Add multiple of vector register size to scalar register
3651   INSN(sve_addpl, 0b11); // Add multiple of predicate register size to scalar register
3652 #undef INSN
3653 
3654 // SVE inc/dec register by element count
3655 #define INSN(NAME, op) \
3656   void NAME(Register Xdn, SIMD_RegVariant T, unsigned imm4 = 1, int pattern = 0b11111) { \
3657     starti;                                                                              \
3658     assert(T != Q, "invalid size");                                                      \
3659     f(0b00000100,31, 24), f(T, 23, 22), f(0b11, 21, 20);                                 \
3660     f(imm4 - 1, 19, 16), f(0b11100, 15, 11), f(op, 10), f(pattern, 9, 5), rf(Xdn, 0);    \
3661   }
3662 
3663   INSN(sve_inc, 0);
3664   INSN(sve_dec, 1);
3665 #undef INSN
3666 
3667 // SVE predicate logical operations
3668 #define INSN(NAME, op1, op2, op3) \
3669   void NAME(PRegister Pd, PRegister Pg, PRegister Pn, PRegister Pm) { \
3670     starti;                                                           \
3671     f(0b00100101, 31, 24), f(op1, 23, 22), f(0b00, 21, 20);           \
3672     prf(Pm, 16), f(0b01, 15, 14), prf(Pg, 10), f(op2, 9);             \
3673     prf(Pn, 5), f(op3, 4), prf(Pd, 0);                                \
3674   }
3675 
3676   INSN(sve_and,  0b00, 0b0, 0b0);
3677   INSN(sve_ands, 0b01, 0b0, 0b0);
3678   INSN(sve_eor,  0b00, 0b1, 0b0);
3679   INSN(sve_eors, 0b01, 0b1, 0b0);
3680   INSN(sve_orr,  0b10, 0b0, 0b0);
3681   INSN(sve_orrs, 0b11, 0b0, 0b0);
3682   INSN(sve_bic,  0b00, 0b0, 0b1);
3683 #undef INSN
3684 
3685   // SVE increment register by predicate count
3686   void sve_incp(const Register rd, SIMD_RegVariant T, PRegister pg) {
3687     starti;
3688     assert(T != Q, "invalid size");
3689     f(0b00100101, 31, 24), f(T, 23, 22), f(0b1011001000100, 21, 9),
3690     prf(pg, 5), rf(rd, 0);
3691   }
3692 
3693   // SVE broadcast general-purpose register to vector elements (unpredicated)
3694   void sve_dup(FloatRegister Zd, SIMD_RegVariant T, Register Rn) {
3695     starti;
3696     assert(T != Q, "invalid size");
3697     f(0b00000101, 31, 24), f(T, 23, 22), f(0b100000001110, 21, 10);
3698     srf(Rn, 5), rf(Zd, 0);
3699   }
3700 
3701   // SVE broadcast signed immediate to vector elements (unpredicated)
3702   void sve_dup(FloatRegister Zd, SIMD_RegVariant T, int imm8) {
3703     starti;
3704     assert(T != Q, "invalid size");
3705     int sh = 0;
3706     if (imm8 <= 127 && imm8 >= -128) {
3707       sh = 0;
3708     } else if (T != B && imm8 <= 32512 && imm8 >= -32768 && (imm8 & 0xff) == 0) {
3709       sh = 1;
3710       imm8 = (imm8 >> 8);
3711     } else {
3712       guarantee(false, "invalid immediate");
3713     }
3714     f(0b00100101, 31, 24), f(T, 23, 22), f(0b11100011, 21, 14);
3715     f(sh, 13), sf(imm8, 12, 5), rf(Zd, 0);
3716   }
3717 
3718   // SVE predicate test
3719   void sve_ptest(PRegister Pg, PRegister Pn) {
3720     starti;
3721     f(0b001001010101000011, 31, 14), prf(Pg, 10), f(0, 9), prf(Pn, 5), f(0, 4, 0);
3722   }
3723 
3724   // SVE predicate initialize
3725   void sve_ptrue(PRegister pd, SIMD_RegVariant esize, int pattern = 0b11111) {
3726     starti;
3727     f(0b00100101, 31, 24), f(esize, 23, 22), f(0b011000111000, 21, 10);
3728     f(pattern, 9, 5), f(0b0, 4), prf(pd, 0);
3729   }
3730 
3731   // SVE predicate zero
3732   void sve_pfalse(PRegister pd) {
3733     starti;
3734     f(0b00100101, 31, 24), f(0b00, 23, 22), f(0b011000111001, 21, 10);
3735     f(0b000000, 9, 4), prf(pd, 0);
3736   }
3737 
3738 // SVE load/store predicate register
3739 #define INSN(NAME, op1)                                                  \
3740   void NAME(PRegister Pt, const Address &a)  {                           \
3741     starti;                                                              \
3742     assert(a.index() == noreg, "invalid address variant");               \
3743     f(op1, 31, 29), f(0b0010110, 28, 22), sf(a.offset() >> 3, 21, 16),   \
3744     f(0b000, 15, 13), f(a.offset() & 0x7, 12, 10), srf(a.base(), 5),     \
3745     f(0, 4), prf(Pt, 0);                                                 \
3746   }
3747 
3748   INSN(sve_ldr, 0b100); // LDR (predicate)
3749   INSN(sve_str, 0b111); // STR (predicate)
3750 #undef INSN
3751 
3752   // SVE move predicate register
3753   void sve_mov(PRegister Pd, PRegister Pn) {
3754     starti;
3755     f(0b001001011000, 31, 20), prf(Pn, 16), f(0b01, 15, 14), prf(Pn, 10);
3756     f(0, 9), prf(Pn, 5), f(0, 4), prf(Pd, 0);
3757   }
3758 
3759   // SVE copy general-purpose register to vector elements (predicated)
3760   void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, Register Rn) {
3761     starti;
3762     assert(T != Q, "invalid size");
3763     f(0b00000101, 31, 24), f(T, 23, 22), f(0b101000101, 21, 13);
3764     pgrf(Pg, 10), srf(Rn, 5), rf(Zd, 0);
3765   }
3766 
3767 private:
3768   void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, int imm8,
3769                bool isMerge, bool isFloat) {
3770     starti;
3771     assert(T != Q, "invalid size");
3772     int sh = 0;
3773     if (imm8 <= 127 && imm8 >= -128) {
3774       sh = 0;
3775     } else if (T != B && imm8 <= 32512 && imm8 >= -32768 && (imm8 & 0xff) == 0) {
3776       sh = 1;
3777       imm8 = (imm8 >> 8);
3778     } else {
3779       guarantee(false, "invalid immediate");
3780     }
3781     int m = isMerge ? 1 : 0;
3782     f(0b00000101, 31, 24), f(T, 23, 22), f(0b01, 21, 20);
3783     prf(Pg, 16), f(isFloat ? 1 : 0, 15), f(m, 14), f(sh, 13), sf(imm8, 12, 5), rf(Zd, 0);
3784   }
3785 
3786 public:
3787   // SVE copy signed integer immediate to vector elements (predicated)
3788   void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, int imm8, bool isMerge) {
3789     sve_cpy(Zd, T, Pg, imm8, isMerge, /*isFloat*/false);
3790   }
3791   // SVE copy floating-point immediate to vector elements (predicated)
3792   void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, double d) {
3793     sve_cpy(Zd, T, Pg, checked_cast<int8_t>(pack(d)), /*isMerge*/true, /*isFloat*/true);
3794   }
3795 
3796   // SVE conditionally select elements from two vectors
3797   void sve_sel(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg,
3798                FloatRegister Zn, FloatRegister Zm) {
3799     starti;
3800     assert(T != Q, "invalid size");
3801     f(0b00000101, 31, 24), f(T, 23, 22), f(0b1, 21), rf(Zm, 16);
3802     f(0b11, 15, 14), prf(Pg, 10), rf(Zn, 5), rf(Zd, 0);
3803   }
3804 
3805   // SVE Permute Vector - Extract
3806   void sve_ext(FloatRegister Zdn, FloatRegister Zm, int imm8) {
3807     starti;
3808     f(0b00000101001, 31, 21), f(imm8 >> 3, 20, 16), f(0b000, 15, 13);
3809     f(imm8 & 0b111, 12, 10), rf(Zm, 5), rf(Zdn, 0);
3810   }
3811 
3812 // SVE Integer/Floating-Point Compare - Vectors
3813 #define INSN(NAME, op1, op2, fp)  \
3814   void NAME(Condition cond, PRegister Pd, SIMD_RegVariant T, PRegister Pg,             \
3815             FloatRegister Zn, FloatRegister Zm) {                                      \
3816     starti;                                                                            \
3817     assert(T != Q, "invalid size");                                                    \
3818     bool is_absolute = op2 == 0b11;                                                    \
3819     if (fp == 1) {                                                                     \
3820       assert(T != B, "invalid size");                                                  \
3821       if (is_absolute) {                                                               \
3822         assert(cond == GT || cond == GE, "invalid condition for fac");                 \
3823       } else {                                                                         \
3824         assert(cond != HI && cond != HS, "invalid condition for fcm");                 \
3825       }                                                                                \
3826     }                                                                                  \
3827     int cond_op;                                                                       \
3828     switch(cond) {                                                                     \
3829       case EQ: cond_op = (op2 << 2) | 0b10; break;                                     \
3830       case NE: cond_op = (op2 << 2) | 0b11; break;                                     \
3831       case GE: cond_op = (op2 << 2) | (is_absolute ? 0b01 : 0b00); break;              \
3832       case GT: cond_op = (op2 << 2) | (is_absolute ? 0b11 : 0b01); break;              \
3833       case HI: cond_op = 0b0001; break;                                                \
3834       case HS: cond_op = 0b0000; break;                                                \
3835       default:                                                                         \
3836         ShouldNotReachHere();                                                          \
3837     }                                                                                  \
3838     f(op1, 31, 24), f(T, 23, 22), f(0, 21), rf(Zm, 16), f((cond_op >> 1) & 7, 15, 13); \
3839     pgrf(Pg, 10), rf(Zn, 5), f(cond_op & 1, 4), prf(Pd, 0);                            \
3840   }
3841 
3842   INSN(sve_cmp, 0b00100100, 0b10, 0); // Integer compare vectors
3843   INSN(sve_fcm, 0b01100101, 0b01, 1); // Floating-point compare vectors
3844   INSN(sve_fac, 0b01100101, 0b11, 1); // Floating-point absolute compare vectors
3845 #undef INSN
3846 
3847 private:
3848   // Convert Assembler::Condition to op encoding - used by sve integer compare encoding
3849   static int assembler_cond_to_sve_op(Condition cond, bool &is_unsigned) {
3850     if (cond == HI || cond == HS || cond == LO || cond == LS) {
3851       is_unsigned = true;
3852     } else {
3853       is_unsigned = false;
3854     }
3855 
3856     switch (cond) {
3857       case HI:
3858       case GT:
3859         return 0b0001;
3860       case HS:
3861       case GE:
3862         return 0b0000;
3863       case LO:
3864       case LT:
3865         return 0b0010;
3866       case LS:
3867       case LE:
3868         return 0b0011;
3869       case EQ:
3870         return 0b1000;
3871       case NE:
3872         return 0b1001;
3873       default:
3874         ShouldNotReachHere();
3875         return -1;
3876     }
3877   }
3878 
3879 public:
3880   // SVE Integer Compare - 5 bits signed imm and 7 bits unsigned imm
3881   void sve_cmp(Condition cond, PRegister Pd, SIMD_RegVariant T,
3882                PRegister Pg, FloatRegister Zn, int imm) {
3883     starti;
3884     assert(T != Q, "invalid size");
3885     bool is_unsigned = false;
3886     int cond_op = assembler_cond_to_sve_op(cond, is_unsigned);
3887     f(is_unsigned ? 0b00100100 : 0b00100101, 31, 24), f(T, 23, 22);
3888     f(is_unsigned ? 0b1 : 0b0, 21);
3889     if (is_unsigned) {
3890       f(imm, 20, 14), f((cond_op >> 1) & 0x1, 13);
3891     } else {
3892       sf(imm, 20, 16), f((cond_op >> 1) & 0x7, 15, 13);
3893     }
3894     pgrf(Pg, 10), rf(Zn, 5), f(cond_op & 0x1, 4), prf(Pd, 0);
3895   }
3896 
3897   // SVE Floating-point compare vector with zero
3898   void sve_fcm(Condition cond, PRegister Pd, SIMD_RegVariant T,
3899                PRegister Pg, FloatRegister Zn, double d) {
3900     starti;
3901     assert(T != Q, "invalid size");
3902     guarantee(d == 0.0, "invalid immediate");
3903     int cond_op;
3904     switch(cond) {
3905       case EQ: cond_op = 0b100; break;
3906       case GT: cond_op = 0b001; break;
3907       case GE: cond_op = 0b000; break;
3908       case LT: cond_op = 0b010; break;
3909       case LE: cond_op = 0b011; break;
3910       case NE: cond_op = 0b110; break;
3911       default:
3912         ShouldNotReachHere();
3913     }
3914     f(0b01100101, 31, 24), f(T, 23, 22), f(0b0100, 21, 18),
3915     f((cond_op >> 1) & 0x3, 17, 16), f(0b001, 15, 13),
3916     pgrf(Pg, 10), rf(Zn, 5);
3917     f(cond_op & 0x1, 4), prf(Pd, 0);
3918   }
3919 
3920 // SVE unpack vector elements
3921 protected:
3922   void _sve_xunpk(bool is_unsigned, bool is_high, FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn) {
3923     starti;
3924     assert(T != B && T != Q, "invalid size");
3925     f(0b00000101, 31, 24), f(T, 23, 22), f(0b1100, 21, 18);
3926     f(is_unsigned ? 1 : 0, 17), f(is_high ? 1 : 0, 16),
3927     f(0b001110, 15, 10), rf(Zn, 5), rf(Zd, 0);
3928   }
3929 
3930 public:
3931 #define INSN(NAME, is_unsigned, is_high)                                  \
3932   void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn) {      \
3933     _sve_xunpk(is_unsigned, is_high, Zd, T, Zn);                          \
3934   }
3935 
3936   INSN(sve_uunpkhi, true,  true ); // Unsigned unpack and extend half of vector - high half
3937   INSN(sve_uunpklo, true,  false); // Unsigned unpack and extend half of vector - low half
3938   INSN(sve_sunpkhi, false, true ); // Signed unpack and extend half of vector - high half
3939   INSN(sve_sunpklo, false, false); // Signed unpack and extend half of vector - low half
3940 #undef INSN
3941 
3942 // SVE unpack predicate elements
3943 #define INSN(NAME, op) \
3944   void NAME(PRegister Pd, PRegister Pn) { \
3945     starti;                                                          \
3946     f(0b000001010011000, 31, 17), f(op, 16), f(0b0100000, 15, 9);    \
3947     prf(Pn, 5), f(0b0, 4), prf(Pd, 0);                               \
3948   }
3949 
3950   INSN(sve_punpkhi, 0b1); // Unpack and widen high half of predicate
3951   INSN(sve_punpklo, 0b0); // Unpack and widen low half of predicate
3952 #undef INSN
3953 
3954 // SVE permute vector elements
3955 #define INSN(NAME, op) \
3956   void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, FloatRegister Zm) { \
3957     starti;                                                                            \
3958     assert(T != Q, "invalid size");                                                    \
3959     f(0b00000101, 31, 24), f(T, 23, 22), f(0b1, 21), rf(Zm, 16);                       \
3960     f(0b01101, 15, 11), f(op, 10), rf(Zn, 5), rf(Zd, 0);                               \
3961   }
3962 
3963   INSN(sve_uzp1, 0b0); // Concatenate even elements from two vectors
3964   INSN(sve_uzp2, 0b1); // Concatenate odd elements from two vectors
3965 #undef INSN
3966 
3967 // SVE permute predicate elements
3968 #define INSN(NAME, op) \
3969   void NAME(PRegister Pd, SIMD_RegVariant T, PRegister Pn, PRegister Pm) {             \
3970     starti;                                                                            \
3971     assert(T != Q, "invalid size");                                                    \
3972     f(0b00000101, 31, 24), f(T, 23, 22), f(0b10, 21, 20), prf(Pm, 16);                 \
3973     f(0b01001, 15, 11), f(op, 10), f(0b0, 9), prf(Pn, 5), f(0b0, 4), prf(Pd, 0);       \
3974   }
3975 
3976   INSN(sve_uzp1, 0b0); // Concatenate even elements from two predicates
3977   INSN(sve_uzp2, 0b1); // Concatenate odd elements from two predicates
3978 #undef INSN
3979 
3980 // SVE integer compare scalar count and limit
3981 #define INSN(NAME, sf, op)                                                \
3982   void NAME(PRegister Pd, SIMD_RegVariant T, Register Rn, Register Rm) {  \
3983     starti;                                                               \
3984     assert(T != Q, "invalid register variant");                           \
3985     f(0b00100101, 31, 24), f(T, 23, 22), f(1, 21),                        \
3986     zrf(Rm, 16), f(0, 15, 13), f(sf, 12), f(op >> 1, 11, 10),             \
3987     zrf(Rn, 5), f(op & 1, 4), prf(Pd, 0);                                 \
3988   }
3989   // While incrementing signed scalar less than scalar
3990   INSN(sve_whileltw, 0b0, 0b010);
3991   INSN(sve_whilelt,  0b1, 0b010);
3992   // While incrementing signed scalar less than or equal to scalar
3993   INSN(sve_whilelew, 0b0, 0b011);
3994   INSN(sve_whilele,  0b1, 0b011);
3995   // While incrementing unsigned scalar lower than scalar
3996   INSN(sve_whilelow, 0b0, 0b110);
3997   INSN(sve_whilelo,  0b1, 0b110);
3998   // While incrementing unsigned scalar lower than or the same as scalar
3999   INSN(sve_whilelsw, 0b0, 0b111);
4000   INSN(sve_whilels,  0b1, 0b111);
4001 #undef INSN
4002 
4003   // SVE predicate reverse
4004   void sve_rev(PRegister Pd, SIMD_RegVariant T, PRegister Pn) {
4005     starti;
4006     assert(T != Q, "invalid size");
4007     f(0b00000101, 31, 24), f(T, 23, 22), f(0b1101000100000, 21, 9);
4008     prf(Pn, 5), f(0, 4), prf(Pd, 0);
4009   }
4010 
4011 // SVE partition break condition
4012 #define INSN(NAME, op) \
4013   void NAME(PRegister Pd, PRegister Pg, PRegister Pn, bool isMerge) {      \
4014     starti;                                                                \
4015     f(0b00100101, 31, 24), f(op, 23, 22), f(0b01000001, 21, 14);           \
4016     prf(Pg, 10), f(0b0, 9), prf(Pn, 5), f(isMerge ? 1 : 0, 4), prf(Pd, 0); \
4017   }
4018 
4019   INSN(sve_brka, 0b00); // Break after first true condition
4020   INSN(sve_brkb, 0b10); // Break before first true condition
4021 #undef INSN
4022 
4023 // Element count and increment scalar (SVE)
4024 #define INSN(NAME, TYPE)                                                             \
4025   void NAME(Register Xdn, unsigned imm4 = 1, int pattern = 0b11111) {                \
4026     starti;                                                                          \
4027     f(0b00000100, 31, 24), f(TYPE, 23, 22), f(0b10, 21, 20);                         \
4028     f(imm4 - 1, 19, 16), f(0b11100, 15, 11), f(0, 10), f(pattern, 9, 5), rf(Xdn, 0); \
4029   }
4030 
4031   INSN(sve_cntb, B);  // Set scalar to multiple of 8-bit predicate constraint element count
4032   INSN(sve_cnth, H);  // Set scalar to multiple of 16-bit predicate constraint element count
4033   INSN(sve_cntw, S);  // Set scalar to multiple of 32-bit predicate constraint element count
4034   INSN(sve_cntd, D);  // Set scalar to multiple of 64-bit predicate constraint element count
4035 #undef INSN
4036 
4037   // Set scalar to active predicate element count
4038   void sve_cntp(Register Xd, SIMD_RegVariant T, PRegister Pg, PRegister Pn) {
4039     starti;
4040     assert(T != Q, "invalid size");
4041     f(0b00100101, 31, 24), f(T, 23, 22), f(0b10000010, 21, 14);
4042     prf(Pg, 10), f(0, 9), prf(Pn, 5), rf(Xd, 0);
4043   }
4044 
4045   // SVE convert signed integer to floating-point (predicated)
4046   void sve_scvtf(FloatRegister Zd, SIMD_RegVariant T_dst, PRegister Pg,
4047                  FloatRegister Zn, SIMD_RegVariant T_src) {
4048     starti;
4049     assert(T_src != B && T_dst != B && T_src != Q && T_dst != Q &&
4050            (T_src != H || T_dst == T_src), "invalid register variant");
4051     int opc = T_dst;
4052     int opc2 = T_src;
4053     // In most cases we can treat T_dst, T_src as opc, opc2,
4054     // except for the following two combinations.
4055     // +-----+------+---+------------------------------------+
4056     // | opc | opc2 | U |        Instruction Details         |
4057     // +-----+------+---+------------------------------------+
4058     // |  11 |   00 | 0 | SCVTF - 32-bit to double-precision |
4059     // |  11 |   10 | 0 | SCVTF - 64-bit to single-precision |
4060     // +-----+------+---+------------------------------------+
4061     if (T_src == S && T_dst == D) {
4062       opc = 0b11;
4063       opc2 = 0b00;
4064     } else if (T_src == D && T_dst == S) {
4065       opc = 0b11;
4066       opc2 = 0b10;
4067     }
4068     f(0b01100101, 31, 24), f(opc, 23, 22), f(0b010, 21, 19);
4069     f(opc2, 18, 17), f(0b0101, 16, 13);
4070     pgrf(Pg, 10), rf(Zn, 5), rf(Zd, 0);
4071   }
4072 
4073   // SVE floating-point convert to signed integer, rounding toward zero (predicated)
4074   void sve_fcvtzs(FloatRegister Zd, SIMD_RegVariant T_dst, PRegister Pg,
4075                   FloatRegister Zn, SIMD_RegVariant T_src) {
4076     starti;
4077     assert(T_src != B && T_dst != B && T_src != Q && T_dst != Q &&
4078            (T_dst != H || T_src == H), "invalid register variant");
4079     int opc = T_src;
4080     int opc2 = T_dst;
4081     // In most cases we can treat T_src, T_dst as opc, opc2,
4082     // except for the following two combinations.
4083     // +-----+------+---+-------------------------------------+
4084     // | opc | opc2 | U |         Instruction Details         |
4085     // +-----+------+---+-------------------------------------+
4086     // |  11 |  10  | 0 | FCVTZS - single-precision to 64-bit |
4087     // |  11 |  00  | 0 | FCVTZS - double-precision to 32-bit |
4088     // +-----+------+---+-------------------------------------+
4089     if (T_src == S && T_dst == D) {
4090       opc = 0b11;
4091       opc2 = 0b10;
4092     } else if (T_src == D && T_dst == S) {
4093       opc = 0b11;
4094       opc2 = 0b00;
4095     }
4096     f(0b01100101, 31, 24), f(opc, 23, 22), f(0b011, 21, 19);
4097     f(opc2, 18, 17), f(0b0101, 16, 13);
4098     pgrf(Pg, 10), rf(Zn, 5), rf(Zd, 0);
4099   }
4100 
4101   // SVE floating-point convert precision (predicated)
4102   void sve_fcvt(FloatRegister Zd, SIMD_RegVariant T_dst, PRegister Pg,
4103                 FloatRegister Zn, SIMD_RegVariant T_src) {
4104     starti;
4105     assert(T_src != B && T_dst != B && T_src != Q && T_dst != Q &&
4106            T_src != T_dst, "invalid register variant");
4107     // The encodings of fields op1 (bits 17-16) and op2 (bits 23-22)
4108     // depend on T_src and T_dst as given below -
4109     // +-----+------+---------------------------------------------+
4110     // | op2 | op1  |             Instruction Details             |
4111     // +-----+------+---------------------------------------------+
4112     // |  10 |  01  | FCVT - half-precision to single-precision   |
4113     // |  11 |  01  | FCVT - half-precision to double-precision   |
4114     // |  10 |  00  | FCVT - single-precision to half-precision   |
4115     // |  11 |  11  | FCVT - single-precision to double-precision |
4116     // |  11 |  00  | FCVT - double-preciison to half-precision   |
4117     // |  11 |  10  | FCVT - double-precision to single-precision |
4118     // +-----+------+---+-----------------------------------------+
4119     int op1 = 0b00;
4120     int op2 = (T_src == D || T_dst == D) ? 0b11 : 0b10;
4121     if (T_src == H) {
4122       op1 = 0b01;
4123     } else if (T_dst == S) {
4124       op1 = 0b10;
4125     } else if (T_dst == D) {
4126       op1 = 0b11;
4127     }
4128     f(0b01100101, 31, 24), f(op2, 23, 22), f(0b0010, 21, 18);
4129     f(op1, 17, 16), f(0b101, 15, 13);
4130     pgrf(Pg, 10), rf(Zn, 5), rf(Zd, 0);
4131   }
4132 
4133 // SVE extract element to general-purpose register
4134 #define INSN(NAME, before)                                                      \
4135   void NAME(Register Rd, SIMD_RegVariant T, PRegister Pg,  FloatRegister Zn) {  \
4136     starti;                                                                     \
4137     f(0b00000101, 31, 24), f(T, 23, 22), f(0b10000, 21, 17);                    \
4138     f(before, 16), f(0b101, 15, 13);                                            \
4139     pgrf(Pg, 10), rf(Zn, 5), rf(Rd, 0);                                         \
4140   }
4141 
4142   INSN(sve_lasta, 0b0);
4143   INSN(sve_lastb, 0b1);
4144 #undef INSN
4145 
4146 // SVE extract element to SIMD&FP scalar register
4147 #define INSN(NAME, before)                                                           \
4148   void NAME(FloatRegister Vd, SIMD_RegVariant T, PRegister Pg,  FloatRegister Zn) {  \
4149     starti;                                                                          \
4150     f(0b00000101, 31, 24), f(T, 23, 22), f(0b10001, 21, 17);                         \
4151     f(before, 16), f(0b100, 15, 13);                                                 \
4152     pgrf(Pg, 10), rf(Zn, 5), rf(Vd, 0);                                              \
4153   }
4154 
4155   INSN(sve_lasta, 0b0);
4156   INSN(sve_lastb, 0b1);
4157 #undef INSN
4158 
4159 // SVE reverse within elements
4160 #define INSN(NAME, opc, cond)                                                        \
4161   void NAME(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg,  FloatRegister Zn) {  \
4162     starti;                                                                          \
4163     assert(cond, "invalid size");                                                    \
4164     f(0b00000101, 31, 24), f(T, 23, 22), f(0b1001, 21, 18), f(opc, 17, 16);          \
4165     f(0b100, 15, 13), pgrf(Pg, 10), rf(Zn, 5), rf(Zd, 0);                            \
4166   }
4167 
4168   INSN(sve_revb, 0b00, T == H || T == S || T == D);
4169   INSN(sve_rbit, 0b11, T != Q);
4170 #undef INSN
4171 
4172   // SVE Create index starting from general-purpose register and incremented by immediate
4173   void sve_index(FloatRegister Zd, SIMD_RegVariant T, Register Rn, int imm) {
4174     starti;
4175     assert(T != Q, "invalid size");
4176     f(0b00000100, 31, 24), f(T, 23, 22), f(0b1, 21);
4177     sf(imm, 20, 16), f(0b010001, 15, 10);
4178     rf(Rn, 5), rf(Zd, 0);
4179   }
4180 
4181   // SVE create index starting from and incremented by immediate
4182   void sve_index(FloatRegister Zd, SIMD_RegVariant T, int imm1, int imm2) {
4183     starti;
4184     assert(T != Q, "invalid size");
4185     f(0b00000100, 31, 24), f(T, 23, 22), f(0b1, 21);
4186     sf(imm2, 20, 16), f(0b010000, 15, 10);
4187     sf(imm1, 9, 5), rf(Zd, 0);
4188   }
4189 
4190   // SVE programmable table lookup/permute using vector of element indices
4191   void sve_tbl(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, FloatRegister Zm) {
4192     starti;
4193     assert(T != Q, "invalid size");
4194     f(0b00000101, 31, 24), f(T, 23, 22), f(0b1, 21), rf(Zm, 16);
4195     f(0b001100, 15, 10), rf(Zn, 5), rf(Zd, 0);
4196   }
4197 
4198   // Shuffle active elements of vector to the right and fill with zero
4199   void sve_compact(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, PRegister Pg) {
4200     starti;
4201     assert(T == S || T == D, "invalid size");
4202     f(0b00000101, 31, 24), f(T, 23, 22), f(0b100001100, 21, 13);
4203     pgrf(Pg, 10), rf(Zn, 5), rf(Zd, 0);
4204   }
4205 
4206   // SVE2 Count matching elements in vector
4207   void sve_histcnt(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg,
4208                    FloatRegister Zn, FloatRegister Zm) {
4209     starti;
4210     assert(T == S || T == D, "invalid size");
4211     f(0b01000101, 31, 24), f(T, 23, 22), f(0b1, 21), rf(Zm, 16);
4212     f(0b110, 15, 13), pgrf(Pg, 10), rf(Zn, 5), rf(Zd, 0);
4213   }
4214 
4215 // SVE2 bitwise permute
4216 #define INSN(NAME, opc)                                                                  \
4217   void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn,  FloatRegister Zm) {  \
4218     starti;                                                                              \
4219     assert(T != Q, "invalid size");                                                      \
4220     f(0b01000101, 31, 24), f(T, 23, 22), f(0b0, 21);                                     \
4221     rf(Zm, 16), f(0b1011, 15, 12), f(opc, 11, 10);                                       \
4222     rf(Zn, 5), rf(Zd, 0);                                                                \
4223   }
4224 
4225   INSN(sve_bext, 0b00);
4226   INSN(sve_bdep, 0b01);
4227 #undef INSN
4228 
4229 // SVE2 bitwise ternary operations
4230 #define INSN(NAME, opc)                                               \
4231   void NAME(FloatRegister Zdn, FloatRegister Zm, FloatRegister Zk) {  \
4232     starti;                                                           \
4233     f(0b00000100, 31, 24), f(opc, 23, 21), rf(Zm, 16);                \
4234     f(0b001110, 15, 10), rf(Zk, 5), rf(Zdn, 0);                       \
4235   }
4236 
4237   INSN(sve_eor3, 0b001); // Bitwise exclusive OR of three vectors
4238 #undef INSN
4239 
4240 // SVE2 saturating operations - predicate
4241 #define INSN(NAME, op1, op2)                                                          \
4242   void NAME(FloatRegister Zdn, SIMD_RegVariant T, PRegister Pg, FloatRegister Znm) {  \
4243     assert(T != Q, "invalid register variant");                                       \
4244     sve_predicate_reg_insn(op1, op2, Zdn, T, Pg, Znm);                                \
4245   }
4246 
4247   INSN(sve_sqadd, 0b01000100, 0b011000100); // signed saturating add
4248   INSN(sve_sqsub, 0b01000100, 0b011010100); // signed saturating sub
4249   INSN(sve_uqadd, 0b01000100, 0b011001100); // unsigned saturating add
4250   INSN(sve_uqsub, 0b01000100, 0b011011100); // unsigned saturating sub
4251 
4252 #undef INSN
4253 
4254   Assembler(CodeBuffer* code) : AbstractAssembler(code) {
4255   }
4256 
4257   // Stack overflow checking
4258   virtual void bang_stack_with_offset(int offset);
4259 
4260   static bool operand_valid_for_logical_immediate(bool is32, uint64_t imm);
4261   static bool operand_valid_for_sve_logical_immediate(unsigned elembits, uint64_t imm);
4262   static bool operand_valid_for_add_sub_immediate(int64_t imm);
4263   static bool operand_valid_for_sve_add_sub_immediate(int64_t imm);
4264   static bool operand_valid_for_float_immediate(double imm);
4265   static int  operand_valid_for_movi_immediate(uint64_t imm64, SIMD_Arrangement T);
4266 
4267   void emit_data64(jlong data, relocInfo::relocType rtype, int format = 0);
4268   void emit_data64(jlong data, RelocationHolder const& rspec, int format = 0);
4269 };
4270 
4271 inline Assembler::Membar_mask_bits operator|(Assembler::Membar_mask_bits a,
4272                                              Assembler::Membar_mask_bits b) {
4273   return Assembler::Membar_mask_bits(unsigned(a)|unsigned(b));
4274 }
4275 
4276 Instruction_aarch64::~Instruction_aarch64() {
4277   assem->emit_int32(insn);
4278   assert_cond(get_bits() == 0xffffffff);
4279 }
4280 
4281 #undef f
4282 #undef sf
4283 #undef rf
4284 #undef srf
4285 #undef zrf
4286 #undef prf
4287 #undef pgrf
4288 #undef fixed
4289 
4290 #undef starti
4291 
4292 // Invert a condition
4293 inline Assembler::Condition operator~(const Assembler::Condition cond) {
4294   return Assembler::Condition(int(cond) ^ 1);
4295 }
4296 
4297 extern "C" void das(uint64_t start, int len);
4298 
4299 #endif // CPU_AARCH64_ASSEMBLER_AARCH64_HPP