1 /*
   2  * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #ifndef CPU_AARCH64_ASSEMBLER_AARCH64_HPP
  27 #define CPU_AARCH64_ASSEMBLER_AARCH64_HPP
  28 
  29 #include "asm/register.hpp"
  30 #include "metaprogramming/enableIf.hpp"
  31 #include "utilities/debug.hpp"
  32 #include "utilities/globalDefinitions.hpp"
  33 #include "utilities/macros.hpp"
  34 #include <type_traits>
  35 
  36 #ifdef __GNUC__
  37 
  38 // __nop needs volatile so that compiler doesn't optimize it away
  39 #define NOP() asm volatile ("nop");
  40 
  41 #elif defined(_MSC_VER)
  42 
  43 // Use MSVC intrinsic: https://docs.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=vs-2019#I
  44 #define NOP() __nop();
  45 
  46 #endif
  47 
  48 
  49 // definitions of various symbolic names for machine registers
  50 
  51 // First intercalls between C and Java which use 8 general registers
  52 // and 8 floating registers
  53 
  54 // we also have to copy between x86 and ARM registers but that's a
  55 // secondary complication -- not all code employing C call convention
  56 // executes as x86 code though -- we generate some of it
  57 
  58 class Argument {
  59  public:
  60   enum {
  61     n_int_register_parameters_c   = 8,  // r0, r1, ... r7 (c_rarg0, c_rarg1, ...)
  62     n_float_register_parameters_c = 8,  // v0, v1, ... v7 (c_farg0, c_farg1, ... )
  63 
  64     n_int_register_parameters_j   = 8, // r1, ... r7, r0 (rj_rarg0, j_rarg1, ...
  65     n_float_register_parameters_j = 8  // v0, v1, ... v7 (j_farg0, j_farg1, ...
  66   };
  67 };
  68 
  69 constexpr Register c_rarg0 = r0;
  70 constexpr Register c_rarg1 = r1;
  71 constexpr Register c_rarg2 = r2;
  72 constexpr Register c_rarg3 = r3;
  73 constexpr Register c_rarg4 = r4;
  74 constexpr Register c_rarg5 = r5;
  75 constexpr Register c_rarg6 = r6;
  76 constexpr Register c_rarg7 = r7;
  77 
  78 constexpr FloatRegister c_farg0 = v0;
  79 constexpr FloatRegister c_farg1 = v1;
  80 constexpr FloatRegister c_farg2 = v2;
  81 constexpr FloatRegister c_farg3 = v3;
  82 constexpr FloatRegister c_farg4 = v4;
  83 constexpr FloatRegister c_farg5 = v5;
  84 constexpr FloatRegister c_farg6 = v6;
  85 constexpr FloatRegister c_farg7 = v7;
  86 
  87 // Symbolically name the register arguments used by the Java calling convention.
  88 // We have control over the convention for java so we can do what we please.
  89 // What pleases us is to offset the java calling convention so that when
  90 // we call a suitable jni method the arguments are lined up and we don't
  91 // have to do much shuffling. A suitable jni method is non-static and a
  92 // small number of arguments
  93 //
  94 //  |--------------------------------------------------------------------|
  95 //  | c_rarg0  c_rarg1  c_rarg2 c_rarg3 c_rarg4 c_rarg5 c_rarg6 c_rarg7  |
  96 //  |--------------------------------------------------------------------|
  97 //  | r0       r1       r2      r3      r4      r5      r6      r7       |
  98 //  |--------------------------------------------------------------------|
  99 //  | j_rarg7  j_rarg0  j_rarg1 j_rarg2 j_rarg3 j_rarg4 j_rarg5 j_rarg6  |
 100 //  |--------------------------------------------------------------------|
 101 
 102 
 103 constexpr Register j_rarg0 = c_rarg1;
 104 constexpr Register j_rarg1 = c_rarg2;
 105 constexpr Register j_rarg2 = c_rarg3;
 106 constexpr Register j_rarg3 = c_rarg4;
 107 constexpr Register j_rarg4 = c_rarg5;
 108 constexpr Register j_rarg5 = c_rarg6;
 109 constexpr Register j_rarg6 = c_rarg7;
 110 constexpr Register j_rarg7 = c_rarg0;
 111 
 112 // Java floating args are passed as per C
 113 
 114 constexpr FloatRegister j_farg0 = v0;
 115 constexpr FloatRegister j_farg1 = v1;
 116 constexpr FloatRegister j_farg2 = v2;
 117 constexpr FloatRegister j_farg3 = v3;
 118 constexpr FloatRegister j_farg4 = v4;
 119 constexpr FloatRegister j_farg5 = v5;
 120 constexpr FloatRegister j_farg6 = v6;
 121 constexpr FloatRegister j_farg7 = v7;
 122 
 123 // registers used to hold VM data either temporarily within a method
 124 // or across method calls
 125 
 126 // volatile (caller-save) registers
 127 
 128 // r8 is used for indirect result location return
 129 // we use it and r9 as scratch registers
 130 constexpr Register rscratch1 = r8;
 131 constexpr Register rscratch2 = r9;
 132 
 133 // current method -- must be in a call-clobbered register
 134 constexpr Register rmethod = r12;
 135 
 136 // non-volatile (callee-save) registers are r16-29
 137 // of which the following are dedicated global state
 138 
 139 constexpr Register lr            = r30; // link register
 140 constexpr Register rfp           = r29; // frame pointer
 141 constexpr Register rthread       = r28; // current thread
 142 constexpr Register rheapbase     = r27; // base of heap
 143 constexpr Register rcpool        = r26; // constant pool cache
 144 constexpr Register rlocals       = r24; // locals on stack
 145 constexpr Register rbcp          = r22; // bytecode pointer
 146 constexpr Register rdispatch     = r21; // dispatch table base
 147 constexpr Register esp           = r20; // Java expression stack pointer
 148 constexpr Register r19_sender_sp = r19; // sender's SP while in interpreter
 149 
 150 // Preserved predicate register with all elements set TRUE.
 151 constexpr PRegister ptrue = p7;
 152 
 153 #define assert_cond(ARG1) assert(ARG1, #ARG1)
 154 
 155 namespace asm_util {
 156   uint32_t encode_logical_immediate(bool is32, uint64_t imm);
 157   uint32_t encode_sve_logical_immediate(unsigned elembits, uint64_t imm);
 158   bool operand_valid_for_immediate_bits(int64_t imm, unsigned nbits);
 159 };
 160 
 161 using namespace asm_util;
 162 
 163 
 164 class Assembler;
 165 
 166 class Instruction_aarch64 {
 167   unsigned insn;
 168 #ifdef ASSERT
 169   unsigned bits;
 170 #endif
 171   Assembler *assem;
 172 
 173 public:
 174 
 175   Instruction_aarch64(class Assembler *as) {
 176 #ifdef ASSERT
 177     bits = 0;
 178 #endif
 179     insn = 0;
 180     assem = as;
 181   }
 182 
 183   inline ~Instruction_aarch64();
 184 
 185   unsigned &get_insn() { return insn; }
 186 #ifdef ASSERT
 187   unsigned &get_bits() { return bits; }
 188 #endif
 189 
 190   static inline int32_t extend(unsigned val, int hi = 31, int lo = 0) {
 191     union {
 192       unsigned u;
 193       int n;
 194     };
 195 
 196     u = val << (31 - hi);
 197     n = n >> (31 - hi + lo);
 198     return n;
 199   }
 200 
 201   static inline uint32_t extract(uint32_t val, int msb, int lsb) {
 202     int nbits = msb - lsb + 1;
 203     assert_cond(msb >= lsb);
 204     uint32_t mask = checked_cast<uint32_t>(right_n_bits(nbits));
 205     uint32_t result = val >> lsb;
 206     result &= mask;
 207     return result;
 208   }
 209 
 210   static inline int32_t sextract(uint32_t val, int msb, int lsb) {
 211     uint32_t uval = extract(val, msb, lsb);
 212     return extend(uval, msb - lsb);
 213   }
 214 
 215   static ALWAYSINLINE void patch(address a, int msb, int lsb, uint64_t val) {
 216     int nbits = msb - lsb + 1;
 217     guarantee(val < (1ULL << nbits), "Field too big for insn");
 218     assert_cond(msb >= lsb);
 219     unsigned mask = checked_cast<unsigned>(right_n_bits(nbits));
 220     val <<= lsb;
 221     mask <<= lsb;
 222     unsigned target = *(unsigned *)a;
 223     target &= ~mask;
 224     target |= val;
 225     *(unsigned *)a = target;
 226   }
 227 
 228   static void spatch(address a, int msb, int lsb, int64_t val) {
 229     int nbits = msb - lsb + 1;
 230     int64_t chk = val >> (nbits - 1);
 231     guarantee (chk == -1 || chk == 0, "Field too big for insn at " INTPTR_FORMAT, p2i(a));
 232     unsigned uval = val;
 233     unsigned mask = checked_cast<unsigned>(right_n_bits(nbits));
 234     uval &= mask;
 235     uval <<= lsb;
 236     mask <<= lsb;
 237     unsigned target = *(unsigned *)a;
 238     target &= ~mask;
 239     target |= uval;
 240     *(unsigned *)a = target;
 241   }
 242 
 243   void f(unsigned val, int msb, int lsb) {
 244     int nbits = msb - lsb + 1;
 245     guarantee(val < (1ULL << nbits), "Field too big for insn");
 246     assert_cond(msb >= lsb);
 247     val <<= lsb;
 248     insn |= val;
 249 #ifdef ASSERT
 250     unsigned mask = checked_cast<unsigned>(right_n_bits(nbits));
 251     mask <<= lsb;
 252     assert_cond((bits & mask) == 0);
 253     bits |= mask;
 254 #endif
 255   }
 256 
 257   void f(unsigned val, int bit) {
 258     f(val, bit, bit);
 259   }
 260 
 261   void sf(int64_t val, int msb, int lsb) {
 262     int nbits = msb - lsb + 1;
 263     int64_t chk = val >> (nbits - 1);
 264     guarantee (chk == -1 || chk == 0, "Field too big for insn");
 265     unsigned uval = val;
 266     unsigned mask = checked_cast<unsigned>(right_n_bits(nbits));
 267     uval &= mask;
 268     f(uval, lsb + nbits - 1, lsb);
 269   }
 270 
 271   void rf(Register r, int lsb) {
 272     f(r->raw_encoding(), lsb + 4, lsb);
 273   }
 274 
 275   // reg|ZR
 276   void zrf(Register r, int lsb) {
 277     f(r->raw_encoding() - (r == zr), lsb + 4, lsb);
 278   }
 279 
 280   // reg|SP
 281   void srf(Register r, int lsb) {
 282     f(r == sp ? 31 : r->raw_encoding(), lsb + 4, lsb);
 283   }
 284 
 285   void rf(FloatRegister r, int lsb) {
 286     f(r->raw_encoding(), lsb + 4, lsb);
 287   }
 288 
 289   void prf(PRegister r, int lsb) {
 290     f(r->raw_encoding(), lsb + 3, lsb);
 291   }
 292 
 293   void pgrf(PRegister r, int lsb) {
 294     f(r->raw_encoding(), lsb + 2, lsb);
 295   }
 296 
 297   unsigned get(int msb = 31, int lsb = 0) {
 298     int nbits = msb - lsb + 1;
 299     unsigned mask = checked_cast<unsigned>(right_n_bits(nbits)) << lsb;
 300     assert_cond((bits & mask) == mask);
 301     return (insn & mask) >> lsb;
 302   }
 303 };
 304 
 305 #define starti Instruction_aarch64 current_insn(this);
 306 
 307 class PrePost {
 308   int _offset;
 309   Register _r;
 310 protected:
 311   PrePost(Register reg, int o) : _offset(o), _r(reg) { }
 312   ~PrePost() = default;
 313   PrePost(const PrePost&) = default;
 314   PrePost& operator=(const PrePost&) = default;
 315 public:
 316   int offset() const { return _offset; }
 317   Register reg() const { return _r; }
 318 };
 319 
 320 class Pre : public PrePost {
 321 public:
 322   Pre(Register reg, int o) : PrePost(reg, o) { }
 323 };
 324 
 325 class Post : public PrePost {
 326   Register _idx;
 327   bool _is_postreg;
 328 public:
 329   Post(Register reg, int o) : PrePost(reg, o), _idx(noreg), _is_postreg(false) {}
 330   Post(Register reg, Register idx) : PrePost(reg, 0), _idx(idx), _is_postreg(true) {}
 331   Register idx_reg() const { return _idx; }
 332   bool is_postreg() const { return _is_postreg; }
 333 };
 334 
 335 namespace ext
 336 {
 337   enum operation { uxtb, uxth, uxtw, uxtx, sxtb, sxth, sxtw, sxtx };
 338 };
 339 
 340 // Addressing modes
 341 class Address {
 342  public:
 343 
 344   enum mode { no_mode, base_plus_offset, pre, post, post_reg,
 345               base_plus_offset_reg, literal };
 346 
 347   // Shift and extend for base reg + reg offset addressing
 348   class extend {
 349     int _option, _shift;
 350     ext::operation _op;
 351   public:
 352     extend() { }
 353     extend(int s, int o, ext::operation op) : _option(o), _shift(s), _op(op) { }
 354     int option() const{ return _option; }
 355     int shift() const { return _shift; }
 356     ext::operation op() const { return _op; }
 357   };
 358 
 359   static extend uxtw(int shift = -1) { return extend(shift, 0b010, ext::uxtw); }
 360   static extend lsl(int shift = -1)  { return extend(shift, 0b011, ext::uxtx); }
 361   static extend sxtw(int shift = -1) { return extend(shift, 0b110, ext::sxtw); }
 362   static extend sxtx(int shift = -1) { return extend(shift, 0b111, ext::sxtx); }
 363 
 364  private:
 365   struct Nonliteral {
 366     Nonliteral(Register base, Register index, int64_t offset, extend ext = extend())
 367       : _base(base), _index(index), _offset(offset), _ext(ext) {}
 368     Register _base;
 369     Register _index;
 370     int64_t _offset;
 371     extend _ext;
 372   };
 373 
 374   struct Literal {
 375     Literal(address target, const RelocationHolder& rspec)
 376       : _target(target), _rspec(rspec) {}
 377 
 378     // If the target is far we'll need to load the ea of this to a
 379     // register to reach it. Otherwise if near we can do PC-relative
 380     // addressing.
 381     address _target;
 382 
 383     RelocationHolder _rspec;
 384   };
 385 
 386   void assert_is_nonliteral() const NOT_DEBUG_RETURN;
 387   void assert_is_literal() const NOT_DEBUG_RETURN;
 388 
 389   // Discriminated union, based on _mode.
 390   // - no_mode: uses dummy _nonliteral, for ease of copying.
 391   // - literal: only _literal is used.
 392   // - others: only _nonliteral is used.
 393   enum mode _mode;
 394   union {
 395     Nonliteral _nonliteral;
 396     Literal _literal;
 397   };
 398 
 399   // Helper for copy constructor and assignment operator.
 400   // Copy mode-relevant part of a into this.
 401   void copy_data(const Address& a) {
 402     assert(_mode == a._mode, "precondition");
 403     if (_mode == literal) {
 404       new (&_literal) Literal(a._literal);
 405     } else {
 406       // non-literal mode or no_mode.
 407       new (&_nonliteral) Nonliteral(a._nonliteral);
 408     }
 409   }
 410 
 411  public:
 412   // no_mode initializes _nonliteral for ease of copying.
 413   Address() :
 414     _mode(no_mode),
 415     _nonliteral(noreg, noreg, 0)
 416   {}
 417 
 418   Address(Register r) :
 419     _mode(base_plus_offset),
 420     _nonliteral(r, noreg, 0)
 421   {}
 422 
 423   template<typename T, ENABLE_IF(std::is_integral<T>::value)>
 424   Address(Register r, T o) :
 425     _mode(base_plus_offset),
 426     _nonliteral(r, noreg, o)
 427   {}
 428 
 429   Address(Register r, ByteSize disp) : Address(r, in_bytes(disp)) {}
 430 
 431   Address(Register r, Register r1, extend ext = lsl()) :
 432     _mode(base_plus_offset_reg),
 433     _nonliteral(r, r1, 0, ext)
 434   {}
 435 
 436   Address(Pre p) :
 437     _mode(pre),
 438     _nonliteral(p.reg(), noreg, p.offset())
 439   {}
 440 
 441   Address(Post p) :
 442     _mode(p.is_postreg() ? post_reg : post),
 443     _nonliteral(p.reg(), p.idx_reg(), p.offset())
 444   {}
 445 
 446   Address(address target, const RelocationHolder& rspec) :
 447     _mode(literal),
 448     _literal(target, rspec)
 449   {}
 450 
 451   Address(address target, relocInfo::relocType rtype = relocInfo::external_word_type);
 452 
 453   Address(Register base, RegisterOrConstant index, extend ext = lsl()) {
 454     if (index.is_register()) {
 455       _mode = base_plus_offset_reg;
 456       new (&_nonliteral) Nonliteral(base, index.as_register(), 0, ext);
 457     } else {
 458       guarantee(ext.option() == ext::uxtx, "should be");
 459       assert(index.is_constant(), "should be");
 460       _mode = base_plus_offset;
 461       new (&_nonliteral) Nonliteral(base,
 462                                     noreg,
 463                                     index.as_constant() << ext.shift());
 464     }
 465   }
 466 
 467   Address(const Address& a) : _mode(a._mode) { copy_data(a); }
 468 
 469   // Verify the value is trivially destructible regardless of mode, so our
 470   // destructor can also be trivial, and so our assignment operator doesn't
 471   // need to destruct the old value before copying over it.
 472   static_assert(std::is_trivially_destructible<Literal>::value, "must be");
 473   static_assert(std::is_trivially_destructible<Nonliteral>::value, "must be");
 474 
 475   Address& operator=(const Address& a) {
 476     _mode = a._mode;
 477     copy_data(a);
 478     return *this;
 479   }
 480 
 481   ~Address() = default;
 482 
 483   Register base() const {
 484     assert_is_nonliteral();
 485     return _nonliteral._base;
 486   }
 487 
 488   int64_t offset() const {
 489     assert_is_nonliteral();
 490     return _nonliteral._offset;
 491   }
 492 
 493   Register index() const {
 494     assert_is_nonliteral();
 495     return _nonliteral._index;
 496   }
 497 
 498   extend ext() const {
 499     assert_is_nonliteral();
 500     return _nonliteral._ext;
 501   }
 502 
 503   mode getMode() const {
 504     return _mode;
 505   }
 506 
 507   bool uses(Register reg) const {
 508     switch (_mode) {
 509     case literal:
 510     case no_mode:
 511       return false;
 512     case base_plus_offset:
 513     case base_plus_offset_reg:
 514     case pre:
 515     case post:
 516     case post_reg:
 517       return base() == reg || index() == reg;
 518     default:
 519       ShouldNotReachHere();
 520       return false;
 521     }
 522   }
 523 
 524   address target() const {
 525     assert_is_literal();
 526     return _literal._target;
 527   }
 528 
 529   const RelocationHolder& rspec() const {
 530     assert_is_literal();
 531     return _literal._rspec;
 532   }
 533 
 534   void encode(Instruction_aarch64 *i) const {
 535     i->f(0b111, 29, 27);
 536     i->srf(base(), 5);
 537 
 538     switch(_mode) {
 539     case base_plus_offset:
 540       {
 541         unsigned size = i->get(31, 30);
 542         if (i->get(26, 26) && i->get(23, 23)) {
 543           // SIMD Q Type - Size = 128 bits
 544           assert(size == 0, "bad size");
 545           size = 0b100;
 546         }
 547         assert(offset_ok_for_immed(offset(), size),
 548                "must be, was: " INT64_FORMAT ", %d", offset(), size);
 549         unsigned mask = (1 << size) - 1;
 550         if (offset() < 0 || offset() & mask) {
 551           i->f(0b00, 25, 24);
 552           i->f(0, 21), i->f(0b00, 11, 10);
 553           i->sf(offset(), 20, 12);
 554         } else {
 555           i->f(0b01, 25, 24);
 556           i->f(offset() >> size, 21, 10);
 557         }
 558       }
 559       break;
 560 
 561     case base_plus_offset_reg:
 562       {
 563         i->f(0b00, 25, 24);
 564         i->f(1, 21);
 565         i->rf(index(), 16);
 566         i->f(ext().option(), 15, 13);
 567         unsigned size = i->get(31, 30);
 568         if (i->get(26, 26) && i->get(23, 23)) {
 569           // SIMD Q Type - Size = 128 bits
 570           assert(size == 0, "bad size");
 571           size = 0b100;
 572         }
 573         if (size == 0) // It's a byte
 574           i->f(ext().shift() >= 0, 12);
 575         else {
 576           assert(ext().shift() <= 0 || ext().shift() == (int)size, "bad shift");
 577           i->f(ext().shift() > 0, 12);
 578         }
 579         i->f(0b10, 11, 10);
 580       }
 581       break;
 582 
 583     case pre:
 584       i->f(0b00, 25, 24);
 585       i->f(0, 21), i->f(0b11, 11, 10);
 586       i->sf(offset(), 20, 12);
 587       break;
 588 
 589     case post:
 590       i->f(0b00, 25, 24);
 591       i->f(0, 21), i->f(0b01, 11, 10);
 592       i->sf(offset(), 20, 12);
 593       break;
 594 
 595     default:
 596       ShouldNotReachHere();
 597     }
 598   }
 599 
 600   void encode_pair(Instruction_aarch64 *i) const {
 601     switch(_mode) {
 602     case base_plus_offset:
 603       i->f(0b010, 25, 23);
 604       break;
 605     case pre:
 606       i->f(0b011, 25, 23);
 607       break;
 608     case post:
 609       i->f(0b001, 25, 23);
 610       break;
 611     default:
 612       ShouldNotReachHere();
 613     }
 614 
 615     unsigned size; // Operand shift in 32-bit words
 616 
 617     if (i->get(26, 26)) { // float
 618       switch(i->get(31, 30)) {
 619       case 0b10:
 620         size = 2; break;
 621       case 0b01:
 622         size = 1; break;
 623       case 0b00:
 624         size = 0; break;
 625       default:
 626         ShouldNotReachHere();
 627         size = 0;  // unreachable
 628       }
 629     } else {
 630       size = i->get(31, 31);
 631     }
 632 
 633     size = 4 << size;
 634     guarantee(offset() % size == 0, "bad offset");
 635     i->sf(offset() / size, 21, 15);
 636     i->srf(base(), 5);
 637   }
 638 
 639   void encode_nontemporal_pair(Instruction_aarch64 *i) const {
 640     guarantee(_mode == base_plus_offset, "Bad addressing mode for nontemporal op");
 641     i->f(0b000, 25, 23);
 642     unsigned size = i->get(31, 31);
 643     size = 4 << size;
 644     guarantee(offset() % size == 0, "bad offset");
 645     i->sf(offset() / size, 21, 15);
 646     i->srf(base(), 5);
 647   }
 648 
 649   void lea(MacroAssembler *, Register) const;
 650 
 651   static bool offset_ok_for_immed(int64_t offset, uint shift);
 652 
 653   static bool offset_ok_for_sve_immed(int64_t offset, int shift, int vl /* sve vector length */) {
 654     if (offset % vl == 0) {
 655       // Convert address offset into sve imm offset (MUL VL).
 656       int sve_offset = offset / vl;
 657       if (((-(1 << (shift - 1))) <= sve_offset) && (sve_offset < (1 << (shift - 1)))) {
 658         // sve_offset can be encoded
 659         return true;
 660       }
 661     }
 662     return false;
 663   }
 664 };
 665 
 666 // Convenience classes
 667 class RuntimeAddress: public Address {
 668 
 669   public:
 670 
 671   RuntimeAddress(address target) : Address(target, relocInfo::runtime_call_type) {}
 672 
 673 };
 674 
 675 class OopAddress: public Address {
 676 
 677   public:
 678 
 679   OopAddress(address target) : Address(target, relocInfo::oop_type){}
 680 
 681 };
 682 
 683 class ExternalAddress: public Address {
 684  private:
 685   static relocInfo::relocType reloc_for_target(address target) {
 686     // Sometimes ExternalAddress is used for values which aren't
 687     // exactly addresses, like the card table base.
 688     // external_word_type can't be used for values in the first page
 689     // so just skip the reloc in that case.
 690     return external_word_Relocation::can_be_relocated(target) ? relocInfo::external_word_type : relocInfo::none;
 691   }
 692 
 693  public:
 694 
 695   ExternalAddress(address target) : Address(target, reloc_for_target(target)) {}
 696 
 697 };
 698 
 699 class InternalAddress: public Address {
 700 
 701   public:
 702 
 703   InternalAddress(address target) : Address(target, relocInfo::internal_word_type) {}
 704 };
 705 
 706 const int FPUStateSizeInWords = FloatRegister::number_of_registers * FloatRegister::save_slots_per_register;
 707 
 708 typedef enum {
 709   PLDL1KEEP = 0b00000, PLDL1STRM, PLDL2KEEP, PLDL2STRM, PLDL3KEEP, PLDL3STRM,
 710   PSTL1KEEP = 0b10000, PSTL1STRM, PSTL2KEEP, PSTL2STRM, PSTL3KEEP, PSTL3STRM,
 711   PLIL1KEEP = 0b01000, PLIL1STRM, PLIL2KEEP, PLIL2STRM, PLIL3KEEP, PLIL3STRM
 712 } prfop;
 713 
 714 class Assembler : public AbstractAssembler {
 715 
 716 public:
 717 
 718 #ifndef PRODUCT
 719   static const uintptr_t asm_bp;
 720 
 721   void emit_int32(jint x) {
 722     if ((uintptr_t)pc() == asm_bp)
 723       NOP();
 724     AbstractAssembler::emit_int32(x);
 725   }
 726 #else
 727   void emit_int32(jint x) {
 728     AbstractAssembler::emit_int32(x);
 729   }
 730 #endif
 731 
 732   enum { instruction_size = 4 };
 733 
 734   //---<  calculate length of instruction  >---
 735   // We just use the values set above.
 736   // instruction must start at passed address
 737   static unsigned int instr_len(unsigned char *instr) { return instruction_size; }
 738 
 739   //---<  longest instructions  >---
 740   static unsigned int instr_maxlen() { return instruction_size; }
 741 
 742   Address adjust(Register base, int offset, bool preIncrement) {
 743     if (preIncrement)
 744       return Address(Pre(base, offset));
 745     else
 746       return Address(Post(base, offset));
 747   }
 748 
 749   Address pre(Register base, int offset) {
 750     return adjust(base, offset, true);
 751   }
 752 
 753   Address post(Register base, int offset) {
 754     return adjust(base, offset, false);
 755   }
 756 
 757   Address post(Register base, Register idx) {
 758     return Address(Post(base, idx));
 759   }
 760 
 761   static address locate_next_instruction(address inst);
 762 
 763 #define f current_insn.f
 764 #define sf current_insn.sf
 765 #define rf current_insn.rf
 766 #define srf current_insn.srf
 767 #define zrf current_insn.zrf
 768 #define prf current_insn.prf
 769 #define pgrf current_insn.pgrf
 770 
 771   typedef void (Assembler::* uncond_branch_insn)(address dest);
 772   typedef void (Assembler::* compare_and_branch_insn)(Register Rt, address dest);
 773   typedef void (Assembler::* test_and_branch_insn)(Register Rt, int bitpos, address dest);
 774   typedef void (Assembler::* prefetch_insn)(address target, prfop);
 775 
 776   void wrap_label(Label &L, uncond_branch_insn insn);
 777   void wrap_label(Register r, Label &L, compare_and_branch_insn insn);
 778   void wrap_label(Register r, int bitpos, Label &L, test_and_branch_insn insn);
 779   void wrap_label(Label &L, prfop, prefetch_insn insn);
 780 
 781   // PC-rel. addressing
 782 
 783   void adr(Register Rd, address dest);
 784   void _adrp(Register Rd, address dest);
 785 
 786   void adr(Register Rd, const Address &dest);
 787   void _adrp(Register Rd, const Address &dest);
 788 
 789   void adr(Register Rd, Label &L) {
 790     wrap_label(Rd, L, &Assembler::Assembler::adr);
 791   }
 792   void _adrp(Register Rd, Label &L) {
 793     wrap_label(Rd, L, &Assembler::_adrp);
 794   }
 795 
 796   void adrp(Register Rd, const Address &dest, uint64_t &offset) = delete;
 797 
 798 #undef INSN
 799 
 800   void add_sub_immediate(Instruction_aarch64 &current_insn, Register Rd, Register Rn,
 801                          unsigned uimm, int op, int negated_op);
 802 
 803   // Add/subtract (immediate)
 804 #define INSN(NAME, decode, negated)                                     \
 805   void NAME(Register Rd, Register Rn, unsigned imm, unsigned shift) {   \
 806     starti;                                                             \
 807     f(decode, 31, 29), f(0b10001, 28, 24), f(shift, 23, 22), f(imm, 21, 10); \
 808     zrf(Rd, 0), srf(Rn, 5);                                             \
 809   }                                                                     \
 810                                                                         \
 811   void NAME(Register Rd, Register Rn, unsigned imm) {                   \
 812     starti;                                                             \
 813     add_sub_immediate(current_insn, Rd, Rn, imm, decode, negated);      \
 814   }
 815 
 816   INSN(addsw, 0b001, 0b011);
 817   INSN(subsw, 0b011, 0b001);
 818   INSN(adds,  0b101, 0b111);
 819   INSN(subs,  0b111, 0b101);
 820 
 821 #undef INSN
 822 
 823 #define INSN(NAME, decode, negated)                     \
 824   void NAME(Register Rd, Register Rn, unsigned imm) {   \
 825     starti;                                             \
 826     add_sub_immediate(current_insn, Rd, Rn, imm, decode, negated);     \
 827   }
 828 
 829   INSN(addw, 0b000, 0b010);
 830   INSN(subw, 0b010, 0b000);
 831   INSN(add,  0b100, 0b110);
 832   INSN(sub,  0b110, 0b100);
 833 
 834 #undef INSN
 835 
 836  // Logical (immediate)
 837 #define INSN(NAME, decode, is32)                                \
 838   void NAME(Register Rd, Register Rn, uint64_t imm) {           \
 839     starti;                                                     \
 840     uint32_t val = encode_logical_immediate(is32, imm);         \
 841     f(decode, 31, 29), f(0b100100, 28, 23), f(val, 22, 10);     \
 842     srf(Rd, 0), zrf(Rn, 5);                                     \
 843   }
 844 
 845   INSN(andw, 0b000, true);
 846   INSN(orrw, 0b001, true);
 847   INSN(eorw, 0b010, true);
 848   INSN(andr, 0b100, false);
 849   INSN(orr,  0b101, false);
 850   INSN(eor,  0b110, false);
 851 
 852 #undef INSN
 853 
 854 #define INSN(NAME, decode, is32)                                \
 855   void NAME(Register Rd, Register Rn, uint64_t imm) {           \
 856     starti;                                                     \
 857     uint32_t val = encode_logical_immediate(is32, imm);         \
 858     f(decode, 31, 29), f(0b100100, 28, 23), f(val, 22, 10);     \
 859     zrf(Rd, 0), zrf(Rn, 5);                                     \
 860   }
 861 
 862   INSN(ands, 0b111, false);
 863   INSN(andsw, 0b011, true);
 864 
 865 #undef INSN
 866 
 867   // Move wide (immediate)
 868 #define INSN(NAME, opcode)                                              \
 869   void NAME(Register Rd, unsigned imm, unsigned shift = 0) {            \
 870     assert_cond((shift/16)*16 == shift);                                \
 871     starti;                                                             \
 872     f(opcode, 31, 29), f(0b100101, 28, 23), f(shift/16, 22, 21),        \
 873       f(imm, 20, 5);                                                    \
 874     zrf(Rd, 0);                                                         \
 875   }
 876 
 877   INSN(movnw, 0b000);
 878   INSN(movzw, 0b010);
 879   INSN(movkw, 0b011);
 880   INSN(movn,  0b100);
 881   INSN(movz,  0b110);
 882   INSN(movk,  0b111);
 883 
 884 #undef INSN
 885 
 886   // Bitfield
 887 #define INSN(NAME, opcode, size)                                        \
 888   void NAME(Register Rd, Register Rn, unsigned immr, unsigned imms) {   \
 889     starti;                                                             \
 890     guarantee(size == 1 || (immr < 32 && imms < 32), "incorrect immr/imms");\
 891     f(opcode, 31, 22), f(immr, 21, 16), f(imms, 15, 10);                \
 892     zrf(Rn, 5), rf(Rd, 0);                                              \
 893   }
 894 
 895   INSN(sbfmw, 0b0001001100, 0);
 896   INSN(bfmw,  0b0011001100, 0);
 897   INSN(ubfmw, 0b0101001100, 0);
 898   INSN(sbfm,  0b1001001101, 1);
 899   INSN(bfm,   0b1011001101, 1);
 900   INSN(ubfm,  0b1101001101, 1);
 901 
 902 #undef INSN
 903 
 904   // Extract
 905 #define INSN(NAME, opcode, size)                                        \
 906   void NAME(Register Rd, Register Rn, Register Rm, unsigned imms) {     \
 907     starti;                                                             \
 908     guarantee(size == 1 || imms < 32, "incorrect imms");                \
 909     f(opcode, 31, 21), f(imms, 15, 10);                                 \
 910     zrf(Rm, 16), zrf(Rn, 5), zrf(Rd, 0);                                \
 911   }
 912 
 913   INSN(extrw, 0b00010011100, 0);
 914   INSN(extr,  0b10010011110, 1);
 915 
 916 #undef INSN
 917 
 918   // The maximum range of a branch is fixed for the AArch64
 919   // architecture.  In debug mode we shrink it in order to test
 920   // trampolines, but not so small that branches in the interpreter
 921   // are out of range.
 922   static const uint64_t branch_range = NOT_DEBUG(128 * M) DEBUG_ONLY(2 * M);
 923 
 924   static bool reachable_from_branch_at(address branch, address target) {
 925     return uabs(target - branch) < branch_range;
 926   }
 927 
 928   // Unconditional branch (immediate)
 929 #define INSN(NAME, opcode)                                              \
 930   void NAME(address dest) {                                             \
 931     starti;                                                             \
 932     int64_t offset = (dest - pc()) >> 2;                                \
 933     DEBUG_ONLY(assert(reachable_from_branch_at(pc(), dest), "debug only")); \
 934     f(opcode, 31), f(0b00101, 30, 26), sf(offset, 25, 0);               \
 935   }                                                                     \
 936   void NAME(Label &L) {                                                 \
 937     wrap_label(L, &Assembler::NAME);                                    \
 938   }                                                                     \
 939   void NAME(const Address &dest);
 940 
 941   INSN(b, 0);
 942   INSN(bl, 1);
 943 
 944 #undef INSN
 945 
 946   // Compare & branch (immediate)
 947 #define INSN(NAME, opcode)                              \
 948   void NAME(Register Rt, address dest) {                \
 949     int64_t offset = (dest - pc()) >> 2;                \
 950     starti;                                             \
 951     f(opcode, 31, 24), sf(offset, 23, 5), rf(Rt, 0);    \
 952   }                                                     \
 953   void NAME(Register Rt, Label &L) {                    \
 954     wrap_label(Rt, L, &Assembler::NAME);                \
 955   }
 956 
 957   INSN(cbzw,  0b00110100);
 958   INSN(cbnzw, 0b00110101);
 959   INSN(cbz,   0b10110100);
 960   INSN(cbnz,  0b10110101);
 961 
 962 #undef INSN
 963 
 964   // Test & branch (immediate)
 965 #define INSN(NAME, opcode)                                              \
 966   void NAME(Register Rt, int bitpos, address dest) {                    \
 967     int64_t offset = (dest - pc()) >> 2;                                \
 968     int b5 = bitpos >> 5;                                               \
 969     bitpos &= 0x1f;                                                     \
 970     starti;                                                             \
 971     f(b5, 31), f(opcode, 30, 24), f(bitpos, 23, 19), sf(offset, 18, 5); \
 972     rf(Rt, 0);                                                          \
 973   }                                                                     \
 974   void NAME(Register Rt, int bitpos, Label &L) {                        \
 975     wrap_label(Rt, bitpos, L, &Assembler::NAME);                        \
 976   }
 977 
 978   INSN(tbz,  0b0110110);
 979   INSN(tbnz, 0b0110111);
 980 
 981 #undef INSN
 982 
 983   // Conditional branch (immediate)
 984   enum Condition
 985     {EQ, NE, HS, CS=HS, LO, CC=LO, MI, PL, VS, VC, HI, LS, GE, LT, GT, LE, AL, NV};
 986 
 987   void br(Condition  cond, address dest) {
 988     int64_t offset = (dest - pc()) >> 2;
 989     starti;
 990     f(0b0101010, 31, 25), f(0, 24), sf(offset, 23, 5), f(0, 4), f(cond, 3, 0);
 991   }
 992 
 993 #define INSN(NAME, cond)                        \
 994   void NAME(address dest) {                     \
 995     br(cond, dest);                             \
 996   }
 997 
 998   INSN(beq, EQ);
 999   INSN(bne, NE);
1000   INSN(bhs, HS);
1001   INSN(bcs, CS);
1002   INSN(blo, LO);
1003   INSN(bcc, CC);
1004   INSN(bmi, MI);
1005   INSN(bpl, PL);
1006   INSN(bvs, VS);
1007   INSN(bvc, VC);
1008   INSN(bhi, HI);
1009   INSN(bls, LS);
1010   INSN(bge, GE);
1011   INSN(blt, LT);
1012   INSN(bgt, GT);
1013   INSN(ble, LE);
1014   INSN(bal, AL);
1015   INSN(bnv, NV);
1016 
1017   void br(Condition cc, Label &L);
1018 
1019 #undef INSN
1020 
1021   // Exception generation
1022   void generate_exception(int opc, int op2, int LL, unsigned imm) {
1023     starti;
1024     f(0b11010100, 31, 24);
1025     f(opc, 23, 21), f(imm, 20, 5), f(op2, 4, 2), f(LL, 1, 0);
1026   }
1027 
1028 #define INSN(NAME, opc, op2, LL)                \
1029   void NAME(unsigned imm) {                     \
1030     generate_exception(opc, op2, LL, imm);      \
1031   }
1032 
1033   INSN(svc, 0b000, 0, 0b01);
1034   INSN(hvc, 0b000, 0, 0b10);
1035   INSN(smc, 0b000, 0, 0b11);
1036   INSN(brk, 0b001, 0, 0b00);
1037   INSN(hlt, 0b010, 0, 0b00);
1038   INSN(dcps1, 0b101, 0, 0b01);
1039   INSN(dcps2, 0b101, 0, 0b10);
1040   INSN(dcps3, 0b101, 0, 0b11);
1041 
1042 #undef INSN
1043 
1044   // System
1045   void system(int op0, int op1, int CRn, int CRm, int op2,
1046               Register rt = dummy_reg)
1047   {
1048     starti;
1049     f(0b11010101000, 31, 21);
1050     f(op0, 20, 19);
1051     f(op1, 18, 16);
1052     f(CRn, 15, 12);
1053     f(CRm, 11, 8);
1054     f(op2, 7, 5);
1055     rf(rt, 0);
1056   }
1057 
1058   // Hint instructions
1059 
1060 #define INSN(NAME, crm, op2)               \
1061   void NAME() {                            \
1062     system(0b00, 0b011, 0b0010, crm, op2); \
1063   }
1064 
1065   INSN(nop,   0b000, 0b0000);
1066   INSN(yield, 0b000, 0b0001);
1067   INSN(wfe,   0b000, 0b0010);
1068   INSN(wfi,   0b000, 0b0011);
1069   INSN(sev,   0b000, 0b0100);
1070   INSN(sevl,  0b000, 0b0101);
1071 
1072   INSN(autia1716, 0b0001, 0b100);
1073   INSN(autiasp,   0b0011, 0b101);
1074   INSN(autiaz,    0b0011, 0b100);
1075   INSN(autib1716, 0b0001, 0b110);
1076   INSN(autibsp,   0b0011, 0b111);
1077   INSN(autibz,    0b0011, 0b110);
1078   INSN(pacia1716, 0b0001, 0b000);
1079   INSN(paciasp,   0b0011, 0b001);
1080   INSN(paciaz,    0b0011, 0b000);
1081   INSN(pacib1716, 0b0001, 0b010);
1082   INSN(pacibsp,   0b0011, 0b011);
1083   INSN(pacibz,    0b0011, 0b010);
1084   INSN(xpaclri,   0b0000, 0b111);
1085 
1086 #undef INSN
1087 
1088   // we only provide mrs and msr for the special purpose system
1089   // registers where op1 (instr[20:19]) == 11 and, (currently) only
1090   // use it for FPSR n.b msr has L (instr[21]) == 0 mrs has L == 1
1091 
1092   void msr(int op1, int CRn, int CRm, int op2, Register rt) {
1093     starti;
1094     f(0b1101010100011, 31, 19);
1095     f(op1, 18, 16);
1096     f(CRn, 15, 12);
1097     f(CRm, 11, 8);
1098     f(op2, 7, 5);
1099     // writing zr is ok
1100     zrf(rt, 0);
1101   }
1102 
1103   void mrs(int op1, int CRn, int CRm, int op2, Register rt) {
1104     starti;
1105     f(0b1101010100111, 31, 19);
1106     f(op1, 18, 16);
1107     f(CRn, 15, 12);
1108     f(CRm, 11, 8);
1109     f(op2, 7, 5);
1110     // reading to zr is a mistake
1111     rf(rt, 0);
1112   }
1113 
1114   enum barrier {OSHLD = 0b0001, OSHST, OSH, NSHLD=0b0101, NSHST, NSH,
1115                 ISHLD = 0b1001, ISHST, ISH, LD=0b1101, ST, SY};
1116 
1117   void dsb(barrier imm) {
1118     system(0b00, 0b011, 0b00011, imm, 0b100);
1119   }
1120 
1121   void dmb(barrier imm) {
1122     system(0b00, 0b011, 0b00011, imm, 0b101);
1123   }
1124 
1125   void isb() {
1126     system(0b00, 0b011, 0b00011, SY, 0b110);
1127   }
1128 
1129   void sys(int op1, int CRn, int CRm, int op2,
1130            Register rt = as_Register(0b11111)) {
1131     system(0b01, op1, CRn, CRm, op2, rt);
1132   }
1133 
1134   // Only implement operations accessible from EL0 or higher, i.e.,
1135   //            op1    CRn    CRm    op2
1136   // IC IVAU     3      7      5      1
1137   // DC CVAC     3      7      10     1
1138   // DC CVAP     3      7      12     1
1139   // DC CVAU     3      7      11     1
1140   // DC CIVAC    3      7      14     1
1141   // DC ZVA      3      7      4      1
1142   // So only deal with the CRm field.
1143   enum icache_maintenance {IVAU = 0b0101};
1144   enum dcache_maintenance {CVAC = 0b1010, CVAP = 0b1100, CVAU = 0b1011, CIVAC = 0b1110, ZVA = 0b100};
1145 
1146   void dc(dcache_maintenance cm, Register Rt) {
1147     sys(0b011, 0b0111, cm, 0b001, Rt);
1148   }
1149 
1150   void ic(icache_maintenance cm, Register Rt) {
1151     sys(0b011, 0b0111, cm, 0b001, Rt);
1152   }
1153 
1154   // A more convenient access to dmb for our purposes
1155   enum Membar_mask_bits {
1156     // We can use ISH for a barrier because the Arm ARM says "This
1157     // architecture assumes that all Processing Elements that use the
1158     // same operating system or hypervisor are in the same Inner
1159     // Shareable shareability domain."
1160     StoreStore = ISHST,
1161     LoadStore  = ISHLD,
1162     LoadLoad   = ISHLD,
1163     StoreLoad  = ISH,
1164     AnyAny     = ISH
1165   };
1166 
1167   void membar(Membar_mask_bits order_constraint) {
1168     dmb(Assembler::barrier(order_constraint));
1169   }
1170 
1171   // Unconditional branch (register)
1172 
1173   void branch_reg(int OP, int A, int M, Register RN, Register RM) {
1174     starti;
1175     f(0b1101011, 31, 25);
1176     f(OP, 24, 21);
1177     f(0b111110000, 20, 12);
1178     f(A, 11, 11);
1179     f(M, 10, 10);
1180     rf(RN, 5);
1181     rf(RM, 0);
1182   }
1183 
1184 #define INSN(NAME, opc)                         \
1185   void NAME(Register RN) {                      \
1186     branch_reg(opc, 0, 0, RN, r0);              \
1187   }
1188 
1189   INSN(br,  0b0000);
1190   INSN(blr, 0b0001);
1191   INSN(ret, 0b0010);
1192 
1193   void ret(void *p); // This forces a compile-time error for ret(0)
1194 
1195 #undef INSN
1196 
1197 #define INSN(NAME, opc)                         \
1198   void NAME() {                                 \
1199     branch_reg(opc, 0, 0, dummy_reg, r0);       \
1200   }
1201 
1202   INSN(eret, 0b0100);
1203   INSN(drps, 0b0101);
1204 
1205 #undef INSN
1206 
1207 #define INSN(NAME, M)                                  \
1208   void NAME() {                                        \
1209     branch_reg(0b0010, 1, M, dummy_reg, dummy_reg);    \
1210   }
1211 
1212   INSN(retaa, 0);
1213   INSN(retab, 1);
1214 
1215 #undef INSN
1216 
1217 #define INSN(NAME, OP, M)                   \
1218   void NAME(Register rn) {                  \
1219     branch_reg(OP, 1, M, rn, dummy_reg);    \
1220   }
1221 
1222   INSN(braaz,  0b0000, 0);
1223   INSN(brabz,  0b0000, 1);
1224   INSN(blraaz, 0b0001, 0);
1225   INSN(blrabz, 0b0001, 1);
1226 
1227 #undef INSN
1228 
1229 #define INSN(NAME, OP, M)                  \
1230   void NAME(Register rn, Register rm) {    \
1231     branch_reg(OP, 1, M, rn, rm);          \
1232   }
1233 
1234   INSN(braa,  0b1000, 0);
1235   INSN(brab,  0b1000, 1);
1236   INSN(blraa, 0b1001, 0);
1237   INSN(blrab, 0b1001, 1);
1238 
1239 #undef INSN
1240 
1241   // Load/store exclusive
1242   enum operand_size { byte, halfword, word, xword };
1243 
1244   void load_store_exclusive(Register Rs, Register Rt1, Register Rt2,
1245     Register Rn, enum operand_size sz, int op, bool ordered) {
1246     starti;
1247     f(sz, 31, 30), f(0b001000, 29, 24), f(op, 23, 21);
1248     rf(Rs, 16), f(ordered, 15), zrf(Rt2, 10), srf(Rn, 5), zrf(Rt1, 0);
1249   }
1250 
1251   void load_exclusive(Register dst, Register addr,
1252                       enum operand_size sz, bool ordered) {
1253     load_store_exclusive(dummy_reg, dst, dummy_reg, addr,
1254                          sz, 0b010, ordered);
1255   }
1256 
1257   void store_exclusive(Register status, Register new_val, Register addr,
1258                        enum operand_size sz, bool ordered) {
1259     load_store_exclusive(status, new_val, dummy_reg, addr,
1260                          sz, 0b000, ordered);
1261   }
1262 
1263 #define INSN4(NAME, sz, op, o0) /* Four registers */                    \
1264   void NAME(Register Rs, Register Rt1, Register Rt2, Register Rn) {     \
1265     guarantee(Rs != Rn && Rs != Rt1 && Rs != Rt2, "unpredictable instruction"); \
1266     load_store_exclusive(Rs, Rt1, Rt2, Rn, sz, op, o0);                 \
1267   }
1268 
1269 #define INSN3(NAME, sz, op, o0) /* Three registers */                   \
1270   void NAME(Register Rs, Register Rt, Register Rn) {                    \
1271     guarantee(Rs != Rn && Rs != Rt, "unpredictable instruction");       \
1272     load_store_exclusive(Rs, Rt, dummy_reg, Rn, sz, op, o0); \
1273   }
1274 
1275 #define INSN2(NAME, sz, op, o0) /* Two registers */                     \
1276   void NAME(Register Rt, Register Rn) {                                 \
1277     load_store_exclusive(dummy_reg, Rt, dummy_reg, \
1278                          Rn, sz, op, o0);                               \
1279   }
1280 
1281 #define INSN_FOO(NAME, sz, op, o0) /* Three registers, encoded differently */ \
1282   void NAME(Register Rt1, Register Rt2, Register Rn) {                  \
1283     guarantee(Rt1 != Rt2, "unpredictable instruction");                 \
1284     load_store_exclusive(dummy_reg, Rt1, Rt2, Rn, sz, op, o0);          \
1285   }
1286 
1287   // bytes
1288   INSN3(stxrb,  byte, 0b000, 0);
1289   INSN3(stlxrb, byte, 0b000, 1);
1290   INSN2(ldxrb,  byte, 0b010, 0);
1291   INSN2(ldaxrb, byte, 0b010, 1);
1292   INSN2(stlrb,  byte, 0b100, 1);
1293   INSN2(ldarb,  byte, 0b110, 1);
1294 
1295   // halfwords
1296   INSN3(stxrh,  halfword, 0b000, 0);
1297   INSN3(stlxrh, halfword, 0b000, 1);
1298   INSN2(ldxrh,  halfword, 0b010, 0);
1299   INSN2(ldaxrh, halfword, 0b010, 1);
1300   INSN2(stlrh,  halfword, 0b100, 1);
1301   INSN2(ldarh,  halfword, 0b110, 1);
1302 
1303   // words
1304   INSN3(stxrw,  word, 0b000, 0);
1305   INSN3(stlxrw, word, 0b000, 1);
1306   INSN4(stxpw,  word, 0b001, 0);
1307   INSN4(stlxpw, word, 0b001, 1);
1308   INSN2(ldxrw,  word, 0b010, 0);
1309   INSN2(ldaxrw, word, 0b010, 1);
1310   INSN2(stlrw,  word, 0b100, 1);
1311   INSN2(ldarw,  word, 0b110, 1);
1312   // pairs of words
1313   INSN_FOO(ldxpw,  word, 0b011, 0);
1314   INSN_FOO(ldaxpw, word, 0b011, 1);
1315 
1316   // xwords
1317   INSN3(stxr,  xword, 0b000, 0);
1318   INSN3(stlxr, xword, 0b000, 1);
1319   INSN4(stxp,  xword, 0b001, 0);
1320   INSN4(stlxp, xword, 0b001, 1);
1321   INSN2(ldxr,  xword, 0b010, 0);
1322   INSN2(ldaxr, xword, 0b010, 1);
1323   INSN2(stlr,  xword, 0b100, 1);
1324   INSN2(ldar,  xword, 0b110, 1);
1325   // pairs of xwords
1326   INSN_FOO(ldxp,  xword, 0b011, 0);
1327   INSN_FOO(ldaxp, xword, 0b011, 1);
1328 
1329 #undef INSN2
1330 #undef INSN3
1331 #undef INSN4
1332 #undef INSN_FOO
1333 
1334   // 8.1 Compare and swap extensions
1335   void lse_cas(Register Rs, Register Rt, Register Rn,
1336                         enum operand_size sz, bool a, bool r, bool not_pair) {
1337     starti;
1338     if (! not_pair) { // Pair
1339       assert(sz == word || sz == xword, "invalid size");
1340       /* The size bit is in bit 30, not 31 */
1341       sz = (operand_size)(sz == word ? 0b00:0b01);
1342     }
1343     f(sz, 31, 30), f(0b001000, 29, 24), f(not_pair ? 1 : 0, 23), f(a, 22), f(1, 21);
1344     zrf(Rs, 16), f(r, 15), f(0b11111, 14, 10), srf(Rn, 5), zrf(Rt, 0);
1345   }
1346 
1347   // CAS
1348 #define INSN(NAME, a, r)                                                \
1349   void NAME(operand_size sz, Register Rs, Register Rt, Register Rn) {   \
1350     assert(Rs != Rn && Rs != Rt, "unpredictable instruction");          \
1351     lse_cas(Rs, Rt, Rn, sz, a, r, true);                                \
1352   }
1353   INSN(cas,   false, false)
1354   INSN(casa,  true,  false)
1355   INSN(casl,  false, true)
1356   INSN(casal, true,  true)
1357 #undef INSN
1358 
1359   // CASP
1360 #define INSN(NAME, a, r)                                                \
1361   void NAME(operand_size sz, Register Rs, Register Rs1,                 \
1362             Register Rt, Register Rt1, Register Rn) {                   \
1363     assert((Rs->encoding() & 1) == 0 && (Rt->encoding() & 1) == 0 &&    \
1364            Rs->successor() == Rs1 && Rt->successor() == Rt1 &&          \
1365            Rs != Rn && Rs1 != Rn && Rs != Rt, "invalid registers");     \
1366     lse_cas(Rs, Rt, Rn, sz, a, r, false);                               \
1367   }
1368   INSN(casp,   false, false)
1369   INSN(caspa,  true,  false)
1370   INSN(caspl,  false, true)
1371   INSN(caspal, true,  true)
1372 #undef INSN
1373 
1374   // 8.1 Atomic operations
1375   void lse_atomic(Register Rs, Register Rt, Register Rn,
1376                   enum operand_size sz, int op1, int op2, bool a, bool r) {
1377     starti;
1378     f(sz, 31, 30), f(0b111000, 29, 24), f(a, 23), f(r, 22), f(1, 21);
1379     zrf(Rs, 16), f(op1, 15), f(op2, 14, 12), f(0, 11, 10), srf(Rn, 5), zrf(Rt, 0);
1380   }
1381 
1382 #define INSN(NAME, NAME_A, NAME_L, NAME_AL, op1, op2)                   \
1383   void NAME(operand_size sz, Register Rs, Register Rt, Register Rn) {   \
1384     lse_atomic(Rs, Rt, Rn, sz, op1, op2, false, false);                 \
1385   }                                                                     \
1386   void NAME_A(operand_size sz, Register Rs, Register Rt, Register Rn) { \
1387     lse_atomic(Rs, Rt, Rn, sz, op1, op2, true, false);                  \
1388   }                                                                     \
1389   void NAME_L(operand_size sz, Register Rs, Register Rt, Register Rn) { \
1390     lse_atomic(Rs, Rt, Rn, sz, op1, op2, false, true);                  \
1391   }                                                                     \
1392   void NAME_AL(operand_size sz, Register Rs, Register Rt, Register Rn) {\
1393     lse_atomic(Rs, Rt, Rn, sz, op1, op2, true, true);                   \
1394   }
1395   INSN(ldadd,  ldadda,  ldaddl,  ldaddal,  0, 0b000);
1396   INSN(ldbic,  ldbica,  ldbicl,  ldbical,  0, 0b001);
1397   INSN(ldeor,  ldeora,  ldeorl,  ldeoral,  0, 0b010);
1398   INSN(ldorr,  ldorra,  ldorrl,  ldorral,  0, 0b011);
1399   INSN(ldsmax, ldsmaxa, ldsmaxl, ldsmaxal, 0, 0b100);
1400   INSN(ldsmin, ldsmina, ldsminl, ldsminal, 0, 0b101);
1401   INSN(ldumax, ldumaxa, ldumaxl, ldumaxal, 0, 0b110);
1402   INSN(ldumin, ldumina, lduminl, lduminal, 0, 0b111);
1403   INSN(swp,    swpa,    swpl,    swpal,    1, 0b000);
1404 #undef INSN
1405 
1406   // Load register (literal)
1407 #define INSN(NAME, opc, V)                                              \
1408   void NAME(Register Rt, address dest) {                                \
1409     int64_t offset = (dest - pc()) >> 2;                                \
1410     starti;                                                             \
1411     f(opc, 31, 30), f(0b011, 29, 27), f(V, 26), f(0b00, 25, 24),        \
1412       sf(offset, 23, 5);                                                \
1413     rf(Rt, 0);                                                          \
1414   }                                                                     \
1415   void NAME(Register Rt, address dest, relocInfo::relocType rtype) {    \
1416     InstructionMark im(this);                                           \
1417     guarantee(rtype == relocInfo::internal_word_type,                   \
1418               "only internal_word_type relocs make sense here");        \
1419     code_section()->relocate(inst_mark(), InternalAddress(dest).rspec()); \
1420     NAME(Rt, dest);                                                     \
1421   }                                                                     \
1422   void NAME(Register Rt, Label &L) {                                    \
1423     wrap_label(Rt, L, &Assembler::NAME);                                \
1424   }
1425 
1426   INSN(ldrw, 0b00, 0);
1427   INSN(ldr, 0b01, 0);
1428   INSN(ldrsw, 0b10, 0);
1429 
1430 #undef INSN
1431 
1432 #define INSN(NAME, opc, V)                                              \
1433   void NAME(FloatRegister Rt, address dest) {                           \
1434     int64_t offset = (dest - pc()) >> 2;                                \
1435     starti;                                                             \
1436     f(opc, 31, 30), f(0b011, 29, 27), f(V, 26), f(0b00, 25, 24),        \
1437       sf(offset, 23, 5);                                                \
1438     rf(as_Register(Rt), 0);                                             \
1439   }
1440 
1441   INSN(ldrs, 0b00, 1);
1442   INSN(ldrd, 0b01, 1);
1443   INSN(ldrq, 0b10, 1);
1444 
1445 #undef INSN
1446 
1447 #define INSN(NAME, size, opc)                                           \
1448   void NAME(FloatRegister Rt, Register Rn) {                            \
1449     starti;                                                             \
1450     f(size, 31, 30), f(0b111100, 29, 24), f(opc, 23, 22), f(0, 21);     \
1451     f(0, 20, 12), f(0b01, 11, 10);                                      \
1452     rf(Rn, 5), rf(as_Register(Rt), 0);                                  \
1453   }
1454 
1455   INSN(ldrs, 0b10, 0b01);
1456   INSN(ldrd, 0b11, 0b01);
1457   INSN(ldrq, 0b00, 0b11);
1458 
1459 #undef INSN
1460 
1461 
1462 #define INSN(NAME, opc, V)                                              \
1463   void NAME(address dest, prfop op = PLDL1KEEP) {                       \
1464     int64_t offset = (dest - pc()) >> 2;                                \
1465     starti;                                                             \
1466     f(opc, 31, 30), f(0b011, 29, 27), f(V, 26), f(0b00, 25, 24),        \
1467       sf(offset, 23, 5);                                                \
1468     f(op, 4, 0);                                                        \
1469   }                                                                     \
1470   void NAME(Label &L, prfop op = PLDL1KEEP) {                           \
1471     wrap_label(L, op, &Assembler::NAME);                                \
1472   }
1473 
1474   INSN(prfm, 0b11, 0);
1475 
1476 #undef INSN
1477 
1478   // Load/store
1479   void ld_st1(int opc, int p1, int V, int L,
1480               Register Rt1, Register Rt2, Address adr, bool no_allocate) {
1481     starti;
1482     f(opc, 31, 30), f(p1, 29, 27), f(V, 26), f(L, 22);
1483     zrf(Rt2, 10), zrf(Rt1, 0);
1484     if (no_allocate) {
1485       adr.encode_nontemporal_pair(&current_insn);
1486     } else {
1487       adr.encode_pair(&current_insn);
1488     }
1489   }
1490 
1491   // Load/store register pair (offset)
1492 #define INSN(NAME, size, p1, V, L, no_allocate)         \
1493   void NAME(Register Rt1, Register Rt2, Address adr) {  \
1494     ld_st1(size, p1, V, L, Rt1, Rt2, adr, no_allocate); \
1495    }
1496 
1497   INSN(stpw,  0b00, 0b101, 0, 0, false);
1498   INSN(ldpw,  0b00, 0b101, 0, 1, false);
1499   INSN(ldpsw, 0b01, 0b101, 0, 1, false);
1500   INSN(stp,   0b10, 0b101, 0, 0, false);
1501   INSN(ldp,   0b10, 0b101, 0, 1, false);
1502 
1503   // Load/store no-allocate pair (offset)
1504   INSN(stnpw, 0b00, 0b101, 0, 0, true);
1505   INSN(ldnpw, 0b00, 0b101, 0, 1, true);
1506   INSN(stnp,  0b10, 0b101, 0, 0, true);
1507   INSN(ldnp,  0b10, 0b101, 0, 1, true);
1508 
1509 #undef INSN
1510 
1511 #define INSN(NAME, size, p1, V, L, no_allocate)                         \
1512   void NAME(FloatRegister Rt1, FloatRegister Rt2, Address adr) {        \
1513     ld_st1(size, p1, V, L,                                              \
1514            as_Register(Rt1), as_Register(Rt2), adr, no_allocate);       \
1515    }
1516 
1517   INSN(stps, 0b00, 0b101, 1, 0, false);
1518   INSN(ldps, 0b00, 0b101, 1, 1, false);
1519   INSN(stpd, 0b01, 0b101, 1, 0, false);
1520   INSN(ldpd, 0b01, 0b101, 1, 1, false);
1521   INSN(stpq, 0b10, 0b101, 1, 0, false);
1522   INSN(ldpq, 0b10, 0b101, 1, 1, false);
1523 
1524 #undef INSN
1525 
1526   // Load/store register (all modes)
1527   void ld_st2(Register Rt, const Address &adr, int size, int op, int V = 0) {
1528     starti;
1529 
1530     f(V, 26); // general reg?
1531     zrf(Rt, 0);
1532 
1533     // Encoding for literal loads is done here (rather than pushed
1534     // down into Address::encode) because the encoding of this
1535     // instruction is too different from all of the other forms to
1536     // make it worth sharing.
1537     if (adr.getMode() == Address::literal) {
1538       assert(size == 0b10 || size == 0b11, "bad operand size in ldr");
1539       assert(op == 0b01, "literal form can only be used with loads");
1540       f(size & 0b01, 31, 30), f(0b011, 29, 27), f(0b00, 25, 24);
1541       int64_t offset = (adr.target() - pc()) >> 2;
1542       sf(offset, 23, 5);
1543       code_section()->relocate(pc(), adr.rspec());
1544       return;
1545     }
1546 
1547     f(size, 31, 30);
1548     f(op, 23, 22); // str
1549     adr.encode(&current_insn);
1550   }
1551 
1552 #define INSN(NAME, size, op)                            \
1553   void NAME(Register Rt, const Address &adr) {          \
1554     ld_st2(Rt, adr, size, op);                          \
1555   }                                                     \
1556 
1557   INSN(str,  0b11, 0b00);
1558   INSN(strw, 0b10, 0b00);
1559   INSN(strb, 0b00, 0b00);
1560   INSN(strh, 0b01, 0b00);
1561 
1562   INSN(ldr,  0b11, 0b01);
1563   INSN(ldrw, 0b10, 0b01);
1564   INSN(ldrb, 0b00, 0b01);
1565   INSN(ldrh, 0b01, 0b01);
1566 
1567   INSN(ldrsb,  0b00, 0b10);
1568   INSN(ldrsbw, 0b00, 0b11);
1569   INSN(ldrsh,  0b01, 0b10);
1570   INSN(ldrshw, 0b01, 0b11);
1571   INSN(ldrsw,  0b10, 0b10);
1572 
1573 #undef INSN
1574 
1575 #define INSN(NAME, size, op)                                    \
1576   void NAME(const Address &adr, prfop pfop = PLDL1KEEP) {       \
1577     ld_st2(as_Register(pfop), adr, size, op);                   \
1578   }
1579 
1580   INSN(prfm, 0b11, 0b10); // FIXME: PRFM should not be used with
1581                           // writeback modes, but the assembler
1582                           // doesn't enfore that.
1583 
1584 #undef INSN
1585 
1586 #define INSN(NAME, size, op)                            \
1587   void NAME(FloatRegister Rt, const Address &adr) {     \
1588     ld_st2(as_Register(Rt), adr, size, op, 1);          \
1589   }
1590 
1591   INSN(strd, 0b11, 0b00);
1592   INSN(strs, 0b10, 0b00);
1593   INSN(ldrd, 0b11, 0b01);
1594   INSN(ldrs, 0b10, 0b01);
1595   INSN(strq, 0b00, 0b10);
1596   INSN(ldrq, 0x00, 0b11);
1597 
1598 #undef INSN
1599 
1600 /* SIMD extensions
1601  *
1602  * We just use FloatRegister in the following. They are exactly the same
1603  * as SIMD registers.
1604  */
1605 public:
1606 
1607   enum SIMD_Arrangement {
1608     T8B, T16B, T4H, T8H, T2S, T4S, T1D, T2D, T1Q, INVALID_ARRANGEMENT
1609   };
1610 
1611   enum SIMD_RegVariant {
1612       B, H, S, D, Q, INVALID
1613   };
1614 
1615 private:
1616 
1617   static SIMD_Arrangement _esize2arrangement_table[9][2];
1618   static SIMD_RegVariant _esize2regvariant[9];
1619 
1620 public:
1621 
1622   static SIMD_Arrangement esize2arrangement(unsigned esize, bool isQ);
1623   static SIMD_RegVariant elemType_to_regVariant(BasicType bt);
1624   static SIMD_RegVariant elemBytes_to_regVariant(unsigned esize);
1625   // Return the corresponding bits for different SIMD_RegVariant value.
1626   static unsigned regVariant_to_elemBits(SIMD_RegVariant T);
1627 
1628   enum shift_kind { LSL, LSR, ASR, ROR };
1629 
1630   void op_shifted_reg(Instruction_aarch64 &current_insn, unsigned decode,
1631                       enum shift_kind kind, unsigned shift,
1632                       unsigned size, unsigned op) {
1633     f(size, 31);
1634     f(op, 30, 29);
1635     f(decode, 28, 24);
1636     f(shift, 15, 10);
1637     f(kind, 23, 22);
1638   }
1639 
1640   // Logical (shifted register)
1641 #define INSN(NAME, size, op, N)                                         \
1642   void NAME(Register Rd, Register Rn, Register Rm,                      \
1643             enum shift_kind kind = LSL, unsigned shift = 0) {           \
1644     starti;                                                             \
1645     guarantee(size == 1 || shift < 32, "incorrect shift");              \
1646     f(N, 21);                                                           \
1647     zrf(Rm, 16), zrf(Rn, 5), zrf(Rd, 0);                                \
1648     op_shifted_reg(current_insn, 0b01010, kind, shift, size, op);       \
1649   }
1650 
1651   INSN(andr,  1, 0b00, 0);
1652   INSN(orr,   1, 0b01, 0);
1653   INSN(eor,   1, 0b10, 0);
1654   INSN(ands,  1, 0b11, 0);
1655   INSN(andw,  0, 0b00, 0);
1656   INSN(orrw,  0, 0b01, 0);
1657   INSN(eorw,  0, 0b10, 0);
1658   INSN(andsw, 0, 0b11, 0);
1659 
1660 #undef INSN
1661 
1662 #define INSN(NAME, size, op, N)                                         \
1663   void NAME(Register Rd, Register Rn, Register Rm,                      \
1664             enum shift_kind kind = LSL, unsigned shift = 0) {           \
1665     starti;                                                             \
1666     f(N, 21);                                                           \
1667     zrf(Rm, 16), zrf(Rn, 5), zrf(Rd, 0);                                \
1668     op_shifted_reg(current_insn, 0b01010, kind, shift, size, op);       \
1669   }                                                                     \
1670                                                                         \
1671   /* These instructions have no immediate form. Provide an overload so  \
1672      that if anyone does try to use an immediate operand -- this has    \
1673      happened! -- we'll get a compile-time error. */                    \
1674   void NAME(Register Rd, Register Rn, unsigned imm,                     \
1675             enum shift_kind kind = LSL, unsigned shift = 0) {           \
1676     assert(false, " can't be used with immediate operand");             \
1677   }
1678 
1679   INSN(bic,   1, 0b00, 1);
1680   INSN(orn,   1, 0b01, 1);
1681   INSN(eon,   1, 0b10, 1);
1682   INSN(bics,  1, 0b11, 1);
1683   INSN(bicw,  0, 0b00, 1);
1684   INSN(ornw,  0, 0b01, 1);
1685   INSN(eonw,  0, 0b10, 1);
1686   INSN(bicsw, 0, 0b11, 1);
1687 
1688 #undef INSN
1689 
1690 #ifdef _WIN64
1691 // In MSVC, `mvn` is defined as a macro and it affects compilation
1692 #undef mvn
1693 #endif
1694 
1695   // Aliases for short forms of orn
1696 void mvn(Register Rd, Register Rm,
1697             enum shift_kind kind = LSL, unsigned shift = 0) {
1698   orn(Rd, zr, Rm, kind, shift);
1699 }
1700 
1701 void mvnw(Register Rd, Register Rm,
1702             enum shift_kind kind = LSL, unsigned shift = 0) {
1703   ornw(Rd, zr, Rm, kind, shift);
1704 }
1705 
1706   // Add/subtract (shifted register)
1707 #define INSN(NAME, size, op)                            \
1708   void NAME(Register Rd, Register Rn, Register Rm,      \
1709             enum shift_kind kind, unsigned shift = 0) { \
1710     starti;                                             \
1711     f(0, 21);                                           \
1712     assert_cond(kind != ROR);                           \
1713     guarantee(size == 1 || shift < 32, "incorrect shift");\
1714     zrf(Rd, 0), zrf(Rn, 5), zrf(Rm, 16);                \
1715     op_shifted_reg(current_insn, 0b01011, kind, shift, size, op);      \
1716   }
1717 
1718   INSN(add,  1, 0b000);
1719   INSN(sub,  1, 0b10);
1720   INSN(addw, 0, 0b000);
1721   INSN(subw, 0, 0b10);
1722 
1723   INSN(adds,  1, 0b001);
1724   INSN(subs,  1, 0b11);
1725   INSN(addsw, 0, 0b001);
1726   INSN(subsw, 0, 0b11);
1727 
1728 #undef INSN
1729 
1730   // Add/subtract (extended register)
1731 #define INSN(NAME, op)                                                  \
1732   void NAME(Register Rd, Register Rn, Register Rm,                      \
1733            ext::operation option, int amount = 0) {                     \
1734     starti;                                                             \
1735     zrf(Rm, 16), srf(Rn, 5), srf(Rd, 0);                                \
1736     add_sub_extended_reg(current_insn, op, 0b01011, Rd, Rn, Rm, 0b00, option, amount); \
1737   }
1738 
1739   void add_sub_extended_reg(Instruction_aarch64 &current_insn, unsigned op, unsigned decode,
1740     Register Rd, Register Rn, Register Rm,
1741     unsigned opt, ext::operation option, unsigned imm) {
1742     guarantee(imm <= 4, "shift amount must be <= 4");
1743     f(op, 31, 29), f(decode, 28, 24), f(opt, 23, 22), f(1, 21);
1744     f(option, 15, 13), f(imm, 12, 10);
1745   }
1746 
1747   INSN(addw, 0b000);
1748   INSN(subw, 0b010);
1749   INSN(add,  0b100);
1750   INSN(sub,  0b110);
1751 
1752 #undef INSN
1753 
1754 #define INSN(NAME, op)                                                  \
1755   void NAME(Register Rd, Register Rn, Register Rm,                      \
1756            ext::operation option, int amount = 0) {                     \
1757     starti;                                                             \
1758     zrf(Rm, 16), srf(Rn, 5), zrf(Rd, 0);                                \
1759     add_sub_extended_reg(current_insn, op, 0b01011, Rd, Rn, Rm, 0b00, option, amount); \
1760   }
1761 
1762   INSN(addsw, 0b001);
1763   INSN(subsw, 0b011);
1764   INSN(adds,  0b101);
1765   INSN(subs,  0b111);
1766 
1767 #undef INSN
1768 
1769   // Aliases for short forms of add and sub
1770 #define INSN(NAME)                                      \
1771   void NAME(Register Rd, Register Rn, Register Rm) {    \
1772     if (Rd == sp || Rn == sp)                           \
1773       NAME(Rd, Rn, Rm, ext::uxtx);                      \
1774     else                                                \
1775       NAME(Rd, Rn, Rm, LSL);                            \
1776   }
1777 
1778   INSN(addw);
1779   INSN(subw);
1780   INSN(add);
1781   INSN(sub);
1782 
1783   INSN(addsw);
1784   INSN(subsw);
1785   INSN(adds);
1786   INSN(subs);
1787 
1788 #undef INSN
1789 
1790   // Add/subtract (with carry)
1791   void add_sub_carry(unsigned op, Register Rd, Register Rn, Register Rm) {
1792     starti;
1793     f(op, 31, 29);
1794     f(0b11010000, 28, 21);
1795     f(0b000000, 15, 10);
1796     zrf(Rm, 16), zrf(Rn, 5), zrf(Rd, 0);
1797   }
1798 
1799   #define INSN(NAME, op)                                \
1800     void NAME(Register Rd, Register Rn, Register Rm) {  \
1801       add_sub_carry(op, Rd, Rn, Rm);                    \
1802     }
1803 
1804   INSN(adcw,  0b000);
1805   INSN(adcsw, 0b001);
1806   INSN(sbcw,  0b010);
1807   INSN(sbcsw, 0b011);
1808   INSN(adc,   0b100);
1809   INSN(adcs,  0b101);
1810   INSN(sbc,   0b110);
1811   INSN(sbcs,  0b111);
1812 
1813 #undef INSN
1814 
1815   // Conditional compare (both kinds)
1816   void conditional_compare(unsigned op, int o1, int o2, int o3,
1817                            Register Rn, unsigned imm5, unsigned nzcv,
1818                            unsigned cond) {
1819     starti;
1820     f(op, 31, 29);
1821     f(0b11010010, 28, 21);
1822     f(cond, 15, 12);
1823     f(o1, 11);
1824     f(o2, 10);
1825     f(o3, 4);
1826     f(nzcv, 3, 0);
1827     f(imm5, 20, 16), zrf(Rn, 5);
1828   }
1829 
1830 #define INSN(NAME, op)                                                  \
1831   void NAME(Register Rn, Register Rm, int imm, Condition cond) {        \
1832     int regNumber = (Rm == zr ? 31 : Rm->encoding());                   \
1833     conditional_compare(op, 0, 0, 0, Rn, regNumber, imm, cond);         \
1834   }                                                                     \
1835                                                                         \
1836   void NAME(Register Rn, int imm5, int imm, Condition cond) {           \
1837     conditional_compare(op, 1, 0, 0, Rn, imm5, imm, cond);              \
1838   }
1839 
1840   INSN(ccmnw, 0b001);
1841   INSN(ccmpw, 0b011);
1842   INSN(ccmn, 0b101);
1843   INSN(ccmp, 0b111);
1844 
1845 #undef INSN
1846 
1847   // Conditional select
1848   void conditional_select(unsigned op, unsigned op2,
1849                           Register Rd, Register Rn, Register Rm,
1850                           unsigned cond) {
1851     starti;
1852     f(op, 31, 29);
1853     f(0b11010100, 28, 21);
1854     f(cond, 15, 12);
1855     f(op2, 11, 10);
1856     zrf(Rm, 16), zrf(Rn, 5), rf(Rd, 0);
1857   }
1858 
1859 #define INSN(NAME, op, op2)                                             \
1860   void NAME(Register Rd, Register Rn, Register Rm, Condition cond) {    \
1861     conditional_select(op, op2, Rd, Rn, Rm, cond);                      \
1862   }
1863 
1864   INSN(cselw,  0b000, 0b00);
1865   INSN(csincw, 0b000, 0b01);
1866   INSN(csinvw, 0b010, 0b00);
1867   INSN(csnegw, 0b010, 0b01);
1868   INSN(csel,   0b100, 0b00);
1869   INSN(csinc,  0b100, 0b01);
1870   INSN(csinv,  0b110, 0b00);
1871   INSN(csneg,  0b110, 0b01);
1872 
1873 #undef INSN
1874 
1875   // Data processing
1876   void data_processing(Instruction_aarch64 &current_insn, unsigned op29, unsigned opcode,
1877                        Register Rd, Register Rn) {
1878     f(op29, 31, 29), f(0b11010110, 28, 21);
1879     f(opcode, 15, 10);
1880     rf(Rn, 5), rf(Rd, 0);
1881   }
1882 
1883   // (1 source)
1884 #define INSN(NAME, op29, opcode2, opcode)                       \
1885   void NAME(Register Rd, Register Rn) {                         \
1886     starti;                                                     \
1887     f(opcode2, 20, 16);                                         \
1888     data_processing(current_insn, op29, opcode, Rd, Rn);        \
1889   }
1890 
1891   INSN(rbitw,  0b010, 0b00000, 0b00000);
1892   INSN(rev16w, 0b010, 0b00000, 0b00001);
1893   INSN(revw,   0b010, 0b00000, 0b00010);
1894   INSN(clzw,   0b010, 0b00000, 0b00100);
1895   INSN(clsw,   0b010, 0b00000, 0b00101);
1896 
1897   INSN(rbit,   0b110, 0b00000, 0b00000);
1898   INSN(rev16,  0b110, 0b00000, 0b00001);
1899   INSN(rev32,  0b110, 0b00000, 0b00010);
1900   INSN(rev,    0b110, 0b00000, 0b00011);
1901   INSN(clz,    0b110, 0b00000, 0b00100);
1902   INSN(cls,    0b110, 0b00000, 0b00101);
1903 
1904   // PAC instructions
1905   INSN(pacia,  0b110, 0b00001, 0b00000);
1906   INSN(pacib,  0b110, 0b00001, 0b00001);
1907   INSN(pacda,  0b110, 0b00001, 0b00010);
1908   INSN(pacdb,  0b110, 0b00001, 0b00011);
1909   INSN(autia,  0b110, 0b00001, 0b00100);
1910   INSN(autib,  0b110, 0b00001, 0b00101);
1911   INSN(autda,  0b110, 0b00001, 0b00110);
1912   INSN(autdb,  0b110, 0b00001, 0b00111);
1913 
1914 #undef INSN
1915 
1916 #define INSN(NAME, op29, opcode2, opcode)                       \
1917   void NAME(Register Rd) {                                      \
1918     starti;                                                     \
1919     f(opcode2, 20, 16);                                         \
1920     data_processing(current_insn, op29, opcode, Rd, dummy_reg); \
1921   }
1922 
1923   // PAC instructions (with zero modifier)
1924   INSN(paciza,  0b110, 0b00001, 0b01000);
1925   INSN(pacizb,  0b110, 0b00001, 0b01001);
1926   INSN(pacdza,  0b110, 0b00001, 0b01010);
1927   INSN(pacdzb,  0b110, 0b00001, 0b01011);
1928   INSN(autiza,  0b110, 0b00001, 0b01100);
1929   INSN(autizb,  0b110, 0b00001, 0b01101);
1930   INSN(autdza,  0b110, 0b00001, 0b01110);
1931   INSN(autdzb,  0b110, 0b00001, 0b01111);
1932   INSN(xpaci,   0b110, 0b00001, 0b10000);
1933   INSN(xpacd,   0b110, 0b00001, 0b10001);
1934 
1935 #undef INSN
1936 
1937   // Data-processing (2 source)
1938 #define INSN(NAME, op29, opcode)                                \
1939   void NAME(Register Rd, Register Rn, Register Rm) {            \
1940     starti;                                                     \
1941     rf(Rm, 16);                                                 \
1942     data_processing(current_insn, op29, opcode, Rd, Rn);        \
1943   }
1944 
1945   INSN(udivw, 0b000, 0b000010);
1946   INSN(sdivw, 0b000, 0b000011);
1947   INSN(lslvw, 0b000, 0b001000);
1948   INSN(lsrvw, 0b000, 0b001001);
1949   INSN(asrvw, 0b000, 0b001010);
1950   INSN(rorvw, 0b000, 0b001011);
1951 
1952   INSN(udiv, 0b100, 0b000010);
1953   INSN(sdiv, 0b100, 0b000011);
1954   INSN(lslv, 0b100, 0b001000);
1955   INSN(lsrv, 0b100, 0b001001);
1956   INSN(asrv, 0b100, 0b001010);
1957   INSN(rorv, 0b100, 0b001011);
1958 
1959 #undef INSN
1960 
1961   // Data-processing (3 source)
1962   void data_processing(unsigned op54, unsigned op31, unsigned o0,
1963                        Register Rd, Register Rn, Register Rm,
1964                        Register Ra) {
1965     starti;
1966     f(op54, 31, 29), f(0b11011, 28, 24);
1967     f(op31, 23, 21), f(o0, 15);
1968     zrf(Rm, 16), zrf(Ra, 10), zrf(Rn, 5), zrf(Rd, 0);
1969   }
1970 
1971 #define INSN(NAME, op54, op31, o0)                                      \
1972   void NAME(Register Rd, Register Rn, Register Rm, Register Ra) {       \
1973     data_processing(op54, op31, o0, Rd, Rn, Rm, Ra);                    \
1974   }
1975 
1976   INSN(maddw,  0b000, 0b000, 0);
1977   INSN(msubw,  0b000, 0b000, 1);
1978   INSN(madd,   0b100, 0b000, 0);
1979   INSN(msub,   0b100, 0b000, 1);
1980   INSN(smaddl, 0b100, 0b001, 0);
1981   INSN(smsubl, 0b100, 0b001, 1);
1982   INSN(umaddl, 0b100, 0b101, 0);
1983   INSN(umsubl, 0b100, 0b101, 1);
1984 
1985 #undef INSN
1986 
1987 #define INSN(NAME, op54, op31, o0)                                      \
1988   void NAME(Register Rd, Register Rn, Register Rm) {                    \
1989     data_processing(op54, op31, o0, Rd, Rn, Rm, as_Register(31));       \
1990   }
1991 
1992   INSN(smulh, 0b100, 0b010, 0);
1993   INSN(umulh, 0b100, 0b110, 0);
1994 
1995 #undef INSN
1996 
1997   // Floating-point data-processing (1 source)
1998   void data_processing(unsigned type, unsigned opcode,
1999                        FloatRegister Vd, FloatRegister Vn) {
2000     starti;
2001     f(0b000, 31, 29);
2002     f(0b11110, 28, 24);
2003     f(type, 23, 22), f(1, 21), f(opcode, 20, 15), f(0b10000, 14, 10);
2004     rf(Vn, 5), rf(Vd, 0);
2005   }
2006 
2007 #define INSN(NAME, type, opcode)                        \
2008   void NAME(FloatRegister Vd, FloatRegister Vn) {       \
2009     data_processing(type, opcode, Vd, Vn);              \
2010   }
2011 
2012   INSN(fmovs,  0b00, 0b000000);
2013   INSN(fabss,  0b00, 0b000001);
2014   INSN(fnegs,  0b00, 0b000010);
2015   INSN(fsqrts, 0b00, 0b000011);
2016   INSN(fcvts,  0b00, 0b000101);   // Single-precision to double-precision
2017   INSN(fcvths, 0b11, 0b000100);   // Half-precision to single-precision
2018   INSN(fcvtsh, 0b00, 0b000111);   // Single-precision to half-precision
2019 
2020   INSN(fmovd,  0b01, 0b000000);
2021   INSN(fabsd,  0b01, 0b000001);
2022   INSN(fnegd,  0b01, 0b000010);
2023   INSN(fsqrtd, 0b01, 0b000011);
2024   INSN(fcvtd,  0b01, 0b000100);   // Double-precision to single-precision
2025 
2026 private:
2027   void _fcvt_narrow_extend(FloatRegister Vd, SIMD_Arrangement Ta,
2028                            FloatRegister Vn, SIMD_Arrangement Tb, bool do_extend) {
2029     assert((do_extend && (Tb >> 1) + 1 == (Ta >> 1))
2030            || (!do_extend && (Ta >> 1) + 1 == (Tb >> 1)), "Incompatible arrangement");
2031     starti;
2032     int op30 = (do_extend ? Tb : Ta) & 1;
2033     int op22 = ((do_extend ? Ta : Tb) >> 1) & 1;
2034     f(0, 31), f(op30, 30), f(0b0011100, 29, 23), f(op22, 22);
2035     f(0b100001011, 21, 13), f(do_extend ? 1 : 0, 12), f(0b10, 11, 10);
2036     rf(Vn, 5), rf(Vd, 0);
2037   }
2038 
2039 public:
2040   void fcvtl(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn,  SIMD_Arrangement Tb) {
2041     assert(Tb == T4H || Tb == T8H|| Tb == T2S || Tb == T4S, "invalid arrangement");
2042     _fcvt_narrow_extend(Vd, Ta, Vn, Tb, true);
2043   }
2044 
2045   void fcvtn(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn,  SIMD_Arrangement Tb) {
2046     assert(Ta == T4H || Ta == T8H|| Ta == T2S || Ta == T4S, "invalid arrangement");
2047     _fcvt_narrow_extend(Vd, Ta, Vn, Tb, false);
2048   }
2049 
2050 #undef INSN
2051 
2052   // Floating-point data-processing (2 source)
2053   void data_processing(unsigned op31, unsigned type, unsigned opcode,
2054                        FloatRegister Vd, FloatRegister Vn, FloatRegister Vm) {
2055     starti;
2056     f(op31, 31, 29);
2057     f(0b11110, 28, 24);
2058     f(type, 23, 22), f(1, 21), f(opcode, 15, 10);
2059     rf(Vm, 16), rf(Vn, 5), rf(Vd, 0);
2060   }
2061 
2062 #define INSN(NAME, op31, type, opcode)                  \
2063   void NAME(FloatRegister Vd, FloatRegister Vn, FloatRegister Vm) {     \
2064     data_processing(op31, type, opcode, Vd, Vn, Vm);    \
2065   }
2066 
2067   INSN(fabds,  0b011, 0b10, 0b110101);
2068   INSN(fmuls,  0b000, 0b00, 0b000010);
2069   INSN(fdivs,  0b000, 0b00, 0b000110);
2070   INSN(fadds,  0b000, 0b00, 0b001010);
2071   INSN(fsubs,  0b000, 0b00, 0b001110);
2072   INSN(fmaxs,  0b000, 0b00, 0b010010);
2073   INSN(fmins,  0b000, 0b00, 0b010110);
2074   INSN(fnmuls, 0b000, 0b00, 0b100010);
2075 
2076   INSN(fabdd,  0b011, 0b11, 0b110101);
2077   INSN(fmuld,  0b000, 0b01, 0b000010);
2078   INSN(fdivd,  0b000, 0b01, 0b000110);
2079   INSN(faddd,  0b000, 0b01, 0b001010);
2080   INSN(fsubd,  0b000, 0b01, 0b001110);
2081   INSN(fmaxd,  0b000, 0b01, 0b010010);
2082   INSN(fmind,  0b000, 0b01, 0b010110);
2083   INSN(fnmuld, 0b000, 0b01, 0b100010);
2084 
2085 #undef INSN
2086 
2087    // Floating-point data-processing (3 source)
2088   void data_processing(unsigned op31, unsigned type, unsigned o1, unsigned o0,
2089                        FloatRegister Vd, FloatRegister Vn, FloatRegister Vm,
2090                        FloatRegister Va) {
2091     starti;
2092     f(op31, 31, 29);
2093     f(0b11111, 28, 24);
2094     f(type, 23, 22), f(o1, 21), f(o0, 15);
2095     rf(Vm, 16), rf(Va, 10), rf(Vn, 5), rf(Vd, 0);
2096   }
2097 
2098 #define INSN(NAME, op31, type, o1, o0)                                  \
2099   void NAME(FloatRegister Vd, FloatRegister Vn, FloatRegister Vm,       \
2100             FloatRegister Va) {                                         \
2101     data_processing(op31, type, o1, o0, Vd, Vn, Vm, Va);                \
2102   }
2103 
2104   INSN(fmadds,  0b000, 0b00, 0, 0);
2105   INSN(fmsubs,  0b000, 0b00, 0, 1);
2106   INSN(fnmadds, 0b000, 0b00, 1, 0);
2107   INSN(fnmsubs, 0b000, 0b00, 1, 1);
2108 
2109   INSN(fmaddd,  0b000, 0b01, 0, 0);
2110   INSN(fmsubd,  0b000, 0b01, 0, 1);
2111   INSN(fnmaddd, 0b000, 0b01, 1, 0);
2112   INSN(fnmsub,  0b000, 0b01, 1, 1);
2113 
2114 #undef INSN
2115 
2116    // Floating-point conditional select
2117   void fp_conditional_select(unsigned op31, unsigned type,
2118                              unsigned op1, unsigned op2,
2119                              Condition cond, FloatRegister Vd,
2120                              FloatRegister Vn, FloatRegister Vm) {
2121     starti;
2122     f(op31, 31, 29);
2123     f(0b11110, 28, 24);
2124     f(type, 23, 22);
2125     f(op1, 21, 21);
2126     f(op2, 11, 10);
2127     f(cond, 15, 12);
2128     rf(Vm, 16), rf(Vn, 5), rf(Vd, 0);
2129   }
2130 
2131 #define INSN(NAME, op31, type, op1, op2)                                \
2132   void NAME(FloatRegister Vd, FloatRegister Vn,                         \
2133             FloatRegister Vm, Condition cond) {                         \
2134     fp_conditional_select(op31, type, op1, op2, cond, Vd, Vn, Vm);      \
2135   }
2136 
2137   INSN(fcsels, 0b000, 0b00, 0b1, 0b11);
2138   INSN(fcseld, 0b000, 0b01, 0b1, 0b11);
2139 
2140 #undef INSN
2141 
2142   // Conversion between floating-point and integer
2143   void float_int_convert(unsigned sflag, unsigned ftype,
2144                          unsigned rmode, unsigned opcode,
2145                          Register Rd, Register Rn) {
2146     starti;
2147     f(sflag, 31);
2148     f(0b00, 30, 29);
2149     f(0b11110, 28, 24);
2150     f(ftype, 23, 22), f(1, 21), f(rmode, 20, 19);
2151     f(opcode, 18, 16), f(0b000000, 15, 10);
2152     zrf(Rn, 5), zrf(Rd, 0);
2153   }
2154 
2155 #define INSN(NAME, sflag, ftype, rmode, opcode)                          \
2156   void NAME(Register Rd, FloatRegister Vn) {                             \
2157     float_int_convert(sflag, ftype, rmode, opcode, Rd, as_Register(Vn)); \
2158   }
2159 
2160   INSN(fcvtzsw, 0b0, 0b00, 0b11, 0b000);
2161   INSN(fcvtzs,  0b1, 0b00, 0b11, 0b000);
2162   INSN(fcvtzdw, 0b0, 0b01, 0b11, 0b000);
2163   INSN(fcvtzd,  0b1, 0b01, 0b11, 0b000);
2164 
2165   // RoundToNearestTiesAway
2166   INSN(fcvtassw, 0b0, 0b00, 0b00, 0b100);  // float -> signed word
2167   INSN(fcvtasd,  0b1, 0b01, 0b00, 0b100);  // double -> signed xword
2168 
2169   // RoundTowardsNegative
2170   INSN(fcvtmssw, 0b0, 0b00, 0b10, 0b000);  // float -> signed word
2171   INSN(fcvtmsd,  0b1, 0b01, 0b10, 0b000);  // double -> signed xword
2172 
2173   INSN(fmovs, 0b0, 0b00, 0b00, 0b110);
2174   INSN(fmovd, 0b1, 0b01, 0b00, 0b110);
2175 
2176   INSN(fmovhid, 0b1, 0b10, 0b01, 0b110);
2177 
2178 #undef INSN
2179 
2180 #define INSN(NAME, sflag, type, rmode, opcode)                          \
2181   void NAME(FloatRegister Vd, Register Rn) {                            \
2182     float_int_convert(sflag, type, rmode, opcode, as_Register(Vd), Rn); \
2183   }
2184 
2185   INSN(fmovs, 0b0, 0b00, 0b00, 0b111);
2186   INSN(fmovd, 0b1, 0b01, 0b00, 0b111);
2187 
2188   INSN(scvtfws, 0b0, 0b00, 0b00, 0b010);
2189   INSN(scvtfs,  0b1, 0b00, 0b00, 0b010);
2190   INSN(scvtfwd, 0b0, 0b01, 0b00, 0b010);
2191   INSN(scvtfd,  0b1, 0b01, 0b00, 0b010);
2192 
2193   // INSN(fmovhid, 0b100, 0b10, 0b01, 0b111);
2194 
2195 #undef INSN
2196 
2197   enum sign_kind { SIGNED, UNSIGNED };
2198 
2199 private:
2200   void _xcvtf_scalar_integer(sign_kind sign, unsigned sz,
2201                              FloatRegister Rd, FloatRegister Rn) {
2202     starti;
2203     f(0b01, 31, 30), f(sign == SIGNED ? 0 : 1, 29);
2204     f(0b111100, 27, 23), f((sz >> 1) & 1, 22), f(0b100001110110, 21, 10);
2205     rf(Rn, 5), rf(Rd, 0);
2206   }
2207 
2208 public:
2209 #define INSN(NAME, sign, sz)                        \
2210   void NAME(FloatRegister Rd, FloatRegister Rn) {   \
2211     _xcvtf_scalar_integer(sign, sz, Rd, Rn);        \
2212   }
2213 
2214   INSN(scvtfs, SIGNED, 0);
2215   INSN(scvtfd, SIGNED, 1);
2216 
2217 #undef INSN
2218 
2219 private:
2220   void _xcvtf_vector_integer(sign_kind sign, SIMD_Arrangement T,
2221                              FloatRegister Rd, FloatRegister Rn) {
2222     assert(T == T2S || T == T4S || T == T2D, "invalid arrangement");
2223     starti;
2224     f(0, 31), f(T & 1, 30), f(sign == SIGNED ? 0 : 1, 29);
2225     f(0b011100, 28, 23), f((T >> 1) & 1, 22), f(0b100001110110, 21, 10);
2226     rf(Rn, 5), rf(Rd, 0);
2227   }
2228 
2229 public:
2230   void scvtfv(SIMD_Arrangement T, FloatRegister Rd, FloatRegister Rn) {
2231     _xcvtf_vector_integer(SIGNED, T, Rd, Rn);
2232   }
2233 
2234   // Floating-point compare
2235   void float_compare(unsigned op31, unsigned type,
2236                      unsigned op, unsigned op2,
2237                      FloatRegister Vn, FloatRegister Vm = as_FloatRegister(0)) {
2238     starti;
2239     f(op31, 31, 29);
2240     f(0b11110, 28, 24);
2241     f(type, 23, 22), f(1, 21);
2242     f(op, 15, 14), f(0b1000, 13, 10), f(op2, 4, 0);
2243     rf(Vn, 5), rf(Vm, 16);
2244   }
2245 
2246 
2247 #define INSN(NAME, op31, type, op, op2)                 \
2248   void NAME(FloatRegister Vn, FloatRegister Vm) {       \
2249     float_compare(op31, type, op, op2, Vn, Vm);         \
2250   }
2251 
2252 #define INSN1(NAME, op31, type, op, op2)        \
2253   void NAME(FloatRegister Vn, double d) {       \
2254     assert_cond(d == 0.0);                      \
2255     float_compare(op31, type, op, op2, Vn);     \
2256   }
2257 
2258   INSN(fcmps, 0b000, 0b00, 0b00, 0b00000);
2259   INSN1(fcmps, 0b000, 0b00, 0b00, 0b01000);
2260   // INSN(fcmpes, 0b000, 0b00, 0b00, 0b10000);
2261   // INSN1(fcmpes, 0b000, 0b00, 0b00, 0b11000);
2262 
2263   INSN(fcmpd, 0b000,   0b01, 0b00, 0b00000);
2264   INSN1(fcmpd, 0b000,  0b01, 0b00, 0b01000);
2265   // INSN(fcmped, 0b000,  0b01, 0b00, 0b10000);
2266   // INSN1(fcmped, 0b000, 0b01, 0b00, 0b11000);
2267 
2268 #undef INSN
2269 #undef INSN1
2270 
2271 // Floating-point compare. 3-registers versions (scalar).
2272 #define INSN(NAME, sz, e)                                             \
2273   void NAME(FloatRegister Vd, FloatRegister Vn, FloatRegister Vm) {   \
2274     starti;                                                           \
2275     f(0b01111110, 31, 24), f(e, 23), f(sz, 22), f(1, 21), rf(Vm, 16); \
2276     f(0b111011, 15, 10), rf(Vn, 5), rf(Vd, 0);                        \
2277   }                                                                   \
2278 
2279   INSN(facged, 1, 0); // facge-double
2280   INSN(facges, 0, 0); // facge-single
2281   INSN(facgtd, 1, 1); // facgt-double
2282   INSN(facgts, 0, 1); // facgt-single
2283 
2284 #undef INSN
2285 
2286   // Floating-point Move (immediate)
2287 private:
2288   unsigned pack(double value);
2289 
2290   void fmov_imm(FloatRegister Vn, double value, unsigned size) {
2291     starti;
2292     f(0b00011110, 31, 24), f(size, 23, 22), f(1, 21);
2293     f(pack(value), 20, 13), f(0b10000000, 12, 5);
2294     rf(Vn, 0);
2295   }
2296 
2297 public:
2298 
2299   void fmovs(FloatRegister Vn, double value) {
2300     if (value)
2301       fmov_imm(Vn, value, 0b00);
2302     else
2303       movi(Vn, T2S, 0);
2304   }
2305   void fmovd(FloatRegister Vn, double value) {
2306     if (value)
2307       fmov_imm(Vn, value, 0b01);
2308     else
2309       movi(Vn, T1D, 0);
2310   }
2311 
2312   // Floating-point data-processing (1 source)
2313 
2314    // Floating-point rounding
2315    // type: half-precision = 11
2316    //       single         = 00
2317    //       double         = 01
2318    // rmode: A = Away     = 100
2319    //        I = current  = 111
2320    //        M = MinusInf = 010
2321    //        N = eveN     = 000
2322    //        P = PlusInf  = 001
2323    //        X = eXact    = 110
2324    //        Z = Zero     = 011
2325   void float_round(unsigned type, unsigned rmode, FloatRegister Rd, FloatRegister Rn) {
2326     starti;
2327     f(0b00011110, 31, 24);
2328     f(type, 23, 22);
2329     f(0b1001, 21, 18);
2330     f(rmode, 17, 15);
2331     f(0b10000, 14, 10);
2332     rf(Rn, 5), rf(Rd, 0);
2333   }
2334 #define INSN(NAME, type, rmode)                   \
2335   void NAME(FloatRegister Vd, FloatRegister Vn) { \
2336     float_round(type, rmode, Vd, Vn);             \
2337   }
2338 
2339 public:
2340   INSN(frintah, 0b11, 0b100);
2341   INSN(frintih, 0b11, 0b111);
2342   INSN(frintmh, 0b11, 0b010);
2343   INSN(frintnh, 0b11, 0b000);
2344   INSN(frintph, 0b11, 0b001);
2345   INSN(frintxh, 0b11, 0b110);
2346   INSN(frintzh, 0b11, 0b011);
2347 
2348   INSN(frintas, 0b00, 0b100);
2349   INSN(frintis, 0b00, 0b111);
2350   INSN(frintms, 0b00, 0b010);
2351   INSN(frintns, 0b00, 0b000);
2352   INSN(frintps, 0b00, 0b001);
2353   INSN(frintxs, 0b00, 0b110);
2354   INSN(frintzs, 0b00, 0b011);
2355 
2356   INSN(frintad, 0b01, 0b100);
2357   INSN(frintid, 0b01, 0b111);
2358   INSN(frintmd, 0b01, 0b010);
2359   INSN(frintnd, 0b01, 0b000);
2360   INSN(frintpd, 0b01, 0b001);
2361   INSN(frintxd, 0b01, 0b110);
2362   INSN(frintzd, 0b01, 0b011);
2363 #undef INSN
2364 
2365 private:
2366   static short SIMD_Size_in_bytes[];
2367 
2368 public:
2369 #define INSN(NAME, op)                                                  \
2370   void NAME(FloatRegister Rt, SIMD_RegVariant T, const Address &adr) {  \
2371     ld_st2(as_Register(Rt), adr, (int)T & 3, op + ((T==Q) ? 0b10:0b00), 1); \
2372   }
2373 
2374   INSN(ldr, 1);
2375   INSN(str, 0);
2376 
2377 #undef INSN
2378 
2379  private:
2380 
2381   void ld_st(FloatRegister Vt, SIMD_Arrangement T, Register Xn, int op1, int op2) {
2382     starti;
2383     f(0,31), f((int)T & 1, 30);
2384     f(op1, 29, 21), f(0, 20, 16), f(op2, 15, 12);
2385     f((int)T >> 1, 11, 10), srf(Xn, 5), rf(Vt, 0);
2386   }
2387   void ld_st(FloatRegister Vt, SIMD_Arrangement T, Register Xn,
2388              int imm, int op1, int op2, int regs) {
2389 
2390     bool replicate = op2 >> 2 == 3;
2391     // post-index value (imm) is formed differently for replicate/non-replicate ld* instructions
2392     int expectedImmediate = replicate ? regs * (1 << (T >> 1)) : SIMD_Size_in_bytes[T] * regs;
2393     guarantee(T < T1Q , "incorrect arrangement");
2394     guarantee(imm == expectedImmediate, "bad offset");
2395     starti;
2396     f(0,31), f((int)T & 1, 30);
2397     f(op1 | 0b100, 29, 21), f(0b11111, 20, 16), f(op2, 15, 12);
2398     f((int)T >> 1, 11, 10), srf(Xn, 5), rf(Vt, 0);
2399   }
2400   void ld_st(FloatRegister Vt, SIMD_Arrangement T, Register Xn,
2401              Register Xm, int op1, int op2) {
2402     starti;
2403     f(0,31), f((int)T & 1, 30);
2404     f(op1 | 0b100, 29, 21), rf(Xm, 16), f(op2, 15, 12);
2405     f((int)T >> 1, 11, 10), srf(Xn, 5), rf(Vt, 0);
2406   }
2407 
2408   void ld_st(FloatRegister Vt, SIMD_Arrangement T, Address a, int op1, int op2, int regs) {
2409     switch (a.getMode()) {
2410     case Address::base_plus_offset:
2411       guarantee(a.offset() == 0, "no offset allowed here");
2412       ld_st(Vt, T, a.base(), op1, op2);
2413       break;
2414     case Address::post:
2415       ld_st(Vt, T, a.base(), a.offset(), op1, op2, regs);
2416       break;
2417     case Address::post_reg:
2418       ld_st(Vt, T, a.base(), a.index(), op1, op2);
2419       break;
2420     default:
2421       ShouldNotReachHere();
2422     }
2423   }
2424 
2425   // Single-structure load/store method (all addressing variants)
2426   void ld_st(FloatRegister Vt, SIMD_RegVariant T, int index, Address a,
2427              int op1, int op2, int regs) {
2428     int expectedImmediate = (regVariant_to_elemBits(T) >> 3) * regs;
2429     int sVal = (T < D) ? (index >> (2 - T)) & 0x01 : 0;
2430     int opcode = (T < D) ? (T << 2) : ((T & 0x02) << 2);
2431     int size = (T < D) ? (index & (0x3 << T)) : 1;  // only care about low 2b
2432     Register Xn = a.base();
2433     int Rm;
2434 
2435     switch (a.getMode()) {
2436     case Address::base_plus_offset:
2437       guarantee(a.offset() == 0, "no offset allowed here");
2438       Rm = 0;
2439       break;
2440     case Address::post:
2441       guarantee(a.offset() == expectedImmediate, "bad offset");
2442       op1 |= 0b100;
2443       Rm = 0b11111;
2444       break;
2445     case Address::post_reg:
2446       op1 |= 0b100;
2447       Rm = a.index()->encoding();
2448       break;
2449     default:
2450       ShouldNotReachHere();
2451       Rm = 0;  // unreachable
2452     }
2453 
2454     starti;
2455     f(0,31), f((index >> (3 - T)), 30);
2456     f(op1, 29, 21), f(Rm, 20, 16), f(op2 | opcode | sVal, 15, 12);
2457     f(size, 11, 10), srf(Xn, 5), rf(Vt, 0);
2458   }
2459 
2460  public:
2461 
2462 #define INSN1(NAME, op1, op2)                                           \
2463   void NAME(FloatRegister Vt, SIMD_Arrangement T, const Address &a) {   \
2464     ld_st(Vt, T, a, op1, op2, 1);                                       \
2465  }
2466 
2467 #define INSN2(NAME, op1, op2)                                           \
2468   void NAME(FloatRegister Vt, FloatRegister Vt2, SIMD_Arrangement T, const Address &a) { \
2469     assert(Vt->successor() == Vt2, "Registers must be ordered");        \
2470     ld_st(Vt, T, a, op1, op2, 2);                                       \
2471   }
2472 
2473 #define INSN3(NAME, op1, op2)                                           \
2474   void NAME(FloatRegister Vt, FloatRegister Vt2, FloatRegister Vt3,     \
2475             SIMD_Arrangement T, const Address &a) {                     \
2476     assert(Vt->successor() == Vt2 && Vt2->successor() == Vt3,           \
2477            "Registers must be ordered");                                \
2478     ld_st(Vt, T, a, op1, op2, 3);                                       \
2479   }
2480 
2481 #define INSN4(NAME, op1, op2)                                           \
2482   void NAME(FloatRegister Vt, FloatRegister Vt2, FloatRegister Vt3,     \
2483             FloatRegister Vt4, SIMD_Arrangement T, const Address &a) {  \
2484     assert(Vt->successor() == Vt2 && Vt2->successor() == Vt3 &&         \
2485            Vt3->successor() == Vt4, "Registers must be ordered");       \
2486     ld_st(Vt, T, a, op1, op2, 4);                                       \
2487   }
2488 
2489   INSN1(ld1,  0b001100010, 0b0111);
2490   INSN2(ld1,  0b001100010, 0b1010);
2491   INSN3(ld1,  0b001100010, 0b0110);
2492   INSN4(ld1,  0b001100010, 0b0010);
2493 
2494   INSN2(ld2,  0b001100010, 0b1000);
2495   INSN3(ld3,  0b001100010, 0b0100);
2496   INSN4(ld4,  0b001100010, 0b0000);
2497 
2498   INSN1(st1,  0b001100000, 0b0111);
2499   INSN2(st1,  0b001100000, 0b1010);
2500   INSN3(st1,  0b001100000, 0b0110);
2501   INSN4(st1,  0b001100000, 0b0010);
2502 
2503   INSN2(st2,  0b001100000, 0b1000);
2504   INSN3(st3,  0b001100000, 0b0100);
2505   INSN4(st4,  0b001100000, 0b0000);
2506 
2507   INSN1(ld1r, 0b001101010, 0b1100);
2508   INSN2(ld2r, 0b001101011, 0b1100);
2509   INSN3(ld3r, 0b001101010, 0b1110);
2510   INSN4(ld4r, 0b001101011, 0b1110);
2511 
2512 #undef INSN1
2513 #undef INSN2
2514 #undef INSN3
2515 #undef INSN4
2516 
2517 // Handle common single-structure ld/st parameter sanity checks
2518 // for all variations (1 to 4) of SIMD reigster inputs.  This
2519 // method will call the routine that generates the opcode.
2520 template<typename R, typename... Rx>
2521   void ldst_sstr(SIMD_RegVariant T, int index, const Address &a,
2522             int op1, int op2, R firstReg, Rx... otherRegs) {
2523     const FloatRegister vtSet[] = { firstReg, otherRegs... };
2524     const int regCount = sizeof...(otherRegs) + 1;
2525     assert(index >= 0 && (T <= D) && ((T == B && index <= 15) ||
2526               (T == H && index <= 7) || (T == S && index <= 3) ||
2527               (T == D && index <= 1)), "invalid index");
2528     assert(regCount >= 1 && regCount <= 4, "illegal register count");
2529 
2530     // Check to make sure when multiple SIMD registers are used
2531     // that they are in successive order.
2532     for (int i = 0; i < regCount - 1; i++) {
2533       assert(vtSet[i]->successor() == vtSet[i + 1],
2534              "Registers must be ordered");
2535     }
2536 
2537     ld_st(firstReg, T, index, a, op1, op2, regCount);
2538   }
2539 
2540 // Define a set of INSN1/2/3/4 macros to handle single-structure
2541 // load/store instructions.
2542 #define INSN1(NAME, op1, op2)                                           \
2543   void NAME(FloatRegister Vt, SIMD_RegVariant T, int index,             \
2544             const Address &a) {                                         \
2545     ldst_sstr(T, index, a, op1, op2, Vt);                               \
2546  }
2547 
2548 #define INSN2(NAME, op1, op2)                                           \
2549   void NAME(FloatRegister Vt, FloatRegister Vt2, SIMD_RegVariant T,     \
2550             int index, const Address &a) {                              \
2551     ldst_sstr(T, index, a, op1, op2, Vt, Vt2);                          \
2552   }
2553 
2554 #define INSN3(NAME, op1, op2)                                           \
2555   void NAME(FloatRegister Vt, FloatRegister Vt2, FloatRegister Vt3,     \
2556             SIMD_RegVariant T, int index, const Address &a) {           \
2557     ldst_sstr(T, index, a, op1, op2, Vt, Vt2, Vt3);                     \
2558   }
2559 
2560 #define INSN4(NAME, op1, op2)                                           \
2561   void NAME(FloatRegister Vt, FloatRegister Vt2, FloatRegister Vt3,     \
2562             FloatRegister Vt4, SIMD_RegVariant T, int index,            \
2563             const Address &a) {                                         \
2564     ldst_sstr(T, index, a, op1, op2, Vt, Vt2, Vt3, Vt4);                \
2565   }
2566 
2567   INSN1(st1, 0b001101000, 0b0000);
2568   INSN2(st2, 0b001101001, 0b0000);
2569   INSN3(st3, 0b001101000, 0b0010);
2570   INSN4(st4, 0b001101001, 0b0010);
2571 
2572 #undef INSN1
2573 #undef INSN2
2574 #undef INSN3
2575 #undef INSN4
2576 
2577 #define INSN(NAME, opc)                                                                 \
2578   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \
2579     starti;                                                                             \
2580     assert(T == T8B || T == T16B, "must be T8B or T16B");                               \
2581     f(0, 31), f((int)T & 1, 30), f(opc, 29, 21);                                        \
2582     rf(Vm, 16), f(0b000111, 15, 10), rf(Vn, 5), rf(Vd, 0);                              \
2583   }
2584 
2585   INSN(eor,  0b101110001);
2586   INSN(orr,  0b001110101);
2587   INSN(andr, 0b001110001);
2588   INSN(bic,  0b001110011);
2589   INSN(bif,  0b101110111);
2590   INSN(bit,  0b101110101);
2591   INSN(bsl,  0b101110011);
2592   INSN(orn,  0b001110111);
2593 
2594 #undef INSN
2595 
2596   // Advanced SIMD three different
2597 #define INSN(NAME, opc, opc2, acceptT2D)                                                \
2598   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \
2599     guarantee(T != T1Q && T != T1D, "incorrect arrangement");                           \
2600     if (!acceptT2D) guarantee(T != T2D, "incorrect arrangement");                       \
2601     starti;                                                                             \
2602     f(0, 31), f((int)T & 1, 30), f(opc, 29), f(0b01110, 28, 24);                        \
2603     f((int)T >> 1, 23, 22), f(1, 21), rf(Vm, 16), f(opc2, 15, 10);                      \
2604     rf(Vn, 5), rf(Vd, 0);                                                               \
2605   }
2606 
2607   INSN(addv,   0, 0b100001, true);  // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2608   INSN(subv,   1, 0b100001, true);  // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2609   INSN(uqsubv, 1, 0b001011, true);  // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2610   INSN(mulv,   0, 0b100111, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2611   INSN(mlav,   0, 0b100101, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2612   INSN(mlsv,   1, 0b100101, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2613   INSN(sshl,   0, 0b010001, true);  // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2614   INSN(ushl,   1, 0b010001, true);  // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2615   INSN(addpv,  0, 0b101111, true);  // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2616   INSN(smullv, 0, 0b110000, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2617   INSN(umullv, 1, 0b110000, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2618   INSN(umlalv, 1, 0b100000, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2619   INSN(maxv,   0, 0b011001, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2620   INSN(minv,   0, 0b011011, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2621   INSN(smaxp,  0, 0b101001, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2622   INSN(sminp,  0, 0b101011, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2623 
2624 #undef INSN
2625 
2626 #define INSN(NAME, opc, opc2, accepted) \
2627   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {                   \
2628     guarantee(T != T1Q && T != T1D, "incorrect arrangement");                           \
2629     if (accepted < 3) guarantee(T != T2D, "incorrect arrangement");                     \
2630     if (accepted < 2) guarantee(T != T2S, "incorrect arrangement");                     \
2631     if (accepted < 1) guarantee(T == T8B || T == T16B, "incorrect arrangement");        \
2632     starti;                                                                             \
2633     f(0, 31), f((int)T & 1, 30), f(opc, 29), f(0b01110, 28, 24);                        \
2634     f((int)T >> 1, 23, 22), f(opc2, 21, 10);                                            \
2635     rf(Vn, 5), rf(Vd, 0);                                                               \
2636   }
2637 
2638   INSN(absr,   0, 0b100000101110, 3); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2639   INSN(negr,   1, 0b100000101110, 3); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2640   INSN(notr,   1, 0b100000010110, 0); // accepted arrangements: T8B, T16B
2641   INSN(addv,   0, 0b110001101110, 1); // accepted arrangements: T8B, T16B, T4H, T8H,      T4S
2642   INSN(smaxv,  0, 0b110000101010, 1); // accepted arrangements: T8B, T16B, T4H, T8H,      T4S
2643   INSN(umaxv,  1, 0b110000101010, 1); // accepted arrangements: T8B, T16B, T4H, T8H,      T4S
2644   INSN(sminv,  0, 0b110001101010, 1); // accepted arrangements: T8B, T16B, T4H, T8H,      T4S
2645   INSN(uminv,  1, 0b110001101010, 1); // accepted arrangements: T8B, T16B, T4H, T8H,      T4S
2646   INSN(cls,    0, 0b100000010010, 2); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2647   INSN(clz,    1, 0b100000010010, 2); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2648   INSN(cnt,    0, 0b100000010110, 0); // accepted arrangements: T8B, T16B
2649   INSN(uaddlp, 1, 0b100000001010, 2); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2650   INSN(uaddlv, 1, 0b110000001110, 1); // accepted arrangements: T8B, T16B, T4H, T8H,      T4S
2651 
2652 #undef INSN
2653 
2654 #define INSN(NAME, opc) \
2655   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {                  \
2656     starti;                                                                            \
2657     assert(T == T4S, "arrangement must be T4S");                                       \
2658     f(0, 31), f((int)T & 1, 30), f(0b101110, 29, 24), f(opc, 23),                      \
2659     f(T == T4S ? 0 : 1, 22), f(0b110000111110, 21, 10); rf(Vn, 5), rf(Vd, 0);          \
2660   }
2661 
2662   INSN(fmaxv, 0);
2663   INSN(fminv, 1);
2664 
2665 #undef INSN
2666 
2667 // Advanced SIMD modified immediate
2668 #define INSN(NAME, op0, cmode0) \
2669   void NAME(FloatRegister Vd, SIMD_Arrangement T, unsigned imm8, unsigned lsl = 0) {   \
2670     unsigned cmode = cmode0;                                                           \
2671     unsigned op = op0;                                                                 \
2672     starti;                                                                            \
2673     assert(lsl == 0 ||                                                                 \
2674            ((T == T4H || T == T8H) && lsl == 8) ||                                     \
2675            ((T == T2S || T == T4S) && ((lsl >> 3) < 4) && ((lsl & 7) == 0)), "invalid shift");\
2676     cmode |= lsl >> 2;                                                                 \
2677     if (T == T4H || T == T8H) cmode |= 0b1000;                                         \
2678     if (!(T == T4H || T == T8H || T == T2S || T == T4S)) {                             \
2679       assert(op == 0 && cmode0 == 0, "must be MOVI");                                  \
2680       cmode = 0b1110;                                                                  \
2681       if (T == T1D || T == T2D) op = 1;                                                \
2682     }                                                                                  \
2683     f(0, 31), f((int)T & 1, 30), f(op, 29), f(0b0111100000, 28, 19);                   \
2684     f(imm8 >> 5, 18, 16), f(cmode, 15, 12), f(0x01, 11, 10), f(imm8 & 0b11111, 9, 5);  \
2685     rf(Vd, 0);                                                                         \
2686   }
2687 
2688   INSN(movi, 0, 0);
2689   INSN(orri, 0, 1);
2690   INSN(mvni, 1, 0);
2691   INSN(bici, 1, 1);
2692 
2693 #undef INSN
2694 
2695 #define INSN(NAME, op, cmode)                                           \
2696   void NAME(FloatRegister Vd, SIMD_Arrangement T, double imm) {         \
2697     unsigned imm8 = pack(imm);                                          \
2698     starti;                                                             \
2699     f(0, 31), f((int)T & 1, 30), f(op, 29), f(0b0111100000, 28, 19);    \
2700     f(imm8 >> 5, 18, 16), f(cmode, 15, 12), f(0x01, 11, 10), f(imm8 & 0b11111, 9, 5); \
2701     rf(Vd, 0);                                                          \
2702   }
2703 
2704   INSN(fmovs, 0, 0b1111);
2705   INSN(fmovd, 1, 0b1111);
2706 
2707 #undef INSN
2708 
2709 // Advanced SIMD three same
2710 #define INSN(NAME, op1, op2, op3)                                                       \
2711   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \
2712     starti;                                                                             \
2713     assert(T == T2S || T == T4S || T == T2D, "invalid arrangement");                    \
2714     f(0, 31), f((int)T & 1, 30), f(op1, 29), f(0b01110, 28, 24), f(op2, 23);            \
2715     f(T==T2D ? 1:0, 22); f(1, 21), rf(Vm, 16), f(op3, 15, 10), rf(Vn, 5), rf(Vd, 0);    \
2716   }
2717 
2718   INSN(fabd, 1, 1, 0b110101);
2719   INSN(fadd, 0, 0, 0b110101);
2720   INSN(fdiv, 1, 0, 0b111111);
2721   INSN(faddp, 1, 0, 0b110101);
2722   INSN(fmul, 1, 0, 0b110111);
2723   INSN(fsub, 0, 1, 0b110101);
2724   INSN(fmla, 0, 0, 0b110011);
2725   INSN(fmls, 0, 1, 0b110011);
2726   INSN(fmax, 0, 0, 0b111101);
2727   INSN(fmin, 0, 1, 0b111101);
2728   INSN(facgt, 1, 1, 0b111011);
2729 
2730 #undef INSN
2731 
2732   // AdvSIMD vector compare
2733   void cm(Condition cond, FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) {
2734     starti;
2735     assert(T != T1Q && T != T1D, "incorrect arrangement");
2736     int cond_op;
2737     switch (cond) {
2738       case EQ: cond_op = 0b110001; break;
2739       case GT: cond_op = 0b000110; break;
2740       case GE: cond_op = 0b000111; break;
2741       case HI: cond_op = 0b100110; break;
2742       case HS: cond_op = 0b100111; break;
2743       default:
2744         ShouldNotReachHere();
2745         break;
2746     }
2747 
2748     f(0, 31), f((int)T & 1, 30), f((cond_op >> 5) & 1, 29);
2749     f(0b01110, 28, 24), f((int)T >> 1, 23, 22), f(1, 21), rf(Vm, 16);
2750     f(cond_op & 0b11111, 15, 11), f(1, 10), rf(Vn, 5), rf(Vd, 0);
2751   }
2752 
2753   // AdvSIMD Floating-point vector compare
2754   void fcm(Condition cond, FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) {
2755     starti;
2756     assert(T == T2S || T == T4S || T == T2D, "invalid arrangement");
2757     int cond_op;
2758     switch (cond) {
2759       case EQ: cond_op = 0b00; break;
2760       case GT: cond_op = 0b11; break;
2761       case GE: cond_op = 0b10; break;
2762       default:
2763         ShouldNotReachHere();
2764         break;
2765     }
2766 
2767     f(0, 31), f((int)T & 1, 30), f((cond_op >> 1) & 1, 29);
2768     f(0b01110, 28, 24), f(cond_op & 1, 23), f(T == T2D ? 1 : 0, 22);
2769     f(1, 21), rf(Vm, 16), f(0b111001, 15, 10), rf(Vn, 5), rf(Vd, 0);
2770   }
2771 
2772 #define INSN(NAME, opc)                                                                 \
2773   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \
2774     starti;                                                                             \
2775     assert(T == T4S, "arrangement must be T4S");                                        \
2776     f(0b01011110000, 31, 21), rf(Vm, 16), f(opc, 15, 10), rf(Vn, 5), rf(Vd, 0);         \
2777   }
2778 
2779   INSN(sha1c,     0b000000);
2780   INSN(sha1m,     0b001000);
2781   INSN(sha1p,     0b000100);
2782   INSN(sha1su0,   0b001100);
2783   INSN(sha256h2,  0b010100);
2784   INSN(sha256h,   0b010000);
2785   INSN(sha256su1, 0b011000);
2786 
2787 #undef INSN
2788 
2789 #define INSN(NAME, opc)                                                                 \
2790   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {                   \
2791     starti;                                                                             \
2792     assert(T == T4S, "arrangement must be T4S");                                        \
2793     f(0b0101111000101000, 31, 16), f(opc, 15, 10), rf(Vn, 5), rf(Vd, 0);                \
2794   }
2795 
2796   INSN(sha1h,     0b000010);
2797   INSN(sha1su1,   0b000110);
2798   INSN(sha256su0, 0b001010);
2799 
2800 #undef INSN
2801 
2802 #define INSN(NAME, opc)                                                                 \
2803   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \
2804     starti;                                                                             \
2805     assert(T == T2D, "arrangement must be T2D");                                        \
2806     f(0b11001110011, 31, 21), rf(Vm, 16), f(opc, 15, 10), rf(Vn, 5), rf(Vd, 0);         \
2807   }
2808 
2809   INSN(sha512h,   0b100000);
2810   INSN(sha512h2,  0b100001);
2811   INSN(sha512su1, 0b100010);
2812 
2813 #undef INSN
2814 
2815 #define INSN(NAME, opc)                                                                 \
2816   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {                   \
2817     starti;                                                                             \
2818     assert(T == T2D, "arrangement must be T2D");                                        \
2819     f(opc, 31, 10), rf(Vn, 5), rf(Vd, 0);                                               \
2820   }
2821 
2822   INSN(sha512su0, 0b1100111011000000100000);
2823 
2824 #undef INSN
2825 
2826 #define INSN(NAME, opc)                                                                                   \
2827   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm, FloatRegister Va) { \
2828     starti;                                                                                               \
2829     assert(T == T16B, "arrangement must be T16B");                                                        \
2830     f(0b11001110, 31, 24), f(opc, 23, 21), rf(Vm, 16), f(0b0, 15, 15), rf(Va, 10), rf(Vn, 5), rf(Vd, 0);  \
2831   }
2832 
2833   INSN(eor3, 0b000);
2834   INSN(bcax, 0b001);
2835 
2836 #undef INSN
2837 
2838 #define INSN(NAME, opc)                                                                               \
2839   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm, unsigned imm) { \
2840     starti;                                                                                           \
2841     assert(T == T2D, "arrangement must be T2D");                                                      \
2842     f(0b11001110, 31, 24), f(opc, 23, 21), rf(Vm, 16), f(imm, 15, 10), rf(Vn, 5), rf(Vd, 0);          \
2843   }
2844 
2845   INSN(xar, 0b100);
2846 
2847 #undef INSN
2848 
2849 #define INSN(NAME, opc)                                                                           \
2850   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) {           \
2851     starti;                                                                                       \
2852     assert(T == T2D, "arrangement must be T2D");                                                  \
2853     f(0b11001110, 31, 24), f(opc, 23, 21), rf(Vm, 16), f(0b100011, 15, 10), rf(Vn, 5), rf(Vd, 0); \
2854   }
2855 
2856   INSN(rax1, 0b011);
2857 
2858 #undef INSN
2859 
2860 #define INSN(NAME, opc)                           \
2861   void NAME(FloatRegister Vd, FloatRegister Vn) { \
2862     starti;                                       \
2863     f(opc, 31, 10), rf(Vn, 5), rf(Vd, 0);         \
2864   }
2865 
2866   INSN(aese,   0b0100111000101000010010);
2867   INSN(aesd,   0b0100111000101000010110);
2868   INSN(aesmc,  0b0100111000101000011010);
2869   INSN(aesimc, 0b0100111000101000011110);
2870 
2871 #undef INSN
2872 
2873 #define INSN(NAME, op1, op2) \
2874   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm, int index = 0) { \
2875     starti;                                                                                            \
2876     assert(T == T2S || T == T4S || T == T2D, "invalid arrangement");                                   \
2877     assert(index >= 0 && ((T == T2D && index <= 1) || (T != T2D && index <= 3)), "invalid index");     \
2878     f(0, 31), f((int)T & 1, 30), f(op1, 29); f(0b011111, 28, 23);                                      \
2879     f(T == T2D ? 1 : 0, 22), f(T == T2D ? 0 : index & 1, 21), rf(Vm, 16);                              \
2880     f(op2, 15, 12), f(T == T2D ? index : (index >> 1), 11), f(0, 10);                                  \
2881     rf(Vn, 5), rf(Vd, 0);                                                                              \
2882   }
2883 
2884   // FMLA/FMLS - Vector - Scalar
2885   INSN(fmlavs, 0, 0b0001);
2886   INSN(fmlsvs, 0, 0b0101);
2887   // FMULX - Vector - Scalar
2888   INSN(fmulxvs, 1, 0b1001);
2889 
2890 #undef INSN
2891 
2892   // Floating-point Reciprocal Estimate
2893   void frecpe(FloatRegister Vd, FloatRegister Vn, SIMD_RegVariant type) {
2894     assert(type == D || type == S, "Wrong type for frecpe");
2895     starti;
2896     f(0b010111101, 31, 23);
2897     f(type == D ? 1 : 0, 22);
2898     f(0b100001110110, 21, 10);
2899     rf(Vn, 5), rf(Vd, 0);
2900   }
2901 
2902   // (long) {a, b} -> (a + b)
2903   void addpd(FloatRegister Vd, FloatRegister Vn) {
2904     starti;
2905     f(0b0101111011110001101110, 31, 10);
2906     rf(Vn, 5), rf(Vd, 0);
2907   }
2908 
2909   // Floating-point AdvSIMD scalar pairwise
2910 #define INSN(NAME, op1, op2) \
2911   void NAME(FloatRegister Vd, FloatRegister Vn, SIMD_RegVariant type) {                 \
2912     starti;                                                                             \
2913     assert(type == D || type == S, "Wrong type for faddp/fmaxp/fminp");                 \
2914     f(0b0111111, 31, 25), f(op1, 24, 23),                                               \
2915     f(type == S ? 0 : 1, 22), f(0b11000, 21, 17), f(op2, 16, 10), rf(Vn, 5), rf(Vd, 0); \
2916   }
2917 
2918   INSN(faddp, 0b00, 0b0110110);
2919   INSN(fmaxp, 0b00, 0b0111110);
2920   INSN(fminp, 0b01, 0b0111110);
2921 
2922 #undef INSN
2923 
2924   void ins(FloatRegister Vd, SIMD_RegVariant T, FloatRegister Vn, int didx, int sidx) {
2925     starti;
2926     assert(T != Q, "invalid register variant");
2927     f(0b01101110000, 31, 21), f(((didx<<1)|1)<<(int)T, 20, 16), f(0, 15);
2928     f(sidx<<(int)T, 14, 11), f(1, 10), rf(Vn, 5), rf(Vd, 0);
2929   }
2930 
2931 #define INSN(NAME, cond, op1, op2)                                                      \
2932   void NAME(Register Rd, FloatRegister Vn, SIMD_RegVariant T, int idx) {                \
2933     starti;                                                                             \
2934     assert(cond, "invalid register variant");                                           \
2935     f(0, 31), f(op1, 30), f(0b001110000, 29, 21);                                       \
2936     f(((idx << 1) | 1) << (int)T, 20, 16), f(op2, 15, 10);                              \
2937     rf(Vn, 5), rf(Rd, 0);                                                               \
2938   }
2939 
2940   INSN(umov, (T != Q), (T == D ? 1 : 0), 0b001111);
2941   INSN(smov, (T < D),  1,                0b001011);
2942 
2943 #undef INSN
2944 
2945 #define INSN(NAME, opc, opc2, isSHR)                                    \
2946   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, int shift){ \
2947     starti;                                                             \
2948     /* The encodings for the immh:immb fields (bits 22:16) in *SHR are  \
2949      *   0001 xxx       8B/16B, shift = 16  - UInt(immh:immb)           \
2950      *   001x xxx       4H/8H,  shift = 32  - UInt(immh:immb)           \
2951      *   01xx xxx       2S/4S,  shift = 64  - UInt(immh:immb)           \
2952      *   1xxx xxx       1D/2D,  shift = 128 - UInt(immh:immb)           \
2953      *   (1D is RESERVED)                                               \
2954      * for SHL shift is calculated as:                                  \
2955      *   0001 xxx       8B/16B, shift = UInt(immh:immb) - 8             \
2956      *   001x xxx       4H/8H,  shift = UInt(immh:immb) - 16            \
2957      *   01xx xxx       2S/4S,  shift = UInt(immh:immb) - 32            \
2958      *   1xxx xxx       1D/2D,  shift = UInt(immh:immb) - 64            \
2959      *   (1D is RESERVED)                                               \
2960      */                                                                 \
2961     guarantee(!isSHR || (isSHR && (shift != 0)), "impossible encoding");\
2962     assert((1 << ((T>>1)+3)) > shift, "Invalid Shift value");           \
2963     int cVal = (1 << (((T >> 1) + 3) + (isSHR ? 1 : 0)));               \
2964     int encodedShift = isSHR ? cVal - shift : cVal + shift;             \
2965     f(0, 31), f(T & 1, 30), f(opc, 29), f(0b011110, 28, 23),            \
2966     f(encodedShift, 22, 16); f(opc2, 15, 10), rf(Vn, 5), rf(Vd, 0);     \
2967   }
2968 
2969   INSN(shl,  0, 0b010101, /* isSHR = */ false);
2970   INSN(sshr, 0, 0b000001, /* isSHR = */ true);
2971   INSN(ushr, 1, 0b000001, /* isSHR = */ true);
2972   INSN(usra, 1, 0b000101, /* isSHR = */ true);
2973   INSN(ssra, 0, 0b000101, /* isSHR = */ true);
2974   INSN(sli,  1, 0b010101, /* isSHR = */ false);
2975 
2976 #undef INSN
2977 
2978 #define INSN(NAME, opc, opc2, isSHR)                                    \
2979   void NAME(FloatRegister Vd, FloatRegister Vn, int shift){             \
2980     starti;                                                             \
2981     int encodedShift = isSHR ? 128 - shift : 64 + shift;                \
2982     f(0b01, 31, 30), f(opc, 29), f(0b111110, 28, 23),                   \
2983     f(encodedShift, 22, 16); f(opc2, 15, 10), rf(Vn, 5), rf(Vd, 0);     \
2984   }
2985 
2986   INSN(shld,  0, 0b010101, /* isSHR = */ false);
2987   INSN(sshrd, 0, 0b000001, /* isSHR = */ true);
2988   INSN(ushrd, 1, 0b000001, /* isSHR = */ true);
2989 
2990 #undef INSN
2991 
2992 private:
2993   void _xshll(sign_kind sign, FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn, SIMD_Arrangement Tb, int shift) {
2994     starti;
2995     /* The encodings for the immh:immb fields (bits 22:16) are
2996      *   0001 xxx       8H, 8B/16B shift = xxx
2997      *   001x xxx       4S, 4H/8H  shift = xxxx
2998      *   01xx xxx       2D, 2S/4S  shift = xxxxx
2999      *   1xxx xxx       RESERVED
3000      */
3001     assert((Tb >> 1) + 1 == (Ta >> 1), "Incompatible arrangement");
3002     assert((1 << ((Tb>>1)+3)) > shift, "Invalid shift value");
3003     f(0, 31), f(Tb & 1, 30), f(sign == SIGNED ? 0 : 1, 29), f(0b011110, 28, 23);
3004     f((1 << ((Tb>>1)+3))|shift, 22, 16);
3005     f(0b101001, 15, 10), rf(Vn, 5), rf(Vd, 0);
3006   }
3007 
3008 public:
3009   void ushll(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn,  SIMD_Arrangement Tb, int shift) {
3010     assert(Tb == T8B || Tb == T4H || Tb == T2S, "invalid arrangement");
3011     _xshll(UNSIGNED, Vd, Ta, Vn, Tb, shift);
3012   }
3013 
3014   void ushll2(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn,  SIMD_Arrangement Tb, int shift) {
3015     assert(Tb == T16B || Tb == T8H || Tb == T4S, "invalid arrangement");
3016     _xshll(UNSIGNED, Vd, Ta, Vn, Tb, shift);
3017   }
3018 
3019   void uxtl(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn,  SIMD_Arrangement Tb) {
3020     ushll(Vd, Ta, Vn, Tb, 0);
3021   }
3022 
3023   void sshll(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn,  SIMD_Arrangement Tb, int shift) {
3024     assert(Tb == T8B || Tb == T4H || Tb == T2S, "invalid arrangement");
3025     _xshll(SIGNED, Vd, Ta, Vn, Tb, shift);
3026   }
3027 
3028   void sshll2(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn,  SIMD_Arrangement Tb, int shift) {
3029     assert(Tb == T16B || Tb == T8H || Tb == T4S, "invalid arrangement");
3030     _xshll(SIGNED, Vd, Ta, Vn, Tb, shift);
3031   }
3032 
3033   void sxtl(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn,  SIMD_Arrangement Tb) {
3034     sshll(Vd, Ta, Vn, Tb, 0);
3035   }
3036 
3037   // Move from general purpose register
3038   //   mov  Vd.T[index], Rn
3039   void mov(FloatRegister Vd, SIMD_RegVariant T, int index, Register Xn) {
3040     guarantee(T != Q, "invalid register variant");
3041     starti;
3042     f(0b01001110000, 31, 21), f(((1 << T) | (index << (T + 1))), 20, 16);
3043     f(0b000111, 15, 10), zrf(Xn, 5), rf(Vd, 0);
3044   }
3045 
3046   // Move to general purpose register
3047   //   mov  Rd, Vn.T[index]
3048   void mov(Register Xd, FloatRegister Vn, SIMD_RegVariant T, int index) {
3049     guarantee(T == S || T == D, "invalid register variant");
3050     umov(Xd, Vn, T, index);
3051   }
3052 
3053 private:
3054   void _pmull(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn, FloatRegister Vm, SIMD_Arrangement Tb) {
3055     starti;
3056     assert((Ta == T1Q && (Tb == T1D || Tb == T2D)) ||
3057            (Ta == T8H && (Tb == T8B || Tb == T16B)), "Invalid Size specifier");
3058     int size = (Ta == T1Q) ? 0b11 : 0b00;
3059     f(0, 31), f(Tb & 1, 30), f(0b001110, 29, 24), f(size, 23, 22);
3060     f(1, 21), rf(Vm, 16), f(0b111000, 15, 10), rf(Vn, 5), rf(Vd, 0);
3061   }
3062 
3063 public:
3064   void pmull(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn, FloatRegister Vm, SIMD_Arrangement Tb) {
3065     assert(Tb == T1D || Tb == T8B, "pmull assumes T1D or T8B as the second size specifier");
3066     _pmull(Vd, Ta, Vn, Vm, Tb);
3067   }
3068 
3069   void pmull2(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn, FloatRegister Vm, SIMD_Arrangement Tb) {
3070     assert(Tb == T2D || Tb == T16B, "pmull2 assumes T2D or T16B as the second size specifier");
3071     _pmull(Vd, Ta, Vn, Vm, Tb);
3072   }
3073 
3074   void uqxtn(FloatRegister Vd, SIMD_Arrangement Tb, FloatRegister Vn, SIMD_Arrangement Ta) {
3075     starti;
3076     int size_b = (int)Tb >> 1;
3077     int size_a = (int)Ta >> 1;
3078     assert(size_b < 3 && size_b == size_a - 1, "Invalid size specifier");
3079     f(0, 31), f(Tb & 1, 30), f(0b101110, 29, 24), f(size_b, 23, 22);
3080     f(0b100001010010, 21, 10), rf(Vn, 5), rf(Vd, 0);
3081   }
3082 
3083   void xtn(FloatRegister Vd, SIMD_Arrangement Tb, FloatRegister Vn, SIMD_Arrangement Ta) {
3084     starti;
3085     int size_b = (int)Tb >> 1;
3086     int size_a = (int)Ta >> 1;
3087     assert(size_b < 3 && size_b == size_a - 1, "Invalid size specifier");
3088     f(0, 31), f(Tb & 1, 30), f(0b001110, 29, 24), f(size_b, 23, 22);
3089     f(0b100001001010, 21, 10), rf(Vn, 5), rf(Vd, 0);
3090   }
3091 
3092   void dup(FloatRegister Vd, SIMD_Arrangement T, Register Xs)
3093   {
3094     starti;
3095     assert(T != T1D, "reserved encoding");
3096     f(0,31), f((int)T & 1, 30), f(0b001110000, 29, 21);
3097     f((1 << (T >> 1)), 20, 16), f(0b000011, 15, 10), zrf(Xs, 5), rf(Vd, 0);
3098   }
3099 
3100   void dup(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, int index = 0)
3101   {
3102     starti;
3103     assert(T != T1D, "reserved encoding");
3104     f(0, 31), f((int)T & 1, 30), f(0b001110000, 29, 21);
3105     f(((1 << (T >> 1)) | (index << ((T >> 1) + 1))), 20, 16);
3106     f(0b000001, 15, 10), rf(Vn, 5), rf(Vd, 0);
3107   }
3108 
3109   // Advanced SIMD scalar copy
3110   void dup(FloatRegister Vd, SIMD_RegVariant T, FloatRegister Vn, int index = 0)
3111   {
3112     starti;
3113     assert(T != Q, "invalid size");
3114     f(0b01011110000, 31, 21);
3115     f((1 << T) | (index << (T + 1)), 20, 16);
3116     f(0b000001, 15, 10), rf(Vn, 5), rf(Vd, 0);
3117   }
3118 
3119   // AdvSIMD ZIP/UZP/TRN
3120 #define INSN(NAME, opcode)                                              \
3121   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \
3122     guarantee(T != T1D && T != T1Q, "invalid arrangement");             \
3123     starti;                                                             \
3124     f(0, 31), f(0b001110, 29, 24), f(0, 21), f(0, 15);                  \
3125     f(opcode, 14, 12), f(0b10, 11, 10);                                 \
3126     rf(Vm, 16), rf(Vn, 5), rf(Vd, 0);                                   \
3127     f(T & 1, 30), f(T >> 1, 23, 22);                                    \
3128   }
3129 
3130   INSN(uzp1, 0b001);
3131   INSN(trn1, 0b010);
3132   INSN(zip1, 0b011);
3133   INSN(uzp2, 0b101);
3134   INSN(trn2, 0b110);
3135   INSN(zip2, 0b111);
3136 
3137 #undef INSN
3138 
3139   // CRC32 instructions
3140 #define INSN(NAME, c, sf, sz)                                             \
3141   void NAME(Register Rd, Register Rn, Register Rm) {                      \
3142     starti;                                                               \
3143     f(sf, 31), f(0b0011010110, 30, 21), f(0b010, 15, 13), f(c, 12);       \
3144     f(sz, 11, 10), rf(Rm, 16), rf(Rn, 5), rf(Rd, 0);                      \
3145   }
3146 
3147   INSN(crc32b,  0, 0, 0b00);
3148   INSN(crc32h,  0, 0, 0b01);
3149   INSN(crc32w,  0, 0, 0b10);
3150   INSN(crc32x,  0, 1, 0b11);
3151   INSN(crc32cb, 1, 0, 0b00);
3152   INSN(crc32ch, 1, 0, 0b01);
3153   INSN(crc32cw, 1, 0, 0b10);
3154   INSN(crc32cx, 1, 1, 0b11);
3155 
3156 #undef INSN
3157 
3158   // Table vector lookup
3159 #define INSN(NAME, op)                                                  \
3160   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, unsigned registers, FloatRegister Vm) { \
3161     starti;                                                             \
3162     assert(T == T8B || T == T16B, "invalid arrangement");               \
3163     assert(0 < registers && registers <= 4, "invalid number of registers"); \
3164     f(0, 31), f((int)T & 1, 30), f(0b001110000, 29, 21), rf(Vm, 16), f(0, 15); \
3165     f(registers - 1, 14, 13), f(op, 12),f(0b00, 11, 10), rf(Vn, 5), rf(Vd, 0); \
3166   }
3167 
3168   INSN(tbl, 0);
3169   INSN(tbx, 1);
3170 
3171 #undef INSN
3172 
3173   // AdvSIMD two-reg misc
3174   // In this instruction group, the 2 bits in the size field ([23:22]) may be
3175   // fixed or determined by the "SIMD_Arrangement T", or both. The additional
3176   // parameter "tmask" is a 2-bit mask used to indicate which bits in the size
3177   // field are determined by the SIMD_Arrangement. The bit of "tmask" should be
3178   // set to 1 if corresponding bit marked as "x" in the ArmARM.
3179 #define INSN(NAME, U, size, tmask, opcode)                                          \
3180   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {               \
3181        starti;                                                                      \
3182        assert((ASSERTION), MSG);                                                    \
3183        f(0, 31), f((int)T & 1, 30), f(U, 29), f(0b01110, 28, 24);                   \
3184        f(size | ((int)(T >> 1) & tmask), 23, 22), f(0b10000, 21, 17);               \
3185        f(opcode, 16, 12), f(0b10, 11, 10), rf(Vn, 5), rf(Vd, 0);                    \
3186  }
3187 
3188 #define MSG "invalid arrangement"
3189 
3190 #define ASSERTION (T == T2S || T == T4S || T == T2D)
3191   INSN(fsqrt,  1, 0b10, 0b01, 0b11111);
3192   INSN(fabs,   0, 0b10, 0b01, 0b01111);
3193   INSN(fneg,   1, 0b10, 0b01, 0b01111);
3194   INSN(frintn, 0, 0b00, 0b01, 0b11000);
3195   INSN(frintm, 0, 0b00, 0b01, 0b11001);
3196   INSN(frintp, 0, 0b10, 0b01, 0b11000);
3197   INSN(fcvtas, 0, 0b00, 0b01, 0b11100);
3198   INSN(fcvtzs, 0, 0b10, 0b01, 0b11011);
3199   INSN(fcvtms, 0, 0b00, 0b01, 0b11011);
3200 #undef ASSERTION
3201 
3202 #define ASSERTION (T == T8B || T == T16B || T == T4H || T == T8H || T == T2S || T == T4S)
3203   INSN(rev64, 0, 0b00, 0b11, 0b00000);
3204 #undef ASSERTION
3205 
3206 #define ASSERTION (T == T8B || T == T16B || T == T4H || T == T8H)
3207   INSN(rev32, 1, 0b00, 0b11, 0b00000);
3208 #undef ASSERTION
3209 
3210 #define ASSERTION (T == T8B || T == T16B)
3211   INSN(rev16, 0, 0b00, 0b11, 0b00001);
3212   INSN(rbit,  1, 0b01, 0b00, 0b00101);
3213 #undef ASSERTION
3214 
3215 #undef MSG
3216 
3217 #undef INSN
3218 
3219   // AdvSIMD compare with zero (vector)
3220   void cm(Condition cond, FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {
3221     starti;
3222     assert(T != T1Q && T != T1D, "invalid arrangement");
3223     int cond_op;
3224     switch (cond) {
3225       case EQ: cond_op = 0b001; break;
3226       case GE: cond_op = 0b100; break;
3227       case GT: cond_op = 0b000; break;
3228       case LE: cond_op = 0b101; break;
3229       case LT: cond_op = 0b010; break;
3230       default:
3231         ShouldNotReachHere();
3232         break;
3233     }
3234 
3235     f(0, 31), f((int)T & 1, 30), f((cond_op >> 2) & 1, 29);
3236     f(0b01110, 28, 24), f((int)T >> 1, 23, 22), f(0b10000010, 21, 14);
3237     f(cond_op & 0b11, 13, 12), f(0b10, 11, 10), rf(Vn, 5), rf(Vd, 0);
3238   }
3239 
3240   // AdvSIMD Floating-point compare with zero (vector)
3241   void fcm(Condition cond, FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {
3242     starti;
3243     assert(T == T2S || T == T4S || T == T2D, "invalid arrangement");
3244     int cond_op;
3245     switch (cond) {
3246       case EQ: cond_op = 0b010; break;
3247       case GT: cond_op = 0b000; break;
3248       case GE: cond_op = 0b001; break;
3249       case LE: cond_op = 0b011; break;
3250       case LT: cond_op = 0b100; break;
3251       default:
3252         ShouldNotReachHere();
3253         break;
3254     }
3255 
3256     f(0, 31), f((int)T & 1, 30), f(cond_op & 1, 29), f(0b011101, 28, 23);
3257     f(((int)(T >> 1) & 1), 22), f(0b10000011, 21, 14);
3258     f((cond_op >> 1) & 0b11, 13, 12), f(0b10, 11, 10), rf(Vn, 5), rf(Vd, 0);
3259   }
3260 
3261   void ext(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm, int index)
3262   {
3263     starti;
3264     assert(T == T8B || T == T16B, "invalid arrangement");
3265     assert((T == T8B && index <= 0b0111) || (T == T16B && index <= 0b1111), "Invalid index value");
3266     f(0, 31), f((int)T & 1, 30), f(0b101110000, 29, 21);
3267     rf(Vm, 16), f(0, 15), f(index, 14, 11);
3268     f(0, 10), rf(Vn, 5), rf(Vd, 0);
3269   }
3270 
3271 // SVE arithmetic - unpredicated
3272 #define INSN(NAME, opcode)                                                             \
3273   void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, FloatRegister Zm) { \
3274     starti;                                                                            \
3275     assert(T != Q, "invalid register variant");                                        \
3276     f(0b00000100, 31, 24), f(T, 23, 22), f(1, 21),                                     \
3277     rf(Zm, 16), f(0, 15, 13), f(opcode, 12, 10), rf(Zn, 5), rf(Zd, 0);                 \
3278   }
3279   INSN(sve_add, 0b000);
3280   INSN(sve_sub, 0b001);
3281 #undef INSN
3282 
3283 // SVE integer add/subtract immediate (unpredicated)
3284 #define INSN(NAME, op)                                                  \
3285   void NAME(FloatRegister Zd, SIMD_RegVariant T, unsigned imm8) {       \
3286     starti;                                                             \
3287     /* The immediate is an unsigned value in the range 0 to 255, and    \
3288      * for element width of 16 bits or higher it may also be a          \
3289      * positive multiple of 256 in the range 256 to 65280.              \
3290      */                                                                 \
3291     assert(T != Q, "invalid size");                                     \
3292     int sh = 0;                                                         \
3293     if (imm8 <= 0xff) {                                                 \
3294       sh = 0;                                                           \
3295     } else if (T != B && imm8 <= 0xff00 && (imm8 & 0xff) == 0) {        \
3296       sh = 1;                                                           \
3297       imm8 = (imm8 >> 8);                                               \
3298     } else {                                                            \
3299       guarantee(false, "invalid immediate");                            \
3300     }                                                                   \
3301     f(0b00100101, 31, 24), f(T, 23, 22), f(0b10000, 21, 17);            \
3302     f(op, 16, 14), f(sh, 13), f(imm8, 12, 5), rf(Zd, 0);                \
3303   }
3304 
3305   INSN(sve_add, 0b011);
3306   INSN(sve_sub, 0b111);
3307 #undef INSN
3308 
3309 // SVE floating-point arithmetic - unpredicated
3310 #define INSN(NAME, opcode)                                                             \
3311   void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, FloatRegister Zm) { \
3312     starti;                                                                            \
3313     assert(T == S || T == D, "invalid register variant");                              \
3314     f(0b01100101, 31, 24), f(T, 23, 22), f(0, 21),                                     \
3315     rf(Zm, 16), f(0, 15, 13), f(opcode, 12, 10), rf(Zn, 5), rf(Zd, 0);                 \
3316   }
3317 
3318   INSN(sve_fadd, 0b000);
3319   INSN(sve_fmul, 0b010);
3320   INSN(sve_fsub, 0b001);
3321 #undef INSN
3322 
3323 private:
3324   void sve_predicate_reg_insn(unsigned op24, unsigned op13,
3325                               FloatRegister Zd_or_Vd, SIMD_RegVariant T,
3326                               PRegister Pg, FloatRegister Zn_or_Vn) {
3327     starti;
3328     f(op24, 31, 24), f(T, 23, 22), f(op13, 21, 13);
3329     pgrf(Pg, 10), rf(Zn_or_Vn, 5), rf(Zd_or_Vd, 0);
3330   }
3331 
3332   void sve_shift_imm_encoding(SIMD_RegVariant T, int shift, bool isSHR,
3333                               int& tszh, int& tszl_imm) {
3334     /* The encodings for the tszh:tszl:imm3 fields
3335      * for shift right is calculated as:
3336      *   0001 xxx       B, shift = 16  - UInt(tszh:tszl:imm3)
3337      *   001x xxx       H, shift = 32  - UInt(tszh:tszl:imm3)
3338      *   01xx xxx       S, shift = 64  - UInt(tszh:tszl:imm3)
3339      *   1xxx xxx       D, shift = 128 - UInt(tszh:tszl:imm3)
3340      * for shift left is calculated as:
3341      *   0001 xxx       B, shift = UInt(tszh:tszl:imm3) - 8
3342      *   001x xxx       H, shift = UInt(tszh:tszl:imm3) - 16
3343      *   01xx xxx       S, shift = UInt(tszh:tszl:imm3) - 32
3344      *   1xxx xxx       D, shift = UInt(tszh:tszl:imm3) - 64
3345      */
3346     assert(T != Q, "Invalid register variant");
3347     if (isSHR) {
3348       assert(((1 << (T + 3)) >= shift) && (shift > 0) , "Invalid shift value");
3349     } else {
3350       assert(((1 << (T + 3)) > shift) && (shift >= 0) , "Invalid shift value");
3351     }
3352     int cVal = (1 << ((T + 3) + (isSHR ? 1 : 0)));
3353     int encodedShift = isSHR ? cVal - shift : cVal + shift;
3354     tszh = encodedShift >> 5;
3355     tszl_imm = encodedShift & 0x1f;
3356   }
3357 
3358 public:
3359 
3360 // SVE integer arithmetic - predicate
3361 #define INSN(NAME, op1, op2)                                                                            \
3362   void NAME(FloatRegister Zdn_or_Zd_or_Vd, SIMD_RegVariant T, PRegister Pg, FloatRegister Znm_or_Vn) {  \
3363     assert(T != Q, "invalid register variant");                                                         \
3364     sve_predicate_reg_insn(op1, op2, Zdn_or_Zd_or_Vd, T, Pg, Znm_or_Vn);                                \
3365   }
3366 
3367   INSN(sve_abs,   0b00000100, 0b010110101); // vector abs, unary
3368   INSN(sve_add,   0b00000100, 0b000000000); // vector add
3369   INSN(sve_and,   0b00000100, 0b011010000); // vector and
3370   INSN(sve_andv,  0b00000100, 0b011010001); // bitwise and reduction to scalar
3371   INSN(sve_asr,   0b00000100, 0b010000100); // vector arithmetic shift right
3372   INSN(sve_bic,   0b00000100, 0b011011000); // vector bitwise clear
3373   INSN(sve_clz,   0b00000100, 0b011001101); // vector count leading zero bits
3374   INSN(sve_cnt,   0b00000100, 0b011010101); // count non-zero bits
3375   INSN(sve_cpy,   0b00000101, 0b100000100); // copy scalar to each active vector element
3376   INSN(sve_eor,   0b00000100, 0b011001000); // vector eor
3377   INSN(sve_eorv,  0b00000100, 0b011001001); // bitwise xor reduction to scalar
3378   INSN(sve_lsl,   0b00000100, 0b010011100); // vector logical shift left
3379   INSN(sve_lsr,   0b00000100, 0b010001100); // vector logical shift right
3380   INSN(sve_mul,   0b00000100, 0b010000000); // vector mul
3381   INSN(sve_neg,   0b00000100, 0b010111101); // vector neg, unary
3382   INSN(sve_not,   0b00000100, 0b011110101); // bitwise invert vector, unary
3383   INSN(sve_orr,   0b00000100, 0b011000000); // vector or
3384   INSN(sve_orv,   0b00000100, 0b011000001); // bitwise or reduction to scalar
3385   INSN(sve_smax,  0b00000100, 0b001000000); // signed maximum vectors
3386   INSN(sve_smaxv, 0b00000100, 0b001000001); // signed maximum reduction to scalar
3387   INSN(sve_smin,  0b00000100, 0b001010000); // signed minimum vectors
3388   INSN(sve_sminv, 0b00000100, 0b001010001); // signed minimum reduction to scalar
3389   INSN(sve_sub,   0b00000100, 0b000001000); // vector sub
3390   INSN(sve_uaddv, 0b00000100, 0b000001001); // unsigned add reduction to scalar
3391 #undef INSN
3392 
3393 // SVE floating-point arithmetic - predicate
3394 #define INSN(NAME, op1, op2)                                                                          \
3395   void NAME(FloatRegister Zd_or_Zdn_or_Vd, SIMD_RegVariant T, PRegister Pg, FloatRegister Zn_or_Zm) { \
3396     assert(T == S || T == D, "invalid register variant");                                             \
3397     sve_predicate_reg_insn(op1, op2, Zd_or_Zdn_or_Vd, T, Pg, Zn_or_Zm);                               \
3398   }
3399 
3400   INSN(sve_fabd,   0b01100101, 0b001000100); // floating-point absolute difference
3401   INSN(sve_fabs,   0b00000100, 0b011100101);
3402   INSN(sve_fadd,   0b01100101, 0b000000100);
3403   INSN(sve_fadda,  0b01100101, 0b011000001); // add strictly-ordered reduction to scalar Vd
3404   INSN(sve_fdiv,   0b01100101, 0b001101100);
3405   INSN(sve_fmax,   0b01100101, 0b000110100); // floating-point maximum
3406   INSN(sve_fmaxv,  0b01100101, 0b000110001); // floating-point maximum recursive reduction to scalar
3407   INSN(sve_fmin,   0b01100101, 0b000111100); // floating-point minimum
3408   INSN(sve_fminv,  0b01100101, 0b000111001); // floating-point minimum recursive reduction to scalar
3409   INSN(sve_fmul,   0b01100101, 0b000010100);
3410   INSN(sve_fneg,   0b00000100, 0b011101101);
3411   INSN(sve_frintm, 0b01100101, 0b000010101); // floating-point round to integral value, toward minus infinity
3412   INSN(sve_frintn, 0b01100101, 0b000000101); // floating-point round to integral value, nearest with ties to even
3413   INSN(sve_frinta, 0b01100101, 0b000100101); // floating-point round to integral value, nearest with ties to away
3414   INSN(sve_frintp, 0b01100101, 0b000001101); // floating-point round to integral value, toward plus infinity
3415   INSN(sve_fsqrt,  0b01100101, 0b001101101);
3416   INSN(sve_fsub,   0b01100101, 0b000001100);
3417 #undef INSN
3418 
3419   // SVE multiple-add/sub - predicated
3420 #define INSN(NAME, op0, op1, op2)                                                                     \
3421   void NAME(FloatRegister Zda, SIMD_RegVariant T, PRegister Pg, FloatRegister Zn, FloatRegister Zm) { \
3422     starti;                                                                                           \
3423     assert(T != Q, "invalid size");                                                                   \
3424     f(op0, 31, 24), f(T, 23, 22), f(op1, 21), rf(Zm, 16);                                             \
3425     f(op2, 15, 13), pgrf(Pg, 10), rf(Zn, 5), rf(Zda, 0);                                              \
3426   }
3427 
3428   INSN(sve_fmla,  0b01100101, 1, 0b000); // floating-point fused multiply-add, writing addend: Zda = Zda + Zn * Zm
3429   INSN(sve_fmls,  0b01100101, 1, 0b001); // floating-point fused multiply-subtract: Zda = Zda + -Zn * Zm
3430   INSN(sve_fnmla, 0b01100101, 1, 0b010); // floating-point negated fused multiply-add: Zda = -Zda + -Zn * Zm
3431   INSN(sve_fnmls, 0b01100101, 1, 0b011); // floating-point negated fused multiply-subtract: Zda = -Zda + Zn * Zm
3432   INSN(sve_fmad,  0b01100101, 1, 0b100); // floating-point fused multiply-add, writing multiplicand: Zda = Zm + Zda * Zn
3433   INSN(sve_fmsb,  0b01100101, 1, 0b101); // floating-point fused multiply-subtract, writing multiplicand: Zda = Zm + -Zda * Zn
3434   INSN(sve_fnmad, 0b01100101, 1, 0b110); // floating-point negated fused multiply-add, writing multiplicand: Zda = -Zm + -Zda * Zn
3435   INSN(sve_fnmsb, 0b01100101, 1, 0b111); // floating-point negated fused multiply-subtract, writing multiplicand: Zda = -Zm + Zda * Zn
3436   INSN(sve_mla,   0b00000100, 0, 0b010); // multiply-add, writing addend: Zda = Zda + Zn*Zm
3437   INSN(sve_mls,   0b00000100, 0, 0b011); // multiply-subtract, writing addend: Zda = Zda + -Zn*Zm
3438 #undef INSN
3439 
3440 // SVE bitwise logical - unpredicated
3441 #define INSN(NAME, opc)                                              \
3442   void NAME(FloatRegister Zd, FloatRegister Zn, FloatRegister Zm) {  \
3443     starti;                                                          \
3444     f(0b00000100, 31, 24), f(opc, 23, 22), f(1, 21),                 \
3445     rf(Zm, 16), f(0b001100, 15, 10), rf(Zn, 5), rf(Zd, 0);           \
3446   }
3447   INSN(sve_and, 0b00);
3448   INSN(sve_eor, 0b10);
3449   INSN(sve_orr, 0b01);
3450   INSN(sve_bic, 0b11);
3451 #undef INSN
3452 
3453 // SVE bitwise logical with immediate (unpredicated)
3454 #define INSN(NAME, opc)                                                      \
3455   void NAME(FloatRegister Zd, SIMD_RegVariant T, uint64_t imm) {             \
3456     starti;                                                                  \
3457     unsigned elembits = regVariant_to_elemBits(T);                           \
3458     uint32_t val = encode_sve_logical_immediate(elembits, imm);              \
3459     f(0b00000101, 31, 24), f(opc, 23, 22), f(0b0000, 21, 18);                \
3460     f(val, 17, 5), rf(Zd, 0);                                                \
3461   }
3462   INSN(sve_and, 0b10);
3463   INSN(sve_eor, 0b01);
3464   INSN(sve_orr, 0b00);
3465 #undef INSN
3466 
3467 // SVE shift immediate - unpredicated
3468 #define INSN(NAME, opc, isSHR)                                                  \
3469   void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, int shift) { \
3470     starti;                                                                     \
3471     int tszh, tszl_imm;                                                         \
3472     sve_shift_imm_encoding(T, shift, isSHR, tszh, tszl_imm);                    \
3473     f(0b00000100, 31, 24);                                                      \
3474     f(tszh, 23, 22), f(1,21), f(tszl_imm, 20, 16);                              \
3475     f(0b100, 15, 13), f(opc, 12, 10), rf(Zn, 5), rf(Zd, 0);                     \
3476   }
3477 
3478   INSN(sve_asr, 0b100, /* isSHR = */ true);
3479   INSN(sve_lsl, 0b111, /* isSHR = */ false);
3480   INSN(sve_lsr, 0b101, /* isSHR = */ true);
3481 #undef INSN
3482 
3483 // SVE bitwise shift by immediate (predicated)
3484 #define INSN(NAME, opc, isSHR)                                                  \
3485   void NAME(FloatRegister Zdn, SIMD_RegVariant T, PRegister Pg, int shift) {    \
3486     starti;                                                                     \
3487     int tszh, tszl_imm;                                                         \
3488     sve_shift_imm_encoding(T, shift, isSHR, tszh, tszl_imm);                    \
3489     f(0b00000100, 31, 24), f(tszh, 23, 22), f(0b00, 21, 20), f(opc, 19, 16);    \
3490     f(0b100, 15, 13), pgrf(Pg, 10), f(tszl_imm, 9, 5), rf(Zdn, 0);              \
3491   }
3492 
3493   INSN(sve_asr, 0b0000, /* isSHR = */ true);
3494   INSN(sve_lsl, 0b0011, /* isSHR = */ false);
3495   INSN(sve_lsr, 0b0001, /* isSHR = */ true);
3496 #undef INSN
3497 
3498 private:
3499 
3500   // Scalar base + immediate index
3501   void sve_ld_st1(FloatRegister Zt, Register Xn, int imm, PRegister Pg,
3502               SIMD_RegVariant T, int op1, int type, int op2) {
3503     starti;
3504     assert_cond(T >= type);
3505     f(op1, 31, 25), f(type, 24, 23), f(T, 22, 21);
3506     f(0, 20), sf(imm, 19, 16), f(op2, 15, 13);
3507     pgrf(Pg, 10), srf(Xn, 5), rf(Zt, 0);
3508   }
3509 
3510   // Scalar base + scalar index
3511   void sve_ld_st1(FloatRegister Zt, Register Xn, Register Xm, PRegister Pg,
3512               SIMD_RegVariant T, int op1, int type, int op2) {
3513     starti;
3514     assert_cond(T >= type);
3515     f(op1, 31, 25), f(type, 24, 23), f(T, 22, 21);
3516     rf(Xm, 16), f(op2, 15, 13);
3517     pgrf(Pg, 10), srf(Xn, 5), rf(Zt, 0);
3518   }
3519 
3520   void sve_ld_st1(FloatRegister Zt, PRegister Pg,
3521               SIMD_RegVariant T, const Address &a,
3522               int op1, int type, int imm_op2, int scalar_op2) {
3523     switch (a.getMode()) {
3524     case Address::base_plus_offset:
3525       sve_ld_st1(Zt, a.base(), a.offset(), Pg, T, op1, type, imm_op2);
3526       break;
3527     case Address::base_plus_offset_reg:
3528       sve_ld_st1(Zt, a.base(), a.index(), Pg, T, op1, type, scalar_op2);
3529       break;
3530     default:
3531       ShouldNotReachHere();
3532     }
3533   }
3534 
3535 public:
3536 
3537 // SVE contiguous load/store
3538 #define INSN(NAME, op1, type, imm_op2, scalar_op2)                                   \
3539   void NAME(FloatRegister Zt, SIMD_RegVariant T, PRegister Pg, const Address &a) {   \
3540     assert(T != Q, "invalid register variant");                                      \
3541     sve_ld_st1(Zt, Pg, T, a, op1, type, imm_op2, scalar_op2);                        \
3542   }
3543 
3544   INSN(sve_ld1b, 0b1010010, 0b00, 0b101, 0b010);
3545   INSN(sve_st1b, 0b1110010, 0b00, 0b111, 0b010);
3546   INSN(sve_ld1h, 0b1010010, 0b01, 0b101, 0b010);
3547   INSN(sve_st1h, 0b1110010, 0b01, 0b111, 0b010);
3548   INSN(sve_ld1w, 0b1010010, 0b10, 0b101, 0b010);
3549   INSN(sve_st1w, 0b1110010, 0b10, 0b111, 0b010);
3550   INSN(sve_ld1d, 0b1010010, 0b11, 0b101, 0b010);
3551   INSN(sve_st1d, 0b1110010, 0b11, 0b111, 0b010);
3552 #undef INSN
3553 
3554 // Gather/scatter load/store (SVE) - scalar plus vector
3555 #define INSN(NAME, op1, type, op2, op3)                                         \
3556   void NAME(FloatRegister Zt, PRegister Pg, Register Xn, FloatRegister Zm) {    \
3557     starti;                                                                     \
3558     f(op1, 31, 25), f(type, 24, 23), f(op2, 22, 21), rf(Zm, 16);                \
3559     f(op3, 15, 13), pgrf(Pg, 10), srf(Xn, 5), rf(Zt, 0);                        \
3560   }
3561   // SVE 32-bit gather load words (scalar plus 32-bit scaled offsets)
3562   INSN(sve_ld1w_gather,  0b1000010, 0b10, 0b01, 0b010);
3563   // SVE 64-bit gather load (scalar plus 32-bit unpacked scaled offsets)
3564   INSN(sve_ld1d_gather,  0b1100010, 0b11, 0b01, 0b010);
3565   // SVE 32-bit scatter store (scalar plus 32-bit scaled offsets)
3566   INSN(sve_st1w_scatter, 0b1110010, 0b10, 0b11, 0b100);
3567   // SVE 64-bit scatter store (scalar plus unpacked 32-bit scaled offsets)
3568   INSN(sve_st1d_scatter, 0b1110010, 0b11, 0b01, 0b100);
3569 #undef INSN
3570 
3571 // SVE load/store - unpredicated
3572 #define INSN(NAME, op1)                                                         \
3573   void NAME(FloatRegister Zt, const Address &a)  {                              \
3574     starti;                                                                     \
3575     assert(a.index() == noreg, "invalid address variant");                      \
3576     f(op1, 31, 29), f(0b0010110, 28, 22), sf(a.offset() >> 3, 21, 16),          \
3577     f(0b010, 15, 13), f(a.offset() & 0x7, 12, 10), srf(a.base(), 5), rf(Zt, 0); \
3578   }
3579 
3580   INSN(sve_ldr, 0b100); // LDR (vector)
3581   INSN(sve_str, 0b111); // STR (vector)
3582 #undef INSN
3583 
3584 // SVE stack frame adjustment
3585 #define INSN(NAME, op) \
3586   void NAME(Register Xd, Register Xn, int imm6) {                 \
3587     starti;                                                       \
3588     f(0b000001000, 31, 23), f(op, 22, 21);                        \
3589     srf(Xn, 16), f(0b01010, 15, 11), sf(imm6, 10, 5), srf(Xd, 0); \
3590   }
3591 
3592   INSN(sve_addvl, 0b01); // Add multiple of vector register size to scalar register
3593   INSN(sve_addpl, 0b11); // Add multiple of predicate register size to scalar register
3594 #undef INSN
3595 
3596 // SVE inc/dec register by element count
3597 #define INSN(NAME, op) \
3598   void NAME(Register Xdn, SIMD_RegVariant T, unsigned imm4 = 1, int pattern = 0b11111) { \
3599     starti;                                                                              \
3600     assert(T != Q, "invalid size");                                                      \
3601     f(0b00000100,31, 24), f(T, 23, 22), f(0b11, 21, 20);                                 \
3602     f(imm4 - 1, 19, 16), f(0b11100, 15, 11), f(op, 10), f(pattern, 9, 5), rf(Xdn, 0);    \
3603   }
3604 
3605   INSN(sve_inc, 0);
3606   INSN(sve_dec, 1);
3607 #undef INSN
3608 
3609 // SVE predicate logical operations
3610 #define INSN(NAME, op1, op2, op3) \
3611   void NAME(PRegister Pd, PRegister Pg, PRegister Pn, PRegister Pm) { \
3612     starti;                                                           \
3613     f(0b00100101, 31, 24), f(op1, 23, 22), f(0b00, 21, 20);           \
3614     prf(Pm, 16), f(0b01, 15, 14), prf(Pg, 10), f(op2, 9);             \
3615     prf(Pn, 5), f(op3, 4), prf(Pd, 0);                                \
3616   }
3617 
3618   INSN(sve_and,  0b00, 0b0, 0b0);
3619   INSN(sve_ands, 0b01, 0b0, 0b0);
3620   INSN(sve_eor,  0b00, 0b1, 0b0);
3621   INSN(sve_eors, 0b01, 0b1, 0b0);
3622   INSN(sve_orr,  0b10, 0b0, 0b0);
3623   INSN(sve_orrs, 0b11, 0b0, 0b0);
3624   INSN(sve_bic,  0b00, 0b0, 0b1);
3625 #undef INSN
3626 
3627   // SVE increment register by predicate count
3628   void sve_incp(const Register rd, SIMD_RegVariant T, PRegister pg) {
3629     starti;
3630     assert(T != Q, "invalid size");
3631     f(0b00100101, 31, 24), f(T, 23, 22), f(0b1011001000100, 21, 9),
3632     prf(pg, 5), rf(rd, 0);
3633   }
3634 
3635   // SVE broadcast general-purpose register to vector elements (unpredicated)
3636   void sve_dup(FloatRegister Zd, SIMD_RegVariant T, Register Rn) {
3637     starti;
3638     assert(T != Q, "invalid size");
3639     f(0b00000101, 31, 24), f(T, 23, 22), f(0b100000001110, 21, 10);
3640     srf(Rn, 5), rf(Zd, 0);
3641   }
3642 
3643   // SVE broadcast signed immediate to vector elements (unpredicated)
3644   void sve_dup(FloatRegister Zd, SIMD_RegVariant T, int imm8) {
3645     starti;
3646     assert(T != Q, "invalid size");
3647     int sh = 0;
3648     if (imm8 <= 127 && imm8 >= -128) {
3649       sh = 0;
3650     } else if (T != B && imm8 <= 32512 && imm8 >= -32768 && (imm8 & 0xff) == 0) {
3651       sh = 1;
3652       imm8 = (imm8 >> 8);
3653     } else {
3654       guarantee(false, "invalid immediate");
3655     }
3656     f(0b00100101, 31, 24), f(T, 23, 22), f(0b11100011, 21, 14);
3657     f(sh, 13), sf(imm8, 12, 5), rf(Zd, 0);
3658   }
3659 
3660   // SVE predicate test
3661   void sve_ptest(PRegister Pg, PRegister Pn) {
3662     starti;
3663     f(0b001001010101000011, 31, 14), prf(Pg, 10), f(0, 9), prf(Pn, 5), f(0, 4, 0);
3664   }
3665 
3666   // SVE predicate initialize
3667   void sve_ptrue(PRegister pd, SIMD_RegVariant esize, int pattern = 0b11111) {
3668     starti;
3669     f(0b00100101, 31, 24), f(esize, 23, 22), f(0b011000111000, 21, 10);
3670     f(pattern, 9, 5), f(0b0, 4), prf(pd, 0);
3671   }
3672 
3673   // SVE predicate zero
3674   void sve_pfalse(PRegister pd) {
3675     starti;
3676     f(0b00100101, 31, 24), f(0b00, 23, 22), f(0b011000111001, 21, 10);
3677     f(0b000000, 9, 4), prf(pd, 0);
3678   }
3679 
3680 // SVE load/store predicate register
3681 #define INSN(NAME, op1)                                                  \
3682   void NAME(PRegister Pt, const Address &a)  {                           \
3683     starti;                                                              \
3684     assert(a.index() == noreg, "invalid address variant");               \
3685     f(op1, 31, 29), f(0b0010110, 28, 22), sf(a.offset() >> 3, 21, 16),   \
3686     f(0b000, 15, 13), f(a.offset() & 0x7, 12, 10), srf(a.base(), 5),     \
3687     f(0, 4), prf(Pt, 0);                                                 \
3688   }
3689 
3690   INSN(sve_ldr, 0b100); // LDR (predicate)
3691   INSN(sve_str, 0b111); // STR (predicate)
3692 #undef INSN
3693 
3694   // SVE move predicate register
3695   void sve_mov(PRegister Pd, PRegister Pn) {
3696     starti;
3697     f(0b001001011000, 31, 20), prf(Pn, 16), f(0b01, 15, 14), prf(Pn, 10);
3698     f(0, 9), prf(Pn, 5), f(0, 4), prf(Pd, 0);
3699   }
3700 
3701   // SVE copy general-purpose register to vector elements (predicated)
3702   void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, Register Rn) {
3703     starti;
3704     assert(T != Q, "invalid size");
3705     f(0b00000101, 31, 24), f(T, 23, 22), f(0b101000101, 21, 13);
3706     pgrf(Pg, 10), srf(Rn, 5), rf(Zd, 0);
3707   }
3708 
3709 private:
3710   void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, int imm8,
3711                bool isMerge, bool isFloat) {
3712     starti;
3713     assert(T != Q, "invalid size");
3714     int sh = 0;
3715     if (imm8 <= 127 && imm8 >= -128) {
3716       sh = 0;
3717     } else if (T != B && imm8 <= 32512 && imm8 >= -32768 && (imm8 & 0xff) == 0) {
3718       sh = 1;
3719       imm8 = (imm8 >> 8);
3720     } else {
3721       guarantee(false, "invalid immediate");
3722     }
3723     int m = isMerge ? 1 : 0;
3724     f(0b00000101, 31, 24), f(T, 23, 22), f(0b01, 21, 20);
3725     prf(Pg, 16), f(isFloat ? 1 : 0, 15), f(m, 14), f(sh, 13), sf(imm8, 12, 5), rf(Zd, 0);
3726   }
3727 
3728 public:
3729   // SVE copy signed integer immediate to vector elements (predicated)
3730   void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, int imm8, bool isMerge) {
3731     sve_cpy(Zd, T, Pg, imm8, isMerge, /*isFloat*/false);
3732   }
3733   // SVE copy floating-point immediate to vector elements (predicated)
3734   void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, double d) {
3735     sve_cpy(Zd, T, Pg, checked_cast<int8_t>(pack(d)), /*isMerge*/true, /*isFloat*/true);
3736   }
3737 
3738   // SVE conditionally select elements from two vectors
3739   void sve_sel(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg,
3740                FloatRegister Zn, FloatRegister Zm) {
3741     starti;
3742     assert(T != Q, "invalid size");
3743     f(0b00000101, 31, 24), f(T, 23, 22), f(0b1, 21), rf(Zm, 16);
3744     f(0b11, 15, 14), prf(Pg, 10), rf(Zn, 5), rf(Zd, 0);
3745   }
3746 
3747   // SVE Permute Vector - Extract
3748   void sve_ext(FloatRegister Zdn, FloatRegister Zm, int imm8) {
3749     starti;
3750     f(0b00000101001, 31, 21), f(imm8 >> 3, 20, 16), f(0b000, 15, 13);
3751     f(imm8 & 0b111, 12, 10), rf(Zm, 5), rf(Zdn, 0);
3752   }
3753 
3754 // SVE Integer/Floating-Point Compare - Vectors
3755 #define INSN(NAME, op1, op2, fp)  \
3756   void NAME(Condition cond, PRegister Pd, SIMD_RegVariant T, PRegister Pg,             \
3757             FloatRegister Zn, FloatRegister Zm) {                                      \
3758     starti;                                                                            \
3759     assert(T != Q, "invalid size");                                                    \
3760     bool is_absolute = op2 == 0b11;                                                    \
3761     if (fp == 1) {                                                                     \
3762       assert(T != B, "invalid size");                                                  \
3763       if (is_absolute) {                                                               \
3764         assert(cond == GT || cond == GE, "invalid condition for fac");                 \
3765       } else {                                                                         \
3766         assert(cond != HI && cond != HS, "invalid condition for fcm");                 \
3767       }                                                                                \
3768     }                                                                                  \
3769     int cond_op;                                                                       \
3770     switch(cond) {                                                                     \
3771       case EQ: cond_op = (op2 << 2) | 0b10; break;                                     \
3772       case NE: cond_op = (op2 << 2) | 0b11; break;                                     \
3773       case GE: cond_op = (op2 << 2) | (is_absolute ? 0b01 : 0b00); break;              \
3774       case GT: cond_op = (op2 << 2) | (is_absolute ? 0b11 : 0b01); break;              \
3775       case HI: cond_op = 0b0001; break;                                                \
3776       case HS: cond_op = 0b0000; break;                                                \
3777       default:                                                                         \
3778         ShouldNotReachHere();                                                          \
3779     }                                                                                  \
3780     f(op1, 31, 24), f(T, 23, 22), f(0, 21), rf(Zm, 16), f((cond_op >> 1) & 7, 15, 13); \
3781     pgrf(Pg, 10), rf(Zn, 5), f(cond_op & 1, 4), prf(Pd, 0);                            \
3782   }
3783 
3784   INSN(sve_cmp, 0b00100100, 0b10, 0); // Integer compare vectors
3785   INSN(sve_fcm, 0b01100101, 0b01, 1); // Floating-point compare vectors
3786   INSN(sve_fac, 0b01100101, 0b11, 1); // Floating-point absolute compare vectors
3787 #undef INSN
3788 
3789 private:
3790   // Convert Assembler::Condition to op encoding - used by sve integer compare encoding
3791   static int assembler_cond_to_sve_op(Condition cond, bool &is_unsigned) {
3792     if (cond == HI || cond == HS || cond == LO || cond == LS) {
3793       is_unsigned = true;
3794     } else {
3795       is_unsigned = false;
3796     }
3797 
3798     switch (cond) {
3799       case HI:
3800       case GT:
3801         return 0b0001;
3802       case HS:
3803       case GE:
3804         return 0b0000;
3805       case LO:
3806       case LT:
3807         return 0b0010;
3808       case LS:
3809       case LE:
3810         return 0b0011;
3811       case EQ:
3812         return 0b1000;
3813       case NE:
3814         return 0b1001;
3815       default:
3816         ShouldNotReachHere();
3817         return -1;
3818     }
3819   }
3820 
3821 public:
3822   // SVE Integer Compare - 5 bits signed imm and 7 bits unsigned imm
3823   void sve_cmp(Condition cond, PRegister Pd, SIMD_RegVariant T,
3824                PRegister Pg, FloatRegister Zn, int imm) {
3825     starti;
3826     assert(T != Q, "invalid size");
3827     bool is_unsigned = false;
3828     int cond_op = assembler_cond_to_sve_op(cond, is_unsigned);
3829     f(is_unsigned ? 0b00100100 : 0b00100101, 31, 24), f(T, 23, 22);
3830     f(is_unsigned ? 0b1 : 0b0, 21);
3831     if (is_unsigned) {
3832       f(imm, 20, 14), f((cond_op >> 1) & 0x1, 13);
3833     } else {
3834       sf(imm, 20, 16), f((cond_op >> 1) & 0x7, 15, 13);
3835     }
3836     pgrf(Pg, 10), rf(Zn, 5), f(cond_op & 0x1, 4), prf(Pd, 0);
3837   }
3838 
3839   // SVE Floating-point compare vector with zero
3840   void sve_fcm(Condition cond, PRegister Pd, SIMD_RegVariant T,
3841                PRegister Pg, FloatRegister Zn, double d) {
3842     starti;
3843     assert(T != Q, "invalid size");
3844     guarantee(d == 0.0, "invalid immediate");
3845     int cond_op;
3846     switch(cond) {
3847       case EQ: cond_op = 0b100; break;
3848       case GT: cond_op = 0b001; break;
3849       case GE: cond_op = 0b000; break;
3850       case LT: cond_op = 0b010; break;
3851       case LE: cond_op = 0b011; break;
3852       case NE: cond_op = 0b110; break;
3853       default:
3854         ShouldNotReachHere();
3855     }
3856     f(0b01100101, 31, 24), f(T, 23, 22), f(0b0100, 21, 18),
3857     f((cond_op >> 1) & 0x3, 17, 16), f(0b001, 15, 13),
3858     pgrf(Pg, 10), rf(Zn, 5);
3859     f(cond_op & 0x1, 4), prf(Pd, 0);
3860   }
3861 
3862 // SVE unpack vector elements
3863 #define INSN(NAME, op) \
3864   void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn) { \
3865     starti;                                                          \
3866     assert(T != B && T != Q, "invalid size");                        \
3867     f(0b00000101, 31, 24), f(T, 23, 22), f(0b1100, 21, 18);          \
3868     f(op, 17, 16), f(0b001110, 15, 10), rf(Zn, 5), rf(Zd, 0);        \
3869   }
3870 
3871   INSN(sve_uunpkhi, 0b11); // Signed unpack and extend half of vector - high half
3872   INSN(sve_uunpklo, 0b10); // Signed unpack and extend half of vector - low half
3873   INSN(sve_sunpkhi, 0b01); // Unsigned unpack and extend half of vector - high half
3874   INSN(sve_sunpklo, 0b00); // Unsigned unpack and extend half of vector - low half
3875 #undef INSN
3876 
3877 // SVE unpack predicate elements
3878 #define INSN(NAME, op) \
3879   void NAME(PRegister Pd, PRegister Pn) { \
3880     starti;                                                          \
3881     f(0b000001010011000, 31, 17), f(op, 16), f(0b0100000, 15, 9);    \
3882     prf(Pn, 5), f(0b0, 4), prf(Pd, 0);                               \
3883   }
3884 
3885   INSN(sve_punpkhi, 0b1); // Unpack and widen high half of predicate
3886   INSN(sve_punpklo, 0b0); // Unpack and widen low half of predicate
3887 #undef INSN
3888 
3889 // SVE permute vector elements
3890 #define INSN(NAME, op) \
3891   void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, FloatRegister Zm) { \
3892     starti;                                                                            \
3893     assert(T != Q, "invalid size");                                                    \
3894     f(0b00000101, 31, 24), f(T, 23, 22), f(0b1, 21), rf(Zm, 16);                       \
3895     f(0b01101, 15, 11), f(op, 10), rf(Zn, 5), rf(Zd, 0);                               \
3896   }
3897 
3898   INSN(sve_uzp1, 0b0); // Concatenate even elements from two vectors
3899   INSN(sve_uzp2, 0b1); // Concatenate odd elements from two vectors
3900 #undef INSN
3901 
3902 // SVE permute predicate elements
3903 #define INSN(NAME, op) \
3904   void NAME(PRegister Pd, SIMD_RegVariant T, PRegister Pn, PRegister Pm) {             \
3905     starti;                                                                            \
3906     assert(T != Q, "invalid size");                                                    \
3907     f(0b00000101, 31, 24), f(T, 23, 22), f(0b10, 21, 20), prf(Pm, 16);                 \
3908     f(0b01001, 15, 11), f(op, 10), f(0b0, 9), prf(Pn, 5), f(0b0, 4), prf(Pd, 0);       \
3909   }
3910 
3911   INSN(sve_uzp1, 0b0); // Concatenate even elements from two predicates
3912   INSN(sve_uzp2, 0b1); // Concatenate odd elements from two predicates
3913 #undef INSN
3914 
3915 // SVE integer compare scalar count and limit
3916 #define INSN(NAME, sf, op)                                                \
3917   void NAME(PRegister Pd, SIMD_RegVariant T, Register Rn, Register Rm) {  \
3918     starti;                                                               \
3919     assert(T != Q, "invalid register variant");                           \
3920     f(0b00100101, 31, 24), f(T, 23, 22), f(1, 21),                        \
3921     zrf(Rm, 16), f(0, 15, 13), f(sf, 12), f(op >> 1, 11, 10),             \
3922     zrf(Rn, 5), f(op & 1, 4), prf(Pd, 0);                                 \
3923   }
3924   // While incrementing signed scalar less than scalar
3925   INSN(sve_whileltw, 0b0, 0b010);
3926   INSN(sve_whilelt,  0b1, 0b010);
3927   // While incrementing signed scalar less than or equal to scalar
3928   INSN(sve_whilelew, 0b0, 0b011);
3929   INSN(sve_whilele,  0b1, 0b011);
3930   // While incrementing unsigned scalar lower than scalar
3931   INSN(sve_whilelow, 0b0, 0b110);
3932   INSN(sve_whilelo,  0b1, 0b110);
3933   // While incrementing unsigned scalar lower than or the same as scalar
3934   INSN(sve_whilelsw, 0b0, 0b111);
3935   INSN(sve_whilels,  0b1, 0b111);
3936 #undef INSN
3937 
3938   // SVE predicate reverse
3939   void sve_rev(PRegister Pd, SIMD_RegVariant T, PRegister Pn) {
3940     starti;
3941     assert(T != Q, "invalid size");
3942     f(0b00000101, 31, 24), f(T, 23, 22), f(0b1101000100000, 21, 9);
3943     prf(Pn, 5), f(0, 4), prf(Pd, 0);
3944   }
3945 
3946 // SVE partition break condition
3947 #define INSN(NAME, op) \
3948   void NAME(PRegister Pd, PRegister Pg, PRegister Pn, bool isMerge) {      \
3949     starti;                                                                \
3950     f(0b00100101, 31, 24), f(op, 23, 22), f(0b01000001, 21, 14);           \
3951     prf(Pg, 10), f(0b0, 9), prf(Pn, 5), f(isMerge ? 1 : 0, 4), prf(Pd, 0); \
3952   }
3953 
3954   INSN(sve_brka, 0b00); // Break after first true condition
3955   INSN(sve_brkb, 0b10); // Break before first true condition
3956 #undef INSN
3957 
3958 // Element count and increment scalar (SVE)
3959 #define INSN(NAME, TYPE)                                                             \
3960   void NAME(Register Xdn, unsigned imm4 = 1, int pattern = 0b11111) {                \
3961     starti;                                                                          \
3962     f(0b00000100, 31, 24), f(TYPE, 23, 22), f(0b10, 21, 20);                         \
3963     f(imm4 - 1, 19, 16), f(0b11100, 15, 11), f(0, 10), f(pattern, 9, 5), rf(Xdn, 0); \
3964   }
3965 
3966   INSN(sve_cntb, B);  // Set scalar to multiple of 8-bit predicate constraint element count
3967   INSN(sve_cnth, H);  // Set scalar to multiple of 16-bit predicate constraint element count
3968   INSN(sve_cntw, S);  // Set scalar to multiple of 32-bit predicate constraint element count
3969   INSN(sve_cntd, D);  // Set scalar to multiple of 64-bit predicate constraint element count
3970 #undef INSN
3971 
3972   // Set scalar to active predicate element count
3973   void sve_cntp(Register Xd, SIMD_RegVariant T, PRegister Pg, PRegister Pn) {
3974     starti;
3975     assert(T != Q, "invalid size");
3976     f(0b00100101, 31, 24), f(T, 23, 22), f(0b10000010, 21, 14);
3977     prf(Pg, 10), f(0, 9), prf(Pn, 5), rf(Xd, 0);
3978   }
3979 
3980   // SVE convert signed integer to floating-point (predicated)
3981   void sve_scvtf(FloatRegister Zd, SIMD_RegVariant T_dst, PRegister Pg,
3982                  FloatRegister Zn, SIMD_RegVariant T_src) {
3983     starti;
3984     assert(T_src != B && T_dst != B && T_src != Q && T_dst != Q &&
3985            (T_src != H || T_dst == T_src), "invalid register variant");
3986     int opc = T_dst;
3987     int opc2 = T_src;
3988     // In most cases we can treat T_dst, T_src as opc, opc2,
3989     // except for the following two combinations.
3990     // +-----+------+---+------------------------------------+
3991     // | opc | opc2 | U |        Instruction Details         |
3992     // +-----+------+---+------------------------------------+
3993     // |  11 |   00 | 0 | SCVTF - 32-bit to double-precision |
3994     // |  11 |   10 | 0 | SCVTF - 64-bit to single-precision |
3995     // +-----+------+---+------------------------------------+
3996     if (T_src == S && T_dst == D) {
3997       opc = 0b11;
3998       opc2 = 0b00;
3999     } else if (T_src == D && T_dst == S) {
4000       opc = 0b11;
4001       opc2 = 0b10;
4002     }
4003     f(0b01100101, 31, 24), f(opc, 23, 22), f(0b010, 21, 19);
4004     f(opc2, 18, 17), f(0b0101, 16, 13);
4005     pgrf(Pg, 10), rf(Zn, 5), rf(Zd, 0);
4006   }
4007 
4008   // SVE floating-point convert to signed integer, rounding toward zero (predicated)
4009   void sve_fcvtzs(FloatRegister Zd, SIMD_RegVariant T_dst, PRegister Pg,
4010                   FloatRegister Zn, SIMD_RegVariant T_src) {
4011     starti;
4012     assert(T_src != B && T_dst != B && T_src != Q && T_dst != Q &&
4013            (T_dst != H || T_src == H), "invalid register variant");
4014     int opc = T_src;
4015     int opc2 = T_dst;
4016     // In most cases we can treat T_src, T_dst as opc, opc2,
4017     // except for the following two combinations.
4018     // +-----+------+---+-------------------------------------+
4019     // | opc | opc2 | U |         Instruction Details         |
4020     // +-----+------+---+-------------------------------------+
4021     // |  11 |  10  | 0 | FCVTZS - single-precision to 64-bit |
4022     // |  11 |  00  | 0 | FCVTZS - double-precision to 32-bit |
4023     // +-----+------+---+-------------------------------------+
4024     if (T_src == S && T_dst == D) {
4025       opc = 0b11;
4026       opc2 = 0b10;
4027     } else if (T_src == D && T_dst == S) {
4028       opc = 0b11;
4029       opc2 = 0b00;
4030     }
4031     f(0b01100101, 31, 24), f(opc, 23, 22), f(0b011, 21, 19);
4032     f(opc2, 18, 17), f(0b0101, 16, 13);
4033     pgrf(Pg, 10), rf(Zn, 5), rf(Zd, 0);
4034   }
4035 
4036   // SVE floating-point convert precision (predicated)
4037   void sve_fcvt(FloatRegister Zd, SIMD_RegVariant T_dst, PRegister Pg,
4038                 FloatRegister Zn, SIMD_RegVariant T_src) {
4039     starti;
4040     assert(T_src != B && T_dst != B && T_src != Q && T_dst != Q &&
4041            T_src != T_dst, "invalid register variant");
4042     // The encodings of fields op1 (bits 17-16) and op2 (bits 23-22)
4043     // depend on T_src and T_dst as given below -
4044     // +-----+------+---------------------------------------------+
4045     // | op2 | op1  |             Instruction Details             |
4046     // +-----+------+---------------------------------------------+
4047     // |  10 |  01  | FCVT - half-precision to single-precision   |
4048     // |  11 |  01  | FCVT - half-precision to double-precision   |
4049     // |  10 |  00  | FCVT - single-precision to half-precision   |
4050     // |  11 |  11  | FCVT - single-precision to double-precision |
4051     // |  11 |  00  | FCVT - double-preciison to half-precision   |
4052     // |  11 |  10  | FCVT - double-precision to single-precision |
4053     // +-----+------+---+-----------------------------------------+
4054     int op1 = 0b00;
4055     int op2 = (T_src == D || T_dst == D) ? 0b11 : 0b10;
4056     if (T_src == H) {
4057       op1 = 0b01;
4058     } else if (T_dst == S) {
4059       op1 = 0b10;
4060     } else if (T_dst == D) {
4061       op1 = 0b11;
4062     }
4063     f(0b01100101, 31, 24), f(op2, 23, 22), f(0b0010, 21, 18);
4064     f(op1, 17, 16), f(0b101, 15, 13);
4065     pgrf(Pg, 10), rf(Zn, 5), rf(Zd, 0);
4066   }
4067 
4068 // SVE extract element to general-purpose register
4069 #define INSN(NAME, before)                                                      \
4070   void NAME(Register Rd, SIMD_RegVariant T, PRegister Pg,  FloatRegister Zn) {  \
4071     starti;                                                                     \
4072     f(0b00000101, 31, 24), f(T, 23, 22), f(0b10000, 21, 17);                    \
4073     f(before, 16), f(0b101, 15, 13);                                            \
4074     pgrf(Pg, 10), rf(Zn, 5), rf(Rd, 0);                                         \
4075   }
4076 
4077   INSN(sve_lasta, 0b0);
4078   INSN(sve_lastb, 0b1);
4079 #undef INSN
4080 
4081 // SVE extract element to SIMD&FP scalar register
4082 #define INSN(NAME, before)                                                           \
4083   void NAME(FloatRegister Vd, SIMD_RegVariant T, PRegister Pg,  FloatRegister Zn) {  \
4084     starti;                                                                          \
4085     f(0b00000101, 31, 24), f(T, 23, 22), f(0b10001, 21, 17);                         \
4086     f(before, 16), f(0b100, 15, 13);                                                 \
4087     pgrf(Pg, 10), rf(Zn, 5), rf(Vd, 0);                                              \
4088   }
4089 
4090   INSN(sve_lasta, 0b0);
4091   INSN(sve_lastb, 0b1);
4092 #undef INSN
4093 
4094 // SVE reverse within elements
4095 #define INSN(NAME, opc, cond)                                                        \
4096   void NAME(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg,  FloatRegister Zn) {  \
4097     starti;                                                                          \
4098     assert(cond, "invalid size");                                                    \
4099     f(0b00000101, 31, 24), f(T, 23, 22), f(0b1001, 21, 18), f(opc, 17, 16);          \
4100     f(0b100, 15, 13), pgrf(Pg, 10), rf(Zn, 5), rf(Zd, 0);                            \
4101   }
4102 
4103   INSN(sve_revb, 0b00, T == H || T == S || T == D);
4104   INSN(sve_rbit, 0b11, T != Q);
4105 #undef INSN
4106 
4107   // SVE Create index starting from general-purpose register and incremented by immediate
4108   void sve_index(FloatRegister Zd, SIMD_RegVariant T, Register Rn, int imm) {
4109     starti;
4110     assert(T != Q, "invalid size");
4111     f(0b00000100, 31, 24), f(T, 23, 22), f(0b1, 21);
4112     sf(imm, 20, 16), f(0b010001, 15, 10);
4113     rf(Rn, 5), rf(Zd, 0);
4114   }
4115 
4116   // SVE create index starting from and incremented by immediate
4117   void sve_index(FloatRegister Zd, SIMD_RegVariant T, int imm1, int imm2) {
4118     starti;
4119     assert(T != Q, "invalid size");
4120     f(0b00000100, 31, 24), f(T, 23, 22), f(0b1, 21);
4121     sf(imm2, 20, 16), f(0b010000, 15, 10);
4122     sf(imm1, 9, 5), rf(Zd, 0);
4123   }
4124 
4125   // SVE programmable table lookup/permute using vector of element indices
4126   void sve_tbl(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, FloatRegister Zm) {
4127     starti;
4128     assert(T != Q, "invalid size");
4129     f(0b00000101, 31, 24), f(T, 23, 22), f(0b1, 21), rf(Zm, 16);
4130     f(0b001100, 15, 10), rf(Zn, 5), rf(Zd, 0);
4131   }
4132 
4133   // Shuffle active elements of vector to the right and fill with zero
4134   void sve_compact(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, PRegister Pg) {
4135     starti;
4136     assert(T == S || T == D, "invalid size");
4137     f(0b00000101, 31, 24), f(T, 23, 22), f(0b100001100, 21, 13);
4138     pgrf(Pg, 10), rf(Zn, 5), rf(Zd, 0);
4139   }
4140 
4141   // SVE2 Count matching elements in vector
4142   void sve_histcnt(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg,
4143                    FloatRegister Zn, FloatRegister Zm) {
4144     starti;
4145     assert(T == S || T == D, "invalid size");
4146     f(0b01000101, 31, 24), f(T, 23, 22), f(0b1, 21), rf(Zm, 16);
4147     f(0b110, 15, 13), pgrf(Pg, 10), rf(Zn, 5), rf(Zd, 0);
4148   }
4149 
4150 // SVE2 bitwise permute
4151 #define INSN(NAME, opc)                                                                  \
4152   void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn,  FloatRegister Zm) {  \
4153     starti;                                                                              \
4154     assert(T != Q, "invalid size");                                                      \
4155     f(0b01000101, 31, 24), f(T, 23, 22), f(0b0, 21);                                     \
4156     rf(Zm, 16), f(0b1011, 15, 12), f(opc, 11, 10);                                       \
4157     rf(Zn, 5), rf(Zd, 0);                                                                \
4158   }
4159 
4160   INSN(sve_bext, 0b00);
4161   INSN(sve_bdep, 0b01);
4162 #undef INSN
4163 
4164 // SVE2 bitwise ternary operations
4165 #define INSN(NAME, opc)                                               \
4166   void NAME(FloatRegister Zdn, FloatRegister Zm, FloatRegister Zk) {  \
4167     starti;                                                           \
4168     f(0b00000100, 31, 24), f(opc, 23, 21), rf(Zm, 16);                \
4169     f(0b001110, 15, 10), rf(Zk, 5), rf(Zdn, 0);                       \
4170   }
4171 
4172   INSN(sve_eor3, 0b001); // Bitwise exclusive OR of three vectors
4173 #undef INSN
4174 
4175   Assembler(CodeBuffer* code) : AbstractAssembler(code) {
4176   }
4177 
4178   // Stack overflow checking
4179   virtual void bang_stack_with_offset(int offset);
4180 
4181   static bool operand_valid_for_logical_immediate(bool is32, uint64_t imm);
4182   static bool operand_valid_for_sve_logical_immediate(unsigned elembits, uint64_t imm);
4183   static bool operand_valid_for_add_sub_immediate(int64_t imm);
4184   static bool operand_valid_for_sve_add_sub_immediate(int64_t imm);
4185   static bool operand_valid_for_float_immediate(double imm);
4186   static int  operand_valid_for_movi_immediate(uint64_t imm64, SIMD_Arrangement T);
4187 
4188   void emit_data64(jlong data, relocInfo::relocType rtype, int format = 0);
4189   void emit_data64(jlong data, RelocationHolder const& rspec, int format = 0);
4190 };
4191 
4192 inline Assembler::Membar_mask_bits operator|(Assembler::Membar_mask_bits a,
4193                                              Assembler::Membar_mask_bits b) {
4194   return Assembler::Membar_mask_bits(unsigned(a)|unsigned(b));
4195 }
4196 
4197 Instruction_aarch64::~Instruction_aarch64() {
4198   assem->emit_int32(insn);
4199   assert_cond(get_bits() == 0xffffffff);
4200 }
4201 
4202 #undef f
4203 #undef sf
4204 #undef rf
4205 #undef srf
4206 #undef zrf
4207 #undef prf
4208 #undef pgrf
4209 #undef fixed
4210 
4211 #undef starti
4212 
4213 // Invert a condition
4214 inline const Assembler::Condition operator~(const Assembler::Condition cond) {
4215   return Assembler::Condition(int(cond) ^ 1);
4216 }
4217 
4218 extern "C" void das(uint64_t start, int len);
4219 
4220 #endif // CPU_AARCH64_ASSEMBLER_AARCH64_HPP