< prev index next >

src/hotspot/cpu/aarch64/assembler_aarch64.hpp

Print this page




 259     f(r->encoding_nocheck(), lsb + 4, lsb);
 260   }
 261 
 262   // reg|ZR
 263   void zrf(Register r, int lsb) {
 264     f(r->encoding_nocheck() - (r == zr), lsb + 4, lsb);
 265   }
 266 
 267   // reg|SP
 268   void srf(Register r, int lsb) {
 269     f(r == sp ? 31 : r->encoding_nocheck(), lsb + 4, lsb);
 270   }
 271 
 272   void rf(FloatRegister r, int lsb) {
 273     f(r->encoding_nocheck(), lsb + 4, lsb);
 274   }
 275 
 276   unsigned get(int msb = 31, int lsb = 0) {
 277     int nbits = msb - lsb + 1;
 278     unsigned mask = ((1U << nbits) - 1) << lsb;
 279     assert_cond((bits & mask) == mask);
 280     return (insn & mask) >> lsb;
 281   }
 282 
 283   void fixed(unsigned value, unsigned mask) {
 284     assert_cond ((mask & bits) == 0);
 285 #ifdef ASSERT
 286     bits |= mask;
 287 #endif
 288     insn |= value;
 289   }
 290 };
 291 
 292 #define starti Instruction_aarch64 do_not_use(this); set_current(&do_not_use)
 293 
 294 class PrePost {
 295   int _offset;
 296   Register _r;
 297 public:
 298   PrePost(Register reg, int o) : _offset(o), _r(reg) { }
 299   int offset() { return _offset; }
 300   Register reg() { return _r; }
 301 };
 302 
 303 class Pre : public PrePost {
 304 public:
 305   Pre(Register reg, int o) : PrePost(reg, o) { }
 306 };
 307 class Post : public PrePost {
 308   Register _idx;
 309   bool _is_postreg;
 310 public:
 311   Post(Register reg, int o) : PrePost(reg, o) { _idx = NULL; _is_postreg = false; }
 312   Post(Register reg, Register idx) : PrePost(reg, 0) { _idx = idx; _is_postreg = true; }
 313   Register idx_reg() { return _idx; }
 314   bool is_postreg() {return _is_postreg; }
 315 };
 316 
 317 namespace ext
 318 {
 319   enum operation { uxtb, uxth, uxtw, uxtx, sxtb, sxth, sxtw, sxtx };
 320 };
 321 
 322 // Addressing modes
 323 class Address {
 324  public:
 325 
 326   enum mode { no_mode, base_plus_offset, pre, post, post_reg, pcrel,
 327               base_plus_offset_reg, literal };
 328 
 329   // Shift and extend for base reg + reg offset addressing
 330   class extend {
 331     int _option, _shift;
 332     ext::operation _op;
 333   public:
 334     extend() { }


 378     : _mode(no_mode) { }
 379   Address(Register r)
 380     : _base(r), _index(noreg), _offset(0), _mode(base_plus_offset), _target(0) { }
 381   Address(Register r, int o)
 382     : _base(r), _index(noreg), _offset(o), _mode(base_plus_offset), _target(0) { }
 383   Address(Register r, long o)
 384     : _base(r), _index(noreg), _offset(o), _mode(base_plus_offset), _target(0) { }
 385   Address(Register r, unsigned long o)
 386     : _base(r), _index(noreg), _offset(o), _mode(base_plus_offset), _target(0) { }
 387 #ifdef ASSERT
 388   Address(Register r, ByteSize disp)
 389     : _base(r), _index(noreg), _offset(in_bytes(disp)), _mode(base_plus_offset), _target(0) { }
 390 #endif
 391   Address(Register r, Register r1, extend ext = lsl())
 392     : _base(r), _index(r1), _offset(0), _mode(base_plus_offset_reg),
 393       _ext(ext), _target(0) { }
 394   Address(Pre p)
 395     : _base(p.reg()), _offset(p.offset()), _mode(pre) { }
 396   Address(Post p)
 397     : _base(p.reg()),  _index(p.idx_reg()), _offset(p.offset()),
 398       _mode(p.is_postreg() ? post_reg : post), _target(0) { }
 399   Address(address target, RelocationHolder const& rspec)
 400     : _mode(literal),
 401       _rspec(rspec),
 402       _is_lval(false),
 403       _target(target)  { }
 404   Address(address target, relocInfo::relocType rtype = relocInfo::external_word_type);
 405   Address(Register base, RegisterOrConstant index, extend ext = lsl())
 406     : _base (base),
 407       _offset(0), _ext(ext), _target(0) {
 408     if (index.is_register()) {
 409       _mode = base_plus_offset_reg;
 410       _index = index.as_register();
 411     } else {
 412       guarantee(ext.option() == ext::uxtx, "should be");
 413       assert(index.is_constant(), "should be");
 414       _mode = base_plus_offset;
 415       _offset = index.as_constant() << ext.shift();
 416     }
 417   }
 418 


 792   // Move wide (immediate)
 793 #define INSN(NAME, opcode)                                              \
 794   void NAME(Register Rd, unsigned imm, unsigned shift = 0) {            \
 795     assert_cond((shift/16)*16 == shift);                                \
 796     starti;                                                             \
 797     f(opcode, 31, 29), f(0b100101, 28, 23), f(shift/16, 22, 21),        \
 798       f(imm, 20, 5);                                                    \
 799     rf(Rd, 0);                                                          \
 800   }
 801 
 802   INSN(movnw, 0b000);
 803   INSN(movzw, 0b010);
 804   INSN(movkw, 0b011);
 805   INSN(movn, 0b100);
 806   INSN(movz, 0b110);
 807   INSN(movk, 0b111);
 808 
 809 #undef INSN
 810 
 811   // Bitfield
 812 #define INSN(NAME, opcode, size)                                        \
 813   void NAME(Register Rd, Register Rn, unsigned immr, unsigned imms) {   \
 814     starti;                                                             \
 815     guarantee(size == 1 || (immr < 32 && imms < 32), "incorrect immr/imms");\
 816     f(opcode, 31, 22), f(immr, 21, 16), f(imms, 15, 10);                \
 817     zrf(Rn, 5), rf(Rd, 0);                                              \
 818   }
 819 
 820   INSN(sbfmw, 0b0001001100, 0);
 821   INSN(bfmw,  0b0011001100, 0);
 822   INSN(ubfmw, 0b0101001100, 0);
 823   INSN(sbfm,  0b1001001101, 1);
 824   INSN(bfm,   0b1011001101, 1);
 825   INSN(ubfm,  0b1101001101, 1);
 826 
 827 #undef INSN
 828 
 829   // Extract
 830 #define INSN(NAME, opcode, size)                                        \
 831   void NAME(Register Rd, Register Rn, Register Rm, unsigned imms) {     \
 832     starti;                                                             \
 833     guarantee(size == 1 || imms < 32, "incorrect imms");                \
 834     f(opcode, 31, 21), f(imms, 15, 10);                                 \
 835     zrf(Rm, 16), zrf(Rn, 5), zrf(Rd, 0);                                \
 836   }
 837 
 838   INSN(extrw, 0b00010011100, 0);
 839   INSN(extr,  0b10010011110, 1);
 840 
 841 #undef INSN
 842 
 843   // The maximum range of a branch is fixed for the AArch64
 844   // architecture.  In debug mode we shrink it in order to test
 845   // trampolines, but not so small that branches in the interpreter
 846   // are out of range.
 847   static const unsigned long branch_range = NOT_DEBUG(128 * M) DEBUG_ONLY(2 * M);
 848 
 849   static bool reachable_from_branch_at(address branch, address target) {
 850     return uabs(target - branch) < branch_range;
 851   }
 852 
 853   // Unconditional branch (immediate)
 854 #define INSN(NAME, opcode)                                              \
 855   void NAME(address dest) {                                             \
 856     starti;                                                             \
 857     long offset = (dest - pc()) >> 2;                                   \
 858     DEBUG_ONLY(assert(reachable_from_branch_at(pc(), dest), "debug only")); \
 859     f(opcode, 31), f(0b00101, 30, 26), sf(offset, 25, 0);               \


1113 
1114 #undef INSN
1115 
1116 #define INSN(NAME, opc)                         \
1117   void NAME() {                 \
1118     branch_reg(dummy_reg, opc);         \
1119   }
1120 
1121   INSN(eret, 0b0100);
1122   INSN(drps, 0b0101);
1123 
1124 #undef INSN
1125 
1126   // Load/store exclusive
1127   enum operand_size { byte, halfword, word, xword };
1128 
1129   void load_store_exclusive(Register Rs, Register Rt1, Register Rt2,
1130     Register Rn, enum operand_size sz, int op, bool ordered) {
1131     starti;
1132     f(sz, 31, 30), f(0b001000, 29, 24), f(op, 23, 21);
1133     rf(Rs, 16), f(ordered, 15), zrf(Rt2, 10), srf(Rn, 5), zrf(Rt1, 0);
1134   }
1135 
1136   void load_exclusive(Register dst, Register addr,
1137                       enum operand_size sz, bool ordered) {
1138     load_store_exclusive(dummy_reg, dst, dummy_reg, addr,
1139                          sz, 0b010, ordered);
1140   }
1141 
1142   void store_exclusive(Register status, Register new_val, Register addr,
1143                        enum operand_size sz, bool ordered) {
1144     load_store_exclusive(status, new_val, dummy_reg, addr,
1145                          sz, 0b000, ordered);
1146   }
1147 
1148 #define INSN4(NAME, sz, op, o0) /* Four registers */                    \
1149   void NAME(Register Rs, Register Rt1, Register Rt2, Register Rn) {     \
1150     guarantee(Rs != Rn && Rs != Rt1 && Rs != Rt2, "unpredictable instruction"); \
1151     load_store_exclusive(Rs, Rt1, Rt2, Rn, sz, op, o0);                 \
1152   }
1153 


1242   // CASP
1243 #define INSN(NAME, a, r)                                                \
1244   void NAME(operand_size sz, Register Rs, Register Rs1,                 \
1245             Register Rt, Register Rt1, Register Rn) {                   \
1246     assert((Rs->encoding() & 1) == 0 && (Rt->encoding() & 1) == 0 &&    \
1247            Rs->successor() == Rs1 && Rt->successor() == Rt1 &&          \
1248            Rs != Rn && Rs1 != Rn && Rs != Rt, "invalid registers");     \
1249     lse_cas(Rs, Rt, Rn, sz, a, r, false);                               \
1250   }
1251   INSN(casp,    false, false)
1252   INSN(caspa,   true,  false)
1253   INSN(caspl,   false, true)
1254   INSN(caspal,  true,  true)
1255 #undef INSN
1256 
1257   // 8.1 Atomic operations
1258   void lse_atomic(Register Rs, Register Rt, Register Rn,
1259                   enum operand_size sz, int op1, int op2, bool a, bool r) {
1260     starti;
1261     f(sz, 31, 30), f(0b111000, 29, 24), f(a, 23), f(r, 22), f(1, 21);
1262     zrf(Rs, 16), f(op1, 15), f(op2, 14, 12), f(0, 11, 10), srf(Rn, 5), zrf(Rt, 0);
1263   }
1264 
1265 #define INSN(NAME, NAME_A, NAME_L, NAME_AL, op1, op2)                   \
1266   void NAME(operand_size sz, Register Rs, Register Rt, Register Rn) {   \
1267     lse_atomic(Rs, Rt, Rn, sz, op1, op2, false, false);                 \
1268   }                                                                     \
1269   void NAME_A(operand_size sz, Register Rs, Register Rt, Register Rn) { \
1270     lse_atomic(Rs, Rt, Rn, sz, op1, op2, true, false);                  \
1271   }                                                                     \
1272   void NAME_L(operand_size sz, Register Rs, Register Rt, Register Rn) { \
1273     lse_atomic(Rs, Rt, Rn, sz, op1, op2, false, true);                  \
1274   }                                                                     \
1275   void NAME_AL(operand_size sz, Register Rs, Register Rt, Register Rn) {\
1276     lse_atomic(Rs, Rt, Rn, sz, op1, op2, true, true);                   \
1277   }
1278   INSN(ldadd,  ldadda,  ldaddl,  ldaddal,  0, 0b000);
1279   INSN(ldbic,  ldbica,  ldbicl,  ldbical,  0, 0b001);
1280   INSN(ldeor,  ldeora,  ldeorl,  ldeoral,  0, 0b010);
1281   INSN(ldorr,  ldorra,  ldorrl,  ldorral,  0, 0b011);
1282   INSN(ldsmax, ldsmaxa, ldsmaxl, ldsmaxal, 0, 0b100);


1464 
1465 #undef INSN
1466 
1467   enum shift_kind { LSL, LSR, ASR, ROR };
1468 
1469   void op_shifted_reg(unsigned decode,
1470                       enum shift_kind kind, unsigned shift,
1471                       unsigned size, unsigned op) {
1472     f(size, 31);
1473     f(op, 30, 29);
1474     f(decode, 28, 24);
1475     f(shift, 15, 10);
1476     f(kind, 23, 22);
1477   }
1478 
1479   // Logical (shifted register)
1480 #define INSN(NAME, size, op, N)                                 \
1481   void NAME(Register Rd, Register Rn, Register Rm,              \
1482             enum shift_kind kind = LSL, unsigned shift = 0) {   \
1483     starti;                                                     \
1484     guarantee(size == 1 || shift < 32, "incorrect shift");      \
1485     f(N, 21);                                                   \
1486     zrf(Rm, 16), zrf(Rn, 5), zrf(Rd, 0);                        \
1487     op_shifted_reg(0b01010, kind, shift, size, op);             \
1488   }
1489 
1490   INSN(andr, 1, 0b00, 0);
1491   INSN(orr, 1, 0b01, 0);
1492   INSN(eor, 1, 0b10, 0);
1493   INSN(ands, 1, 0b11, 0);
1494   INSN(andw, 0, 0b00, 0);
1495   INSN(orrw, 0, 0b01, 0);
1496   INSN(eorw, 0, 0b10, 0);
1497   INSN(andsw, 0, 0b11, 0);
1498 
1499 #undef INSN
1500 
1501 #define INSN(NAME, size, op, N)                                         \
1502   void NAME(Register Rd, Register Rn, Register Rm,                      \
1503             enum shift_kind kind = LSL, unsigned shift = 0) {           \
1504     starti;                                                             \


1527 #undef INSN
1528 
1529   // Aliases for short forms of orn
1530 void mvn(Register Rd, Register Rm,
1531             enum shift_kind kind = LSL, unsigned shift = 0) {
1532   orn(Rd, zr, Rm, kind, shift);
1533 }
1534 
1535 void mvnw(Register Rd, Register Rm,
1536             enum shift_kind kind = LSL, unsigned shift = 0) {
1537   ornw(Rd, zr, Rm, kind, shift);
1538 }
1539 
1540   // Add/subtract (shifted register)
1541 #define INSN(NAME, size, op)                            \
1542   void NAME(Register Rd, Register Rn, Register Rm,      \
1543             enum shift_kind kind, unsigned shift = 0) { \
1544     starti;                                             \
1545     f(0, 21);                                           \
1546     assert_cond(kind != ROR);                           \
1547     guarantee(size == 1 || shift < 32, "incorrect shift");\
1548     zrf(Rd, 0), zrf(Rn, 5), zrf(Rm, 16);                \
1549     op_shifted_reg(0b01011, kind, shift, size, op);     \
1550   }
1551 
1552   INSN(add, 1, 0b000);
1553   INSN(sub, 1, 0b10);
1554   INSN(addw, 0, 0b000);
1555   INSN(subw, 0, 0b10);
1556 
1557   INSN(adds, 1, 0b001);
1558   INSN(subs, 1, 0b11);
1559   INSN(addsw, 0, 0b001);
1560   INSN(subsw, 0, 0b11);
1561 
1562 #undef INSN
1563 
1564   // Add/subtract (extended register)
1565 #define INSN(NAME, op)                                                  \
1566   void NAME(Register Rd, Register Rn, Register Rm,                      \
1567            ext::operation option, int amount = 0) {                     \
1568     starti;                                                             \
1569     zrf(Rm, 16), srf(Rn, 5), srf(Rd, 0);                                \
1570     add_sub_extended_reg(op, 0b01011, Rd, Rn, Rm, 0b00, option, amount); \
1571   }
1572 
1573   void add_sub_extended_reg(unsigned op, unsigned decode,
1574     Register Rd, Register Rn, Register Rm,
1575     unsigned opt, ext::operation option, unsigned imm) {
1576     guarantee(imm <= 4, "shift amount must be <= 4");
1577     f(op, 31, 29), f(decode, 28, 24), f(opt, 23, 22), f(1, 21);
1578     f(option, 15, 13), f(imm, 12, 10);
1579   }
1580 
1581   INSN(addw, 0b000);
1582   INSN(subw, 0b010);
1583   INSN(add, 0b100);
1584   INSN(sub, 0b110);
1585 
1586 #undef INSN
1587 
1588 #define INSN(NAME, op)                                                  \
1589   void NAME(Register Rd, Register Rn, Register Rm,                      \
1590            ext::operation option, int amount = 0) {                     \
1591     starti;                                                             \
1592     zrf(Rm, 16), srf(Rn, 5), zrf(Rd, 0);                                \
1593     add_sub_extended_reg(op, 0b01011, Rd, Rn, Rm, 0b00, option, amount); \
1594   }
1595 
1596   INSN(addsw, 0b001);


1641   INSN(sbcsw, 0b011);
1642   INSN(adc, 0b100);
1643   INSN(adcs, 0b101);
1644   INSN(sbc,0b110);
1645   INSN(sbcs, 0b111);
1646 
1647 #undef INSN
1648 
1649   // Conditional compare (both kinds)
1650   void conditional_compare(unsigned op, int o1, int o2, int o3,
1651                            Register Rn, unsigned imm5, unsigned nzcv,
1652                            unsigned cond) {
1653     starti;
1654     f(op, 31, 29);
1655     f(0b11010010, 28, 21);
1656     f(cond, 15, 12);
1657     f(o1, 11);
1658     f(o2, 10);
1659     f(o3, 4);
1660     f(nzcv, 3, 0);
1661     f(imm5, 20, 16), zrf(Rn, 5);
1662   }
1663 
1664 #define INSN(NAME, op)                                                  \
1665   void NAME(Register Rn, Register Rm, int imm, Condition cond) {        \
1666     int regNumber = (Rm == zr ? 31 : (uintptr_t)Rm);                    \
1667     conditional_compare(op, 0, 0, 0, Rn, regNumber, imm, cond);         \
1668   }                                                                     \
1669                                                                         \
1670   void NAME(Register Rn, int imm5, int imm, Condition cond) {           \
1671     conditional_compare(op, 1, 0, 0, Rn, imm5, imm, cond);              \
1672   }
1673 
1674   INSN(ccmnw, 0b001);
1675   INSN(ccmpw, 0b011);
1676   INSN(ccmn, 0b101);
1677   INSN(ccmp, 0b111);
1678 
1679 #undef INSN
1680 
1681   // Conditional select


2110 #define INSN(NAME, op)                                            \
2111   void NAME(FloatRegister Rt, SIMD_RegVariant T, const Address &adr) {   \
2112     ld_st2((Register)Rt, adr, (int)T & 3, op + ((T==Q) ? 0b10:0b00), 1); \
2113   }                                                                      \
2114 
2115   INSN(ldr, 1);
2116   INSN(str, 0);
2117 
2118 #undef INSN
2119 
2120  private:
2121 
2122   void ld_st(FloatRegister Vt, SIMD_Arrangement T, Register Xn, int op1, int op2) {
2123     starti;
2124     f(0,31), f((int)T & 1, 30);
2125     f(op1, 29, 21), f(0, 20, 16), f(op2, 15, 12);
2126     f((int)T >> 1, 11, 10), srf(Xn, 5), rf(Vt, 0);
2127   }
2128   void ld_st(FloatRegister Vt, SIMD_Arrangement T, Register Xn,
2129              int imm, int op1, int op2, int regs) {
2130 
2131     bool replicate = op2 >> 2 == 3;
2132     // post-index value (imm) is formed differently for replicate/non-replicate ld* instructions
2133     int expectedImmediate = replicate ? regs * (1 << (T >> 1)) : SIMD_Size_in_bytes[T] * regs;
2134     guarantee(T < T1Q , "incorrect arrangement");
2135     guarantee(imm == expectedImmediate, "bad offset");
2136     starti;
2137     f(0,31), f((int)T & 1, 30);
2138     f(op1 | 0b100, 29, 21), f(0b11111, 20, 16), f(op2, 15, 12);
2139     f((int)T >> 1, 11, 10), srf(Xn, 5), rf(Vt, 0);
2140   }
2141   void ld_st(FloatRegister Vt, SIMD_Arrangement T, Register Xn,
2142              Register Xm, int op1, int op2) {
2143     starti;
2144     f(0,31), f((int)T & 1, 30);
2145     f(op1 | 0b100, 29, 21), rf(Xm, 16), f(op2, 15, 12);
2146     f((int)T >> 1, 11, 10), srf(Xn, 5), rf(Vt, 0);
2147   }
2148 
2149   void ld_st(FloatRegister Vt, SIMD_Arrangement T, Address a, int op1, int op2, int regs) {
2150     switch (a.getMode()) {
2151     case Address::base_plus_offset:
2152       guarantee(a.offset() == 0, "no offset allowed here");
2153       ld_st(Vt, T, a.base(), op1, op2);
2154       break;
2155     case Address::post:


2222 
2223 #define INSN(NAME, opc)                                                                 \
2224   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \
2225     starti;                                                                             \
2226     assert(T == T8B || T == T16B, "must be T8B or T16B");                               \
2227     f(0, 31), f((int)T & 1, 30), f(opc, 29, 21);                                        \
2228     rf(Vm, 16), f(0b000111, 15, 10), rf(Vn, 5), rf(Vd, 0);                              \
2229   }
2230 
2231   INSN(eor,  0b101110001);
2232   INSN(orr,  0b001110101);
2233   INSN(andr, 0b001110001);
2234   INSN(bic,  0b001110011);
2235   INSN(bif,  0b101110111);
2236   INSN(bit,  0b101110101);
2237   INSN(bsl,  0b101110011);
2238   INSN(orn,  0b001110111);
2239 
2240 #undef INSN
2241 
2242 #define INSN(NAME, opc, opc2, acceptT2D)                                                \
2243   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \
2244     guarantee(T != T1Q && T != T1D, "incorrect arrangement");                           \
2245     if (!acceptT2D) guarantee(T != T2D, "incorrect arrangement");                       \
2246     starti;                                                                             \
2247     f(0, 31), f((int)T & 1, 30), f(opc, 29), f(0b01110, 28, 24);                        \
2248     f((int)T >> 1, 23, 22), f(1, 21), rf(Vm, 16), f(opc2, 15, 10);                      \
2249     rf(Vn, 5), rf(Vd, 0);                                                               \
2250   }
2251 
2252   INSN(addv,   0, 0b100001, true);  // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2253   INSN(subv,   1, 0b100001, true);  // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2254   INSN(mulv,   0, 0b100111, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2255   INSN(mlav,   0, 0b100101, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2256   INSN(mlsv,   1, 0b100101, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2257   INSN(sshl,   0, 0b010001, true);  // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2258   INSN(ushl,   1, 0b010001, true);  // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2259   INSN(umullv, 1, 0b110000, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2260   INSN(umlalv, 1, 0b100000, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2261 
2262 #undef INSN
2263 
2264 #define INSN(NAME, opc, opc2, accepted) \
2265   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {                   \
2266     guarantee(T != T1Q && T != T1D, "incorrect arrangement");                           \
2267     if (accepted < 2) guarantee(T != T2S && T != T2D, "incorrect arrangement");         \
2268     if (accepted == 0) guarantee(T == T8B || T == T16B, "incorrect arrangement");       \
2269     starti;                                                                             \
2270     f(0, 31), f((int)T & 1, 30), f(opc, 29), f(0b01110, 28, 24);                        \
2271     f((int)T >> 1, 23, 22), f(opc2, 21, 10);                                            \
2272     rf(Vn, 5), rf(Vd, 0);                                                               \
2273   }
2274 
2275   INSN(absr,   0, 0b100000101110, 1); // accepted arrangements: T8B, T16B, T4H, T8H,      T4S
2276   INSN(negr,   1, 0b100000101110, 2); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
2277   INSN(notr,   1, 0b100000010110, 0); // accepted arrangements: T8B, T16B
2278   INSN(addv,   0, 0b110001101110, 1); // accepted arrangements: T8B, T16B, T4H, T8H,      T4S
2279   INSN(cls,    0, 0b100000010010, 1); // accepted arrangements: T8B, T16B, T4H, T8H,      T4S
2280   INSN(clz,    1, 0b100000010010, 1); // accepted arrangements: T8B, T16B, T4H, T8H,      T4S
2281   INSN(cnt,    0, 0b100000010110, 0); // accepted arrangements: T8B, T16B
2282   INSN(uaddlv, 1, 0b110000001110, 1); // accepted arrangements: T8B, T16B, T4H, T8H,      T4S
2283 
2284 #undef INSN
2285 
2286 #define INSN(NAME, opc) \
2287   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {                  \
2288     starti;                                                                            \
2289     assert(T == T4S, "arrangement must be T4S");                                       \
2290     f(0, 31), f((int)T & 1, 30), f(0b101110, 29, 24), f(opc, 23),                      \
2291     f(T == T4S ? 0 : 1, 22), f(0b110000111110, 21, 10); rf(Vn, 5), rf(Vd, 0);          \
2292   }
2293 
2294   INSN(fmaxv, 0);
2295   INSN(fminv, 1);
2296 
2297 #undef INSN
2298 
2299 #define INSN(NAME, op0, cmode0) \
2300   void NAME(FloatRegister Vd, SIMD_Arrangement T, unsigned imm8, unsigned lsl = 0) {   \
2301     unsigned cmode = cmode0;                                                           \
2302     unsigned op = op0;                                                                 \
2303     starti;                                                                            \
2304     assert(lsl == 0 ||                                                                 \
2305            ((T == T4H || T == T8H) && lsl == 8) ||                                     \
2306            ((T == T2S || T == T4S) && ((lsl >> 3) < 4) && ((lsl & 7) == 0)), "invalid shift");\
2307     cmode |= lsl >> 2;                                                                 \
2308     if (T == T4H || T == T8H) cmode |= 0b1000;                                         \
2309     if (!(T == T4H || T == T8H || T == T2S || T == T4S)) {                             \
2310       assert(op == 0 && cmode0 == 0, "must be MOVI");                                  \
2311       cmode = 0b1110;                                                                  \
2312       if (T == T1D || T == T2D) op = 1;                                                \
2313     }                                                                                  \
2314     f(0, 31), f((int)T & 1, 30), f(op, 29), f(0b0111100000, 28, 19);                   \
2315     f(imm8 >> 5, 18, 16), f(cmode, 15, 12), f(0x01, 11, 10), f(imm8 & 0b11111, 9, 5);  \
2316     rf(Vd, 0);                                                                         \
2317   }
2318 
2319   INSN(movi, 0, 0);
2320   INSN(orri, 0, 1);
2321   INSN(mvni, 1, 0);
2322   INSN(bici, 1, 1);
2323 
2324 #undef INSN
2325 
2326 #define INSN(NAME, op1, op2, op3) \


2447      * for SHL shift is calculated as:                                  \
2448      *   0001 xxx       8B/16B, shift = UInt(immh:immb) - 8             \
2449      *   001x xxx       4H/8H,  shift = UInt(immh:immb) - 16            \
2450      *   01xx xxx       2S/4S,  shift = UInt(immh:immb) - 32            \
2451      *   1xxx xxx       1D/2D,  shift = UInt(immh:immb) - 64            \
2452      *   (1D is RESERVED)                                               \
2453      */                                                                 \
2454     assert((1 << ((T>>1)+3)) > shift, "Invalid Shift value");           \
2455     int cVal = (1 << (((T >> 1) + 3) + (isSHR ? 1 : 0)));               \
2456     int encodedShift = isSHR ? cVal - shift : cVal + shift;             \
2457     f(0, 31), f(T & 1, 30), f(opc, 29), f(0b011110, 28, 23),            \
2458     f(encodedShift, 22, 16); f(opc2, 15, 10), rf(Vn, 5), rf(Vd, 0);     \
2459   }
2460 
2461   INSN(shl,  0, 0b010101, /* isSHR = */ false);
2462   INSN(sshr, 0, 0b000001, /* isSHR = */ true);
2463   INSN(ushr, 1, 0b000001, /* isSHR = */ true);
2464 
2465 #undef INSN
2466 
2467 private:
2468   void _ushll(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn, SIMD_Arrangement Tb, int shift) {
2469     starti;
2470     /* The encodings for the immh:immb fields (bits 22:16) are
2471      *   0001 xxx       8H, 8B/16b shift = xxx
2472      *   001x xxx       4S, 4H/8H  shift = xxxx
2473      *   01xx xxx       2D, 2S/4S  shift = xxxxx
2474      *   1xxx xxx       RESERVED
2475      */
2476     assert((Tb >> 1) + 1 == (Ta >> 1), "Incompatible arrangement");
2477     assert((1 << ((Tb>>1)+3)) > shift, "Invalid shift value");
2478     f(0, 31), f(Tb & 1, 30), f(0b1011110, 29, 23), f((1 << ((Tb>>1)+3))|shift, 22, 16);
2479     f(0b101001, 15, 10), rf(Vn, 5), rf(Vd, 0);
2480   }
2481 
2482 public:
2483   void ushll(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn,  SIMD_Arrangement Tb, int shift) {
2484     assert(Tb == T8B || Tb == T4H || Tb == T2S, "invalid arrangement");
2485     _ushll(Vd, Ta, Vn, Tb, shift);
2486   }
2487 
2488   void ushll2(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn,  SIMD_Arrangement Tb, int shift) {
2489     assert(Tb == T16B || Tb == T8H || Tb == T4S, "invalid arrangement");
2490     _ushll(Vd, Ta, Vn, Tb, shift);
2491   }
2492 
2493   // Move from general purpose register
2494   //   mov  Vd.T[index], Rn
2495   void mov(FloatRegister Vd, SIMD_Arrangement T, int index, Register Xn) {
2496     starti;
2497     f(0b01001110000, 31, 21), f(((1 << (T >> 1)) | (index << ((T >> 1) + 1))), 20, 16);
2498     f(0b000111, 15, 10), zrf(Xn, 5), rf(Vd, 0);
2499   }
2500 
2501   // Move to general purpose register
2502   //   mov  Rd, Vn.T[index]
2503   void mov(Register Xd, FloatRegister Vn, SIMD_Arrangement T, int index) {
2504     guarantee(T >= T2S && T < T1Q, "only D and S arrangements are supported");
2505     starti;
2506     f(0, 31), f((T >= T1D) ? 1:0, 30), f(0b001110000, 29, 21);
2507     f(((1 << (T >> 1)) | (index << ((T >> 1) + 1))), 20, 16);
2508     f(0b001111, 15, 10), rf(Vn, 5), rf(Xd, 0);
2509   }
2510 
2511 private:
2512   void _pmull(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn, FloatRegister Vm, SIMD_Arrangement Tb) {
2513     starti;
2514     assert((Ta == T1Q && (Tb == T1D || Tb == T2D)) ||
2515            (Ta == T8H && (Tb == T8B || Tb == T16B)), "Invalid Size specifier");
2516     int size = (Ta == T1Q) ? 0b11 : 0b00;
2517     f(0, 31), f(Tb & 1, 30), f(0b001110, 29, 24), f(size, 23, 22);
2518     f(1, 21), rf(Vm, 16), f(0b111000, 15, 10), rf(Vn, 5), rf(Vd, 0);
2519   }
2520 
2521 public:
2522   void pmull(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn, FloatRegister Vm, SIMD_Arrangement Tb) {
2523     assert(Tb == T1D || Tb == T8B, "pmull assumes T1D or T8B as the second size specifier");
2524     _pmull(Vd, Ta, Vn, Vm, Tb);
2525   }
2526 
2527   void pmull2(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn, FloatRegister Vm, SIMD_Arrangement Tb) {
2528     assert(Tb == T2D || Tb == T16B, "pmull2 assumes T2D or T16B as the second size specifier");
2529     _pmull(Vd, Ta, Vn, Vm, Tb);
2530   }
2531 
2532   void uqxtn(FloatRegister Vd, SIMD_Arrangement Tb, FloatRegister Vn, SIMD_Arrangement Ta) {
2533     starti;
2534     int size_b = (int)Tb >> 1;
2535     int size_a = (int)Ta >> 1;
2536     assert(size_b < 3 && size_b == size_a - 1, "Invalid size specifier");
2537     f(0, 31), f(Tb & 1, 30), f(0b101110, 29, 24), f(size_b, 23, 22);
2538     f(0b100001010010, 21, 10), rf(Vn, 5), rf(Vd, 0);
2539   }
2540 
2541   void dup(FloatRegister Vd, SIMD_Arrangement T, Register Xs)
2542   {
2543     starti;
2544     assert(T != T1D, "reserved encoding");
2545     f(0,31), f((int)T & 1, 30), f(0b001110000, 29, 21);
2546     f((1 << (T >> 1)), 20, 16), f(0b000011, 15, 10), zrf(Xs, 5), rf(Vd, 0);
2547   }
2548 
2549   void dup(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, int index = 0)
2550   {
2551     starti;
2552     assert(T != T1D, "reserved encoding");
2553     f(0, 31), f((int)T & 1, 30), f(0b001110000, 29, 21);
2554     f(((1 << (T >> 1)) | (index << ((T >> 1) + 1))), 20, 16);
2555     f(0b000001, 15, 10), rf(Vn, 5), rf(Vd, 0);
2556   }
2557 
2558   // AdvSIMD ZIP/UZP/TRN
2559 #define INSN(NAME, opcode)                                              \
2560   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \
2561     guarantee(T != T1D && T != T1Q, "invalid arrangement");             \
2562     starti;                                                             \
2563     f(0, 31), f(0b001110, 29, 24), f(0, 21), f(0, 15);                  \
2564     f(opcode, 14, 12), f(0b10, 11, 10);                                 \
2565     rf(Vm, 16), rf(Vn, 5), rf(Vd, 0);                                   \
2566     f(T & 1, 30), f(T >> 1, 23, 22);                                    \
2567   }
2568 
2569   INSN(uzp1, 0b001);
2570   INSN(trn1, 0b010);
2571   INSN(zip1, 0b011);
2572   INSN(uzp2, 0b101);
2573   INSN(trn2, 0b110);
2574   INSN(zip2, 0b111);
2575 
2576 #undef INSN
2577 
2578   // CRC32 instructions
2579 #define INSN(NAME, c, sf, sz)                                             \
2580   void NAME(Register Rd, Register Rn, Register Rm) {                      \
2581     starti;                                                               \


2627   INSN(fneg,  1, 0b01111);
2628 #undef ASSERTION
2629 
2630 #define ASSERTION (T == T8B || T == T16B || T == T4H || T == T8H || T == T2S || T == T4S)
2631   INSN(rev64, 0, 0b00000);
2632 #undef ASSERTION
2633 
2634 #define ASSERTION (T == T8B || T == T16B || T == T4H || T == T8H)
2635   INSN(rev32, 1, 0b00000);
2636 private:
2637   INSN(_rbit, 1, 0b00101);
2638 public:
2639 
2640 #undef ASSERTION
2641 
2642 #define ASSERTION (T == T8B || T == T16B)
2643   INSN(rev16, 0, 0b00001);
2644   // RBIT only allows T8B and T16B but encodes them oddly.  Argh...
2645   void rbit(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {
2646     assert((ASSERTION), MSG);
2647     _rbit(Vd, SIMD_Arrangement((T & 1) | 0b010), Vn);
2648   }
2649 #undef ASSERTION
2650 
2651 #undef MSG
2652 
2653 #undef INSN
2654 
2655 void ext(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm, int index)
2656   {
2657     starti;
2658     assert(T == T8B || T == T16B, "invalid arrangement");
2659     assert((T == T8B && index <= 0b0111) || (T == T16B && index <= 0b1111), "Invalid index value");
2660     f(0, 31), f((int)T & 1, 30), f(0b101110000, 29, 21);
2661     rf(Vm, 16), f(0, 15), f(index, 14, 11);
2662     f(0, 10), rf(Vn, 5), rf(Vd, 0);
2663   }
2664 
2665 /* Simulator extensions to the ISA
2666 
2667    haltsim




 259     f(r->encoding_nocheck(), lsb + 4, lsb);
 260   }
 261 
 262   // reg|ZR
 263   void zrf(Register r, int lsb) {
 264     f(r->encoding_nocheck() - (r == zr), lsb + 4, lsb);
 265   }
 266 
 267   // reg|SP
 268   void srf(Register r, int lsb) {
 269     f(r == sp ? 31 : r->encoding_nocheck(), lsb + 4, lsb);
 270   }
 271 
 272   void rf(FloatRegister r, int lsb) {
 273     f(r->encoding_nocheck(), lsb + 4, lsb);
 274   }
 275 
 276   unsigned get(int msb = 31, int lsb = 0) {
 277     int nbits = msb - lsb + 1;
 278     unsigned mask = ((1U << nbits) - 1) << lsb;
 279     assert_cond(bits & mask == mask);
 280     return (insn & mask) >> lsb;
 281   }
 282 
 283   void fixed(unsigned value, unsigned mask) {
 284     assert_cond ((mask & bits) == 0);
 285 #ifdef ASSERT
 286     bits |= mask;
 287 #endif
 288     insn |= value;
 289   }
 290 };
 291 
 292 #define starti Instruction_aarch64 do_not_use(this); set_current(&do_not_use)
 293 
 294 class PrePost {
 295   int _offset;
 296   Register _r;
 297 public:
 298   PrePost(Register reg, int o) : _offset(o), _r(reg) { }
 299   int offset() { return _offset; }
 300   Register reg() { return _r; }
 301 };
 302 
 303 class Pre : public PrePost {
 304 public:
 305   Pre(Register reg, int o) : PrePost(reg, o) { }
 306 };
 307 class Post : public PrePost {
 308   Register _idx;

 309 public:
 310   Post(Register reg, int o) : PrePost(reg, o) { _idx = NULL; }
 311   Post(Register reg, Register idx) : PrePost(reg, 0) { _idx = idx; }
 312   Register idx_reg() { return _idx; }

 313 };
 314 
 315 namespace ext
 316 {
 317   enum operation { uxtb, uxth, uxtw, uxtx, sxtb, sxth, sxtw, sxtx };
 318 };
 319 
 320 // Addressing modes
 321 class Address {
 322  public:
 323 
 324   enum mode { no_mode, base_plus_offset, pre, post, post_reg, pcrel,
 325               base_plus_offset_reg, literal };
 326 
 327   // Shift and extend for base reg + reg offset addressing
 328   class extend {
 329     int _option, _shift;
 330     ext::operation _op;
 331   public:
 332     extend() { }


 376     : _mode(no_mode) { }
 377   Address(Register r)
 378     : _base(r), _index(noreg), _offset(0), _mode(base_plus_offset), _target(0) { }
 379   Address(Register r, int o)
 380     : _base(r), _index(noreg), _offset(o), _mode(base_plus_offset), _target(0) { }
 381   Address(Register r, long o)
 382     : _base(r), _index(noreg), _offset(o), _mode(base_plus_offset), _target(0) { }
 383   Address(Register r, unsigned long o)
 384     : _base(r), _index(noreg), _offset(o), _mode(base_plus_offset), _target(0) { }
 385 #ifdef ASSERT
 386   Address(Register r, ByteSize disp)
 387     : _base(r), _index(noreg), _offset(in_bytes(disp)), _mode(base_plus_offset), _target(0) { }
 388 #endif
 389   Address(Register r, Register r1, extend ext = lsl())
 390     : _base(r), _index(r1), _offset(0), _mode(base_plus_offset_reg),
 391       _ext(ext), _target(0) { }
 392   Address(Pre p)
 393     : _base(p.reg()), _offset(p.offset()), _mode(pre) { }
 394   Address(Post p)
 395     : _base(p.reg()),  _index(p.idx_reg()), _offset(p.offset()),
 396       _mode(p.idx_reg() == NULL ? post : post_reg), _target(0) { }
 397   Address(address target, RelocationHolder const& rspec)
 398     : _mode(literal),
 399       _rspec(rspec),
 400       _is_lval(false),
 401       _target(target)  { }
 402   Address(address target, relocInfo::relocType rtype = relocInfo::external_word_type);
 403   Address(Register base, RegisterOrConstant index, extend ext = lsl())
 404     : _base (base),
 405       _offset(0), _ext(ext), _target(0) {
 406     if (index.is_register()) {
 407       _mode = base_plus_offset_reg;
 408       _index = index.as_register();
 409     } else {
 410       guarantee(ext.option() == ext::uxtx, "should be");
 411       assert(index.is_constant(), "should be");
 412       _mode = base_plus_offset;
 413       _offset = index.as_constant() << ext.shift();
 414     }
 415   }
 416 


 790   // Move wide (immediate)
 791 #define INSN(NAME, opcode)                                              \
 792   void NAME(Register Rd, unsigned imm, unsigned shift = 0) {            \
 793     assert_cond((shift/16)*16 == shift);                                \
 794     starti;                                                             \
 795     f(opcode, 31, 29), f(0b100101, 28, 23), f(shift/16, 22, 21),        \
 796       f(imm, 20, 5);                                                    \
 797     rf(Rd, 0);                                                          \
 798   }
 799 
 800   INSN(movnw, 0b000);
 801   INSN(movzw, 0b010);
 802   INSN(movkw, 0b011);
 803   INSN(movn, 0b100);
 804   INSN(movz, 0b110);
 805   INSN(movk, 0b111);
 806 
 807 #undef INSN
 808 
 809   // Bitfield
 810 #define INSN(NAME, opcode)                                              \
 811   void NAME(Register Rd, Register Rn, unsigned immr, unsigned imms) {   \
 812     starti;                                                             \

 813     f(opcode, 31, 22), f(immr, 21, 16), f(imms, 15, 10);                \
 814     zrf(Rn, 5), rf(Rd, 0);                                              \
 815   }
 816 
 817   INSN(sbfmw, 0b0001001100);
 818   INSN(bfmw,  0b0011001100);
 819   INSN(ubfmw, 0b0101001100);
 820   INSN(sbfm,  0b1001001101);
 821   INSN(bfm,   0b1011001101);
 822   INSN(ubfm,  0b1101001101);
 823 
 824 #undef INSN
 825 
 826   // Extract
 827 #define INSN(NAME, opcode)                                              \
 828   void NAME(Register Rd, Register Rn, Register Rm, unsigned imms) {     \
 829     starti;                                                             \

 830     f(opcode, 31, 21), f(imms, 15, 10);                                 \
 831     rf(Rm, 16), rf(Rn, 5), rf(Rd, 0);                                   \
 832   }
 833 
 834   INSN(extrw, 0b00010011100);
 835   INSN(extr,  0b10010011110);
 836 
 837 #undef INSN
 838 
 839   // The maximum range of a branch is fixed for the AArch64
 840   // architecture.  In debug mode we shrink it in order to test
 841   // trampolines, but not so small that branches in the interpreter
 842   // are out of range.
 843   static const unsigned long branch_range = NOT_DEBUG(128 * M) DEBUG_ONLY(2 * M);
 844 
 845   static bool reachable_from_branch_at(address branch, address target) {
 846     return uabs(target - branch) < branch_range;
 847   }
 848 
 849   // Unconditional branch (immediate)
 850 #define INSN(NAME, opcode)                                              \
 851   void NAME(address dest) {                                             \
 852     starti;                                                             \
 853     long offset = (dest - pc()) >> 2;                                   \
 854     DEBUG_ONLY(assert(reachable_from_branch_at(pc(), dest), "debug only")); \
 855     f(opcode, 31), f(0b00101, 30, 26), sf(offset, 25, 0);               \


1109 
1110 #undef INSN
1111 
1112 #define INSN(NAME, opc)                         \
1113   void NAME() {                 \
1114     branch_reg(dummy_reg, opc);         \
1115   }
1116 
1117   INSN(eret, 0b0100);
1118   INSN(drps, 0b0101);
1119 
1120 #undef INSN
1121 
1122   // Load/store exclusive
1123   enum operand_size { byte, halfword, word, xword };
1124 
1125   void load_store_exclusive(Register Rs, Register Rt1, Register Rt2,
1126     Register Rn, enum operand_size sz, int op, bool ordered) {
1127     starti;
1128     f(sz, 31, 30), f(0b001000, 29, 24), f(op, 23, 21);
1129     rf(Rs, 16), f(ordered, 15), rf(Rt2, 10), srf(Rn, 5), zrf(Rt1, 0);
1130   }
1131 
1132   void load_exclusive(Register dst, Register addr,
1133                       enum operand_size sz, bool ordered) {
1134     load_store_exclusive(dummy_reg, dst, dummy_reg, addr,
1135                          sz, 0b010, ordered);
1136   }
1137 
1138   void store_exclusive(Register status, Register new_val, Register addr,
1139                        enum operand_size sz, bool ordered) {
1140     load_store_exclusive(status, new_val, dummy_reg, addr,
1141                          sz, 0b000, ordered);
1142   }
1143 
1144 #define INSN4(NAME, sz, op, o0) /* Four registers */                    \
1145   void NAME(Register Rs, Register Rt1, Register Rt2, Register Rn) {     \
1146     guarantee(Rs != Rn && Rs != Rt1 && Rs != Rt2, "unpredictable instruction"); \
1147     load_store_exclusive(Rs, Rt1, Rt2, Rn, sz, op, o0);                 \
1148   }
1149 


1238   // CASP
1239 #define INSN(NAME, a, r)                                                \
1240   void NAME(operand_size sz, Register Rs, Register Rs1,                 \
1241             Register Rt, Register Rt1, Register Rn) {                   \
1242     assert((Rs->encoding() & 1) == 0 && (Rt->encoding() & 1) == 0 &&    \
1243            Rs->successor() == Rs1 && Rt->successor() == Rt1 &&          \
1244            Rs != Rn && Rs1 != Rn && Rs != Rt, "invalid registers");     \
1245     lse_cas(Rs, Rt, Rn, sz, a, r, false);                               \
1246   }
1247   INSN(casp,    false, false)
1248   INSN(caspa,   true,  false)
1249   INSN(caspl,   false, true)
1250   INSN(caspal,  true,  true)
1251 #undef INSN
1252 
1253   // 8.1 Atomic operations
1254   void lse_atomic(Register Rs, Register Rt, Register Rn,
1255                   enum operand_size sz, int op1, int op2, bool a, bool r) {
1256     starti;
1257     f(sz, 31, 30), f(0b111000, 29, 24), f(a, 23), f(r, 22), f(1, 21);
1258     rf(Rs, 16), f(op1, 15), f(op2, 14, 12), f(0, 11, 10), srf(Rn, 5), zrf(Rt, 0);
1259   }
1260 
1261 #define INSN(NAME, NAME_A, NAME_L, NAME_AL, op1, op2)                   \
1262   void NAME(operand_size sz, Register Rs, Register Rt, Register Rn) {   \
1263     lse_atomic(Rs, Rt, Rn, sz, op1, op2, false, false);                 \
1264   }                                                                     \
1265   void NAME_A(operand_size sz, Register Rs, Register Rt, Register Rn) { \
1266     lse_atomic(Rs, Rt, Rn, sz, op1, op2, true, false);                  \
1267   }                                                                     \
1268   void NAME_L(operand_size sz, Register Rs, Register Rt, Register Rn) { \
1269     lse_atomic(Rs, Rt, Rn, sz, op1, op2, false, true);                  \
1270   }                                                                     \
1271   void NAME_AL(operand_size sz, Register Rs, Register Rt, Register Rn) {\
1272     lse_atomic(Rs, Rt, Rn, sz, op1, op2, true, true);                   \
1273   }
1274   INSN(ldadd,  ldadda,  ldaddl,  ldaddal,  0, 0b000);
1275   INSN(ldbic,  ldbica,  ldbicl,  ldbical,  0, 0b001);
1276   INSN(ldeor,  ldeora,  ldeorl,  ldeoral,  0, 0b010);
1277   INSN(ldorr,  ldorra,  ldorrl,  ldorral,  0, 0b011);
1278   INSN(ldsmax, ldsmaxa, ldsmaxl, ldsmaxal, 0, 0b100);


1460 
1461 #undef INSN
1462 
1463   enum shift_kind { LSL, LSR, ASR, ROR };
1464 
1465   void op_shifted_reg(unsigned decode,
1466                       enum shift_kind kind, unsigned shift,
1467                       unsigned size, unsigned op) {
1468     f(size, 31);
1469     f(op, 30, 29);
1470     f(decode, 28, 24);
1471     f(shift, 15, 10);
1472     f(kind, 23, 22);
1473   }
1474 
1475   // Logical (shifted register)
1476 #define INSN(NAME, size, op, N)                                 \
1477   void NAME(Register Rd, Register Rn, Register Rm,              \
1478             enum shift_kind kind = LSL, unsigned shift = 0) {   \
1479     starti;                                                     \

1480     f(N, 21);                                                   \
1481     zrf(Rm, 16), zrf(Rn, 5), zrf(Rd, 0);                        \
1482     op_shifted_reg(0b01010, kind, shift, size, op);             \
1483   }
1484 
1485   INSN(andr, 1, 0b00, 0);
1486   INSN(orr, 1, 0b01, 0);
1487   INSN(eor, 1, 0b10, 0);
1488   INSN(ands, 1, 0b11, 0);
1489   INSN(andw, 0, 0b00, 0);
1490   INSN(orrw, 0, 0b01, 0);
1491   INSN(eorw, 0, 0b10, 0);
1492   INSN(andsw, 0, 0b11, 0);
1493 
1494 #undef INSN
1495 
1496 #define INSN(NAME, size, op, N)                                         \
1497   void NAME(Register Rd, Register Rn, Register Rm,                      \
1498             enum shift_kind kind = LSL, unsigned shift = 0) {           \
1499     starti;                                                             \


1522 #undef INSN
1523 
1524   // Aliases for short forms of orn
1525 void mvn(Register Rd, Register Rm,
1526             enum shift_kind kind = LSL, unsigned shift = 0) {
1527   orn(Rd, zr, Rm, kind, shift);
1528 }
1529 
1530 void mvnw(Register Rd, Register Rm,
1531             enum shift_kind kind = LSL, unsigned shift = 0) {
1532   ornw(Rd, zr, Rm, kind, shift);
1533 }
1534 
1535   // Add/subtract (shifted register)
1536 #define INSN(NAME, size, op)                            \
1537   void NAME(Register Rd, Register Rn, Register Rm,      \
1538             enum shift_kind kind, unsigned shift = 0) { \
1539     starti;                                             \
1540     f(0, 21);                                           \
1541     assert_cond(kind != ROR);                           \

1542     zrf(Rd, 0), zrf(Rn, 5), zrf(Rm, 16);                \
1543     op_shifted_reg(0b01011, kind, shift, size, op);     \
1544   }
1545 
1546   INSN(add, 1, 0b000);
1547   INSN(sub, 1, 0b10);
1548   INSN(addw, 0, 0b000);
1549   INSN(subw, 0, 0b10);
1550 
1551   INSN(adds, 1, 0b001);
1552   INSN(subs, 1, 0b11);
1553   INSN(addsw, 0, 0b001);
1554   INSN(subsw, 0, 0b11);
1555 
1556 #undef INSN
1557 
1558   // Add/subtract (extended register)
1559 #define INSN(NAME, op)                                                  \
1560   void NAME(Register Rd, Register Rn, Register Rm,                      \
1561            ext::operation option, int amount = 0) {                     \
1562     starti;                                                             \
1563     zrf(Rm, 16), srf(Rn, 5), srf(Rd, 0);                                \
1564     add_sub_extended_reg(op, 0b01011, Rd, Rn, Rm, 0b00, option, amount); \
1565   }
1566 
1567   void add_sub_extended_reg(unsigned op, unsigned decode,
1568     Register Rd, Register Rn, Register Rm,
1569     unsigned opt, ext::operation option, unsigned imm) {
1570     guarantee(imm <= 4, "shift amount must be < 4");
1571     f(op, 31, 29), f(decode, 28, 24), f(opt, 23, 22), f(1, 21);
1572     f(option, 15, 13), f(imm, 12, 10);
1573   }
1574 
1575   INSN(addw, 0b000);
1576   INSN(subw, 0b010);
1577   INSN(add, 0b100);
1578   INSN(sub, 0b110);
1579 
1580 #undef INSN
1581 
1582 #define INSN(NAME, op)                                                  \
1583   void NAME(Register Rd, Register Rn, Register Rm,                      \
1584            ext::operation option, int amount = 0) {                     \
1585     starti;                                                             \
1586     zrf(Rm, 16), srf(Rn, 5), zrf(Rd, 0);                                \
1587     add_sub_extended_reg(op, 0b01011, Rd, Rn, Rm, 0b00, option, amount); \
1588   }
1589 
1590   INSN(addsw, 0b001);


1635   INSN(sbcsw, 0b011);
1636   INSN(adc, 0b100);
1637   INSN(adcs, 0b101);
1638   INSN(sbc,0b110);
1639   INSN(sbcs, 0b111);
1640 
1641 #undef INSN
1642 
1643   // Conditional compare (both kinds)
1644   void conditional_compare(unsigned op, int o1, int o2, int o3,
1645                            Register Rn, unsigned imm5, unsigned nzcv,
1646                            unsigned cond) {
1647     starti;
1648     f(op, 31, 29);
1649     f(0b11010010, 28, 21);
1650     f(cond, 15, 12);
1651     f(o1, 11);
1652     f(o2, 10);
1653     f(o3, 4);
1654     f(nzcv, 3, 0);
1655     f(imm5, 20, 16), rf(Rn, 5);
1656   }
1657 
1658 #define INSN(NAME, op)                                                  \
1659   void NAME(Register Rn, Register Rm, int imm, Condition cond) {        \
1660     int regNumber = (Rm == zr ? 31 : (uintptr_t)Rm);                    \
1661     conditional_compare(op, 0, 0, 0, Rn, regNumber, imm, cond);         \
1662   }                                                                     \
1663                                                                         \
1664   void NAME(Register Rn, int imm5, int imm, Condition cond) {           \
1665     conditional_compare(op, 1, 0, 0, Rn, imm5, imm, cond);              \
1666   }
1667 
1668   INSN(ccmnw, 0b001);
1669   INSN(ccmpw, 0b011);
1670   INSN(ccmn, 0b101);
1671   INSN(ccmp, 0b111);
1672 
1673 #undef INSN
1674 
1675   // Conditional select


2104 #define INSN(NAME, op)                                            \
2105   void NAME(FloatRegister Rt, SIMD_RegVariant T, const Address &adr) {   \
2106     ld_st2((Register)Rt, adr, (int)T & 3, op + ((T==Q) ? 0b10:0b00), 1); \
2107   }                                                                      \
2108 
2109   INSN(ldr, 1);
2110   INSN(str, 0);
2111 
2112 #undef INSN
2113 
2114  private:
2115 
2116   void ld_st(FloatRegister Vt, SIMD_Arrangement T, Register Xn, int op1, int op2) {
2117     starti;
2118     f(0,31), f((int)T & 1, 30);
2119     f(op1, 29, 21), f(0, 20, 16), f(op2, 15, 12);
2120     f((int)T >> 1, 11, 10), srf(Xn, 5), rf(Vt, 0);
2121   }
2122   void ld_st(FloatRegister Vt, SIMD_Arrangement T, Register Xn,
2123              int imm, int op1, int op2, int regs) {
2124     guarantee(T <= T1Q && imm == SIMD_Size_in_bytes[T] * regs, "bad offset");





2125     starti;
2126     f(0,31), f((int)T & 1, 30);
2127     f(op1 | 0b100, 29, 21), f(0b11111, 20, 16), f(op2, 15, 12);
2128     f((int)T >> 1, 11, 10), srf(Xn, 5), rf(Vt, 0);
2129   }
2130   void ld_st(FloatRegister Vt, SIMD_Arrangement T, Register Xn,
2131              Register Xm, int op1, int op2) {
2132     starti;
2133     f(0,31), f((int)T & 1, 30);
2134     f(op1 | 0b100, 29, 21), rf(Xm, 16), f(op2, 15, 12);
2135     f((int)T >> 1, 11, 10), srf(Xn, 5), rf(Vt, 0);
2136   }
2137 
2138   void ld_st(FloatRegister Vt, SIMD_Arrangement T, Address a, int op1, int op2, int regs) {
2139     switch (a.getMode()) {
2140     case Address::base_plus_offset:
2141       guarantee(a.offset() == 0, "no offset allowed here");
2142       ld_st(Vt, T, a.base(), op1, op2);
2143       break;
2144     case Address::post:


2211 
2212 #define INSN(NAME, opc)                                                                 \
2213   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \
2214     starti;                                                                             \
2215     assert(T == T8B || T == T16B, "must be T8B or T16B");                               \
2216     f(0, 31), f((int)T & 1, 30), f(opc, 29, 21);                                        \
2217     rf(Vm, 16), f(0b000111, 15, 10), rf(Vn, 5), rf(Vd, 0);                              \
2218   }
2219 
2220   INSN(eor,  0b101110001);
2221   INSN(orr,  0b001110101);
2222   INSN(andr, 0b001110001);
2223   INSN(bic,  0b001110011);
2224   INSN(bif,  0b101110111);
2225   INSN(bit,  0b101110101);
2226   INSN(bsl,  0b101110011);
2227   INSN(orn,  0b001110111);
2228 
2229 #undef INSN
2230 
2231 #define INSN(NAME, opc, opc2)                                                                 \
2232   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \


2233     starti;                                                                             \
2234     f(0, 31), f((int)T & 1, 30), f(opc, 29), f(0b01110, 28, 24);                        \
2235     f((int)T >> 1, 23, 22), f(1, 21), rf(Vm, 16), f(opc2, 15, 10);                      \
2236     rf(Vn, 5), rf(Vd, 0);                                                               \
2237   }
2238 
2239   INSN(addv,   0, 0b100001);
2240   INSN(subv,   1, 0b100001);
2241   INSN(mulv,   0, 0b100111);
2242   INSN(mlav,   0, 0b100101);
2243   INSN(mlsv,   1, 0b100101);
2244   INSN(sshl,   0, 0b010001);
2245   INSN(ushl,   1, 0b010001);
2246   INSN(umullv, 1, 0b110000);
2247   INSN(umlalv, 1, 0b100000);
2248 
2249 #undef INSN
2250 
2251 #define INSN(NAME, opc, opc2) \
2252   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {                   \



2253     starti;                                                                             \
2254     f(0, 31), f((int)T & 1, 30), f(opc, 29), f(0b01110, 28, 24);                        \
2255     f((int)T >> 1, 23, 22), f(opc2, 21, 10);                                            \
2256     rf(Vn, 5), rf(Vd, 0);                                                               \
2257   }
2258 
2259   INSN(absr,   0, 0b100000101110);
2260   INSN(negr,   1, 0b100000101110);
2261   INSN(notr,   1, 0b100000010110);
2262   INSN(addv,   0, 0b110001101110);
2263   INSN(cls,    0, 0b100000010010);
2264   INSN(clz,    1, 0b100000010010);
2265   INSN(cnt,    0, 0b100000010110);
2266   INSN(uaddlv, 1, 0b110000001110);
2267 
2268 #undef INSN
2269 
2270 #define INSN(NAME, opc) \
2271   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {                  \
2272     starti;                                                                            \
2273     assert(T == T4S, "arrangement must be T4S");                                       \
2274     f(0, 31), f((int)T & 1, 30), f(0b101110, 29, 24), f(opc, 23),                      \
2275     f(T == T4S ? 0 : 1, 22), f(0b110000111110, 21, 10); rf(Vn, 5), rf(Vd, 0);          \
2276   }
2277 
2278   INSN(fmaxv, 0);
2279   INSN(fminv, 1);
2280 
2281 #undef INSN
2282 
2283 #define INSN(NAME, op0, cmode0) \
2284   void NAME(FloatRegister Vd, SIMD_Arrangement T, unsigned imm8, unsigned lsl = 0) {   \
2285     unsigned cmode = cmode0;                                                           \
2286     unsigned op = op0;                                                                 \
2287     starti;                                                                            \
2288     assert(lsl == 0 ||                                                                 \
2289            ((T == T4H || T == T8H) && lsl == 8) ||                                     \
2290            ((T == T2S || T == T4S) && ((lsl >> 3) < 4)), "invalid shift");             \
2291     cmode |= lsl >> 2;                                                                 \
2292     if (T == T4H || T == T8H) cmode |= 0b1000;                                         \
2293     if (!(T == T4H || T == T8H || T == T2S || T == T4S)) {                             \
2294       assert(op == 0 && cmode0 == 0, "must be MOVI");                                  \
2295       cmode = 0b1110;                                                                  \
2296       if (T == T1D || T == T2D) op = 1;                                                \
2297     }                                                                                  \
2298     f(0, 31), f((int)T & 1, 30), f(op, 29), f(0b0111100000, 28, 19);                   \
2299     f(imm8 >> 5, 18, 16), f(cmode, 15, 12), f(0x01, 11, 10), f(imm8 & 0b11111, 9, 5);  \
2300     rf(Vd, 0);                                                                         \
2301   }
2302 
2303   INSN(movi, 0, 0);
2304   INSN(orri, 0, 1);
2305   INSN(mvni, 1, 0);
2306   INSN(bici, 1, 1);
2307 
2308 #undef INSN
2309 
2310 #define INSN(NAME, op1, op2, op3) \


2431      * for SHL shift is calculated as:                                  \
2432      *   0001 xxx       8B/16B, shift = UInt(immh:immb) - 8             \
2433      *   001x xxx       4H/8H,  shift = UInt(immh:immb) - 16            \
2434      *   01xx xxx       2S/4S,  shift = UInt(immh:immb) - 32            \
2435      *   1xxx xxx       1D/2D,  shift = UInt(immh:immb) - 64            \
2436      *   (1D is RESERVED)                                               \
2437      */                                                                 \
2438     assert((1 << ((T>>1)+3)) > shift, "Invalid Shift value");           \
2439     int cVal = (1 << (((T >> 1) + 3) + (isSHR ? 1 : 0)));               \
2440     int encodedShift = isSHR ? cVal - shift : cVal + shift;             \
2441     f(0, 31), f(T & 1, 30), f(opc, 29), f(0b011110, 28, 23),            \
2442     f(encodedShift, 22, 16); f(opc2, 15, 10), rf(Vn, 5), rf(Vd, 0);     \
2443   }
2444 
2445   INSN(shl,  0, 0b010101, /* isSHR = */ false);
2446   INSN(sshr, 0, 0b000001, /* isSHR = */ true);
2447   INSN(ushr, 1, 0b000001, /* isSHR = */ true);
2448 
2449 #undef INSN
2450 
2451   void ushll(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn, SIMD_Arrangement Tb, int shift) {

2452     starti;
2453     /* The encodings for the immh:immb fields (bits 22:16) are
2454      *   0001 xxx       8H, 8B/16b shift = xxx
2455      *   001x xxx       4S, 4H/8H  shift = xxxx
2456      *   01xx xxx       2D, 2S/4S  shift = xxxxx
2457      *   1xxx xxx       RESERVED
2458      */
2459     assert((Tb >> 1) + 1 == (Ta >> 1), "Incompatible arrangement");
2460     assert((1 << ((Tb>>1)+3)) > shift, "Invalid shift value");
2461     f(0, 31), f(Tb & 1, 30), f(0b1011110, 29, 23), f((1 << ((Tb>>1)+3))|shift, 22, 16);
2462     f(0b101001, 15, 10), rf(Vn, 5), rf(Vd, 0);
2463   }







2464   void ushll2(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn,  SIMD_Arrangement Tb, int shift) {
2465     ushll(Vd, Ta, Vn, Tb, shift);

2466   }
2467 
2468   // Move from general purpose register
2469   //   mov  Vd.T[index], Rn
2470   void mov(FloatRegister Vd, SIMD_Arrangement T, int index, Register Xn) {
2471     starti;
2472     f(0b01001110000, 31, 21), f(((1 << (T >> 1)) | (index << ((T >> 1) + 1))), 20, 16);
2473     f(0b000111, 15, 10), rf(Xn, 5), rf(Vd, 0);
2474   }
2475 
2476   // Move to general purpose register
2477   //   mov  Rd, Vn.T[index]
2478   void mov(Register Xd, FloatRegister Vn, SIMD_Arrangement T, int index) {

2479     starti;
2480     f(0, 31), f((T >= T1D) ? 1:0, 30), f(0b001110000, 29, 21);
2481     f(((1 << (T >> 1)) | (index << ((T >> 1) + 1))), 20, 16);
2482     f(0b001111, 15, 10), rf(Vn, 5), rf(Xd, 0);
2483   }
2484 
2485   void pmull(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn, FloatRegister Vm, SIMD_Arrangement Tb) {

2486     starti;
2487     assert((Ta == T1Q && (Tb == T1D || Tb == T2D)) ||
2488            (Ta == T8H && (Tb == T8B || Tb == T16B)), "Invalid Size specifier");
2489     int size = (Ta == T1Q) ? 0b11 : 0b00;
2490     f(0, 31), f(Tb & 1, 30), f(0b001110, 29, 24), f(size, 23, 22);
2491     f(1, 21), rf(Vm, 16), f(0b111000, 15, 10), rf(Vn, 5), rf(Vd, 0);
2492   }







2493   void pmull2(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn, FloatRegister Vm, SIMD_Arrangement Tb) {
2494     assert(Tb == T2D || Tb == T16B, "pmull2 assumes T2D or T16B as the second size specifier");
2495     pmull(Vd, Ta, Vn, Vm, Tb);
2496   }
2497 
2498   void uqxtn(FloatRegister Vd, SIMD_Arrangement Tb, FloatRegister Vn, SIMD_Arrangement Ta) {
2499     starti;
2500     int size_b = (int)Tb >> 1;
2501     int size_a = (int)Ta >> 1;
2502     assert(size_b < 3 && size_b == size_a - 1, "Invalid size specifier");
2503     f(0, 31), f(Tb & 1, 30), f(0b101110, 29, 24), f(size_b, 23, 22);
2504     f(0b100001010010, 21, 10), rf(Vn, 5), rf(Vd, 0);
2505   }
2506 
2507   void dup(FloatRegister Vd, SIMD_Arrangement T, Register Xs)
2508   {
2509     starti;
2510     assert(T != T1D, "reserved encoding");
2511     f(0,31), f((int)T & 1, 30), f(0b001110000, 29, 21);
2512     f((1 << (T >> 1)), 20, 16), f(0b000011, 15, 10), rf(Xs, 5), rf(Vd, 0);
2513   }
2514 
2515   void dup(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, int index = 0)
2516   {
2517     starti;
2518     assert(T != T1D, "reserved encoding");
2519     f(0, 31), f((int)T & 1, 30), f(0b001110000, 29, 21);
2520     f(((1 << (T >> 1)) | (index << ((T >> 1) + 1))), 20, 16);
2521     f(0b000001, 15, 10), rf(Vn, 5), rf(Vd, 0);
2522   }
2523 
2524   // AdvSIMD ZIP/UZP/TRN
2525 #define INSN(NAME, opcode)                                              \
2526   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \

2527     starti;                                                             \
2528     f(0, 31), f(0b001110, 29, 24), f(0, 21), f(0, 15);                  \
2529     f(opcode, 14, 12), f(0b10, 11, 10);                                 \
2530     rf(Vm, 16), rf(Vn, 5), rf(Vd, 0);                                   \
2531     f(T & 1, 30), f(T >> 1, 23, 22);                                    \
2532   }
2533 
2534   INSN(uzp1, 0b001);
2535   INSN(trn1, 0b010);
2536   INSN(zip1, 0b011);
2537   INSN(uzp2, 0b101);
2538   INSN(trn2, 0b110);
2539   INSN(zip2, 0b111);
2540 
2541 #undef INSN
2542 
2543   // CRC32 instructions
2544 #define INSN(NAME, c, sf, sz)                                             \
2545   void NAME(Register Rd, Register Rn, Register Rm) {                      \
2546     starti;                                                               \


2592   INSN(fneg,  1, 0b01111);
2593 #undef ASSERTION
2594 
2595 #define ASSERTION (T == T8B || T == T16B || T == T4H || T == T8H || T == T2S || T == T4S)
2596   INSN(rev64, 0, 0b00000);
2597 #undef ASSERTION
2598 
2599 #define ASSERTION (T == T8B || T == T16B || T == T4H || T == T8H)
2600   INSN(rev32, 1, 0b00000);
2601 private:
2602   INSN(_rbit, 1, 0b00101);
2603 public:
2604 
2605 #undef ASSERTION
2606 
2607 #define ASSERTION (T == T8B || T == T16B)
2608   INSN(rev16, 0, 0b00001);
2609   // RBIT only allows T8B and T16B but encodes them oddly.  Argh...
2610   void rbit(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {
2611     assert((ASSERTION), MSG);
2612     _rbit(Vd, SIMD_Arrangement(T & 1 | 0b010), Vn);
2613   }
2614 #undef ASSERTION
2615 
2616 #undef MSG
2617 
2618 #undef INSN
2619 
2620 void ext(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm, int index)
2621   {
2622     starti;
2623     assert(T == T8B || T == T16B, "invalid arrangement");
2624     assert((T == T8B && index <= 0b0111) || (T == T16B && index <= 0b1111), "Invalid index value");
2625     f(0, 31), f((int)T & 1, 30), f(0b101110000, 29, 21);
2626     rf(Vm, 16), f(0, 15), f(index, 14, 11);
2627     f(0, 10), rf(Vn, 5), rf(Vd, 0);
2628   }
2629 
2630 /* Simulator extensions to the ISA
2631 
2632    haltsim


< prev index next >