< prev index next >

src/hotspot/cpu/aarch64/assembler_aarch64.hpp

Print this page

 566     i->sf(_offset / size, 21, 15);
 567     i->srf(_base, 5);
 568   }
 569 
 570   void encode_nontemporal_pair(Instruction_aarch64 *i) const {
 571     // Only base + offset is allowed
 572     i->f(0b000, 25, 23);
 573     unsigned size = i->get(31, 31);
 574     size = 4 << size;
 575     guarantee(_offset % size == 0, "bad offset");
 576     i->sf(_offset / size, 21, 15);
 577     i->srf(_base, 5);
 578     guarantee(_mode == Address::base_plus_offset,
 579               "Bad addressing mode for non-temporal op");
 580   }
 581 
 582   void lea(MacroAssembler *, Register) const;
 583 
 584   static bool offset_ok_for_immed(int64_t offset, uint shift);
 585 
 586   static bool offset_ok_for_sve_immed(long offset, int shift, int vl /* sve vector length */) {
 587     if (offset % vl == 0) {
 588       // Convert address offset into sve imm offset (MUL VL).
 589       int sve_offset = offset / vl;
 590       if (((-(1 << (shift - 1))) <= sve_offset) && (sve_offset < (1 << (shift - 1)))) {
 591         // sve_offset can be encoded
 592         return true;
 593       }
 594     }
 595     return false;
 596   }
 597 };
 598 
 599 // Convience classes
 600 class RuntimeAddress: public Address {
 601 
 602   public:
 603 
 604   RuntimeAddress(address target) : Address(target, relocInfo::runtime_call_type) {}
 605 
 606 };

2948     starti;                                                                            \
2949     assert(T == S || T == D, "invalid register variant");                              \
2950     f(0b01100101, 31, 24), f(T, 23, 22), f(0, 21),                                     \
2951     rf(Zm, 16), f(0, 15, 13), f(opcode, 12, 10), rf(Zn, 5), rf(Zd, 0);                 \
2952   }
2953 
2954   INSN(sve_fadd, 0b000);
2955   INSN(sve_fmul, 0b010);
2956   INSN(sve_fsub, 0b001);
2957 #undef INSN
2958 
2959 private:
2960   void sve_predicate_reg_insn(unsigned op24, unsigned op13,
2961                               FloatRegister Zd_or_Vd, SIMD_RegVariant T,
2962                               PRegister Pg, FloatRegister Zn_or_Vn) {
2963     starti;
2964     f(op24, 31, 24), f(T, 23, 22), f(op13, 21, 13);
2965     pgrf(Pg, 10), rf(Zn_or_Vn, 5), rf(Zd_or_Vd, 0);
2966   }
2967 


























2968 public:
2969 
2970 // SVE integer arithmetic - predicate
2971 #define INSN(NAME, op1, op2)                                                                            \
2972   void NAME(FloatRegister Zdn_or_Zd_or_Vd, SIMD_RegVariant T, PRegister Pg, FloatRegister Znm_or_Vn) {  \
2973     assert(T != Q, "invalid register variant");                                                         \
2974     sve_predicate_reg_insn(op1, op2, Zdn_or_Zd_or_Vd, T, Pg, Znm_or_Vn);                                \
2975   }
2976 
2977   INSN(sve_abs,  0b00000100, 0b010110101); // vector abs, unary
2978   INSN(sve_add,  0b00000100, 0b000000000); // vector add

2979   INSN(sve_andv, 0b00000100, 0b011010001); // bitwise and reduction to scalar
2980   INSN(sve_asr,  0b00000100, 0b010000100); // vector arithmetic shift right
2981   INSN(sve_cnt,  0b00000100, 0b011010101)  // count non-zero bits
2982   INSN(sve_cpy,  0b00000101, 0b100000100); // copy scalar to each active vector element

2983   INSN(sve_eorv, 0b00000100, 0b011001001); // bitwise xor reduction to scalar
2984   INSN(sve_lsl,  0b00000100, 0b010011100); // vector logical shift left
2985   INSN(sve_lsr,  0b00000100, 0b010001100); // vector logical shift right
2986   INSN(sve_mul,  0b00000100, 0b010000000); // vector mul
2987   INSN(sve_neg,  0b00000100, 0b010111101); // vector neg, unary
2988   INSN(sve_not,  0b00000100, 0b011110101); // bitwise invert vector, unary

2989   INSN(sve_orv,  0b00000100, 0b011000001); // bitwise or reduction to scalar
2990   INSN(sve_smax, 0b00000100, 0b001000000); // signed maximum vectors
2991   INSN(sve_smaxv, 0b00000100, 0b001000001); // signed maximum reduction to scalar
2992   INSN(sve_smin,  0b00000100, 0b001010000); // signed minimum vectors
2993   INSN(sve_sminv, 0b00000100, 0b001010001); // signed minimum reduction to scalar
2994   INSN(sve_sub,   0b00000100, 0b000001000); // vector sub
2995   INSN(sve_uaddv, 0b00000100, 0b000001001); // unsigned add reduction to scalar
2996 #undef INSN
2997 
2998 // SVE floating-point arithmetic - predicate
2999 #define INSN(NAME, op1, op2)                                                                          \
3000   void NAME(FloatRegister Zd_or_Zdn_or_Vd, SIMD_RegVariant T, PRegister Pg, FloatRegister Zn_or_Zm) { \
3001     assert(T == S || T == D, "invalid register variant");                                             \
3002     sve_predicate_reg_insn(op1, op2, Zd_or_Zdn_or_Vd, T, Pg, Zn_or_Zm);                               \
3003   }
3004 
3005   INSN(sve_fabs,    0b00000100, 0b011100101);
3006   INSN(sve_fadd,    0b01100101, 0b000000100);
3007   INSN(sve_fadda,   0b01100101, 0b011000001); // add strictly-ordered reduction to scalar Vd
3008   INSN(sve_fdiv,    0b01100101, 0b001101100);

3011   INSN(sve_fmin,    0b01100101, 0b000111100); // floating-point minimum
3012   INSN(sve_fminv,   0b01100101, 0b000111001); // floating-point minimum recursive reduction to scalar
3013   INSN(sve_fmul,    0b01100101, 0b000010100);
3014   INSN(sve_fneg,    0b00000100, 0b011101101);
3015   INSN(sve_frintm,  0b01100101, 0b000010101); // floating-point round to integral value, toward minus infinity
3016   INSN(sve_frintn,  0b01100101, 0b000000101); // floating-point round to integral value, nearest with ties to even
3017   INSN(sve_frintp,  0b01100101, 0b000001101); // floating-point round to integral value, toward plus infinity
3018   INSN(sve_fsqrt,   0b01100101, 0b001101101);
3019   INSN(sve_fsub,    0b01100101, 0b000001100);
3020 #undef INSN
3021 
3022   // SVE multiple-add/sub - predicated
3023 #define INSN(NAME, op0, op1, op2)                                                                     \
3024   void NAME(FloatRegister Zda, SIMD_RegVariant T, PRegister Pg, FloatRegister Zn, FloatRegister Zm) { \
3025     starti;                                                                                           \
3026     assert(T != Q, "invalid size");                                                                   \
3027     f(op0, 31, 24), f(T, 23, 22), f(op1, 21), rf(Zm, 16);                                             \
3028     f(op2, 15, 13), pgrf(Pg, 10), rf(Zn, 5), rf(Zda, 0);                                              \
3029   }
3030 
3031   INSN(sve_fmla,  0b01100101, 1, 0b000); // floating-point fused multiply-add: Zda = Zda + Zn * Zm
3032   INSN(sve_fmls,  0b01100101, 1, 0b001); // floating-point fused multiply-subtract: Zda = Zda + -Zn * Zm
3033   INSN(sve_fnmla, 0b01100101, 1, 0b010); // floating-point negated fused multiply-add: Zda = -Zda + -Zn * Zm
3034   INSN(sve_fnmls, 0b01100101, 1, 0b011); // floating-point negated fused multiply-subtract: Zda = -Zda + Zn * Zm

3035   INSN(sve_mla,   0b00000100, 0, 0b010); // multiply-add: Zda = Zda + Zn*Zm
3036   INSN(sve_mls,   0b00000100, 0, 0b011); // multiply-subtract: Zda = Zda + -Zn*Zm
3037 #undef INSN
3038 
3039 // SVE bitwise logical - unpredicated
3040 #define INSN(NAME, opc)                                              \
3041   void NAME(FloatRegister Zd, FloatRegister Zn, FloatRegister Zm) {  \
3042     starti;                                                          \
3043     f(0b00000100, 31, 24), f(opc, 23, 22), f(1, 21),                 \
3044     rf(Zm, 16), f(0b001100, 15, 10), rf(Zn, 5), rf(Zd, 0);           \
3045   }
3046   INSN(sve_and, 0b00);
3047   INSN(sve_eor, 0b10);
3048   INSN(sve_orr, 0b01);
3049   INSN(sve_bic, 0b11);
3050 #undef INSN
3051 
3052 // SVE shift immediate - unpredicated
3053 #define INSN(NAME, opc, isSHR)                                                  \
3054   void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, int shift) { \
3055     starti;                                                                     \
3056     /* The encodings for the tszh:tszl:imm3 fields (bits 23:22 20:19 18:16)     \
3057      * for shift right is calculated as:                                        \
3058      *   0001 xxx       B, shift = 16  - UInt(tszh:tszl:imm3)                   \
3059      *   001x xxx       H, shift = 32  - UInt(tszh:tszl:imm3)                   \
3060      *   01xx xxx       S, shift = 64  - UInt(tszh:tszl:imm3)                   \
3061      *   1xxx xxx       D, shift = 128 - UInt(tszh:tszl:imm3)                   \
3062      * for shift left is calculated as:                                         \
3063      *   0001 xxx       B, shift = UInt(tszh:tszl:imm3) - 8                     \
3064      *   001x xxx       H, shift = UInt(tszh:tszl:imm3) - 16                    \
3065      *   01xx xxx       S, shift = UInt(tszh:tszl:imm3) - 32                    \
3066      *   1xxx xxx       D, shift = UInt(tszh:tszl:imm3) - 64                    \
3067      */                                                                         \
3068     assert(T != Q, "Invalid register variant");                                 \
3069     if (isSHR) {                                                                \
3070       assert(((1 << (T + 3)) >= shift) && (shift > 0) , "Invalid shift value"); \
3071     } else {                                                                    \
3072       assert(((1 << (T + 3)) > shift) && (shift >= 0) , "Invalid shift value"); \
3073     }                                                                           \
3074     int cVal = (1 << ((T + 3) + (isSHR ? 1 : 0)));                              \
3075     int encodedShift = isSHR ? cVal - shift : cVal + shift;                     \
3076     int tszh = encodedShift >> 5;                                               \
3077     int tszl_imm = encodedShift & 0x1f;                                         \
3078     f(0b00000100, 31, 24);                                                      \
3079     f(tszh, 23, 22), f(1,21), f(tszl_imm, 20, 16);                              \
3080     f(0b100, 15, 13), f(opc, 12, 10), rf(Zn, 5), rf(Zd, 0);                     \
3081   }
3082 
3083   INSN(sve_asr, 0b100, /* isSHR = */ true);
3084   INSN(sve_lsl, 0b111, /* isSHR = */ false);
3085   INSN(sve_lsr, 0b101, /* isSHR = */ true);
3086 #undef INSN
3087 















3088 private:
3089 
3090   // Scalar base + immediate index
3091   void sve_ld_st1(FloatRegister Zt, Register Xn, int imm, PRegister Pg,
3092               SIMD_RegVariant T, int op1, int type, int op2) {
3093     starti;
3094     assert_cond(T >= type);
3095     f(op1, 31, 25), f(type, 24, 23), f(T, 22, 21);
3096     f(0, 20), sf(imm, 19, 16), f(op2, 15, 13);
3097     pgrf(Pg, 10), srf(Xn, 5), rf(Zt, 0);
3098   }
3099 
3100   // Scalar base + scalar index
3101   void sve_ld_st1(FloatRegister Zt, Register Xn, Register Xm, PRegister Pg,
3102               SIMD_RegVariant T, int op1, int type, int op2) {
3103     starti;
3104     assert_cond(T >= type);
3105     f(op1, 31, 25), f(type, 24, 23), f(T, 22, 21);
3106     rf(Xm, 16), f(op2, 15, 13);
3107     pgrf(Pg, 10), srf(Xn, 5), rf(Zt, 0);

3179     srf(Xn, 16), f(0b01010, 15, 11), sf(imm6, 10, 5), srf(Xd, 0); \
3180   }
3181 
3182   INSN(sve_addvl, 0b01); // Add multiple of vector register size to scalar register
3183   INSN(sve_addpl, 0b11); // Add multiple of predicate register size to scalar register
3184 #undef INSN
3185 
3186 // SVE inc/dec register by element count
3187 #define INSN(NAME, op) \
3188   void NAME(Register Xdn, SIMD_RegVariant T, unsigned imm4 = 1, int pattern = 0b11111) { \
3189     starti;                                                                              \
3190     assert(T != Q, "invalid size");                                                      \
3191     f(0b00000100,31, 24), f(T, 23, 22), f(0b11, 21, 20);                                 \
3192     f(imm4 - 1, 19, 16), f(0b11100, 15, 11), f(op, 10), f(pattern, 9, 5), rf(Xdn, 0);    \
3193   }
3194 
3195   INSN(sve_inc, 0);
3196   INSN(sve_dec, 1);
3197 #undef INSN
3198 


















3199   // SVE increment register by predicate count
3200   void sve_incp(const Register rd, SIMD_RegVariant T, PRegister pg) {
3201     starti;
3202     assert(T != Q, "invalid size");
3203     f(0b00100101, 31, 24), f(T, 23, 22), f(0b1011001000100, 21, 9),
3204     prf(pg, 5), rf(rd, 0);
3205   }
3206 
3207   // SVE broadcast general-purpose register to vector elements (unpredicated)
3208   void sve_dup(FloatRegister Zd, SIMD_RegVariant T, Register Rn) {
3209     starti;
3210     assert(T != Q, "invalid size");
3211     f(0b00000101, 31, 24), f(T, 23, 22), f(0b100000001110, 21, 10);
3212     srf(Rn, 5), rf(Zd, 0);
3213   }
3214 
3215   // SVE broadcast signed immediate to vector elements (unpredicated)
3216   void sve_dup(FloatRegister Zd, SIMD_RegVariant T, int imm8) {
3217     starti;
3218     assert(T != Q, "invalid size");
3219     int sh = 0;
3220     if (imm8 <= 127 && imm8 >= -128) {
3221       sh = 0;
3222     } else if (T != B && imm8 <= 32512 && imm8 >= -32768 && (imm8 & 0xff) == 0) {
3223       sh = 1;
3224       imm8 = (imm8 >> 8);
3225     } else {
3226       guarantee(false, "invalid immediate");
3227     }
3228     f(0b00100101, 31, 24), f(T, 23, 22), f(0b11100011, 21, 14);
3229     f(sh, 13), sf(imm8, 12, 5), rf(Zd, 0);
3230   }
3231 







3232   void sve_ptrue(PRegister pd, SIMD_RegVariant esize, int pattern = 0b11111) {
3233     starti;
3234     f(0b00100101, 31, 24), f(esize, 23, 22), f(0b011000111000, 21, 10);
3235     f(pattern, 9, 5), f(0b0, 4), prf(pd, 0);
3236   }
3237 




























3238   // SVE copy general-purpose register to vector elements (predicated)
3239   void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, Register Rn) {
3240     starti;
3241     assert(T != Q, "invalid size");
3242     f(0b00000101, 31, 24), f(T, 23, 22), f(0b101000101, 21, 13);
3243     pgrf(Pg, 10), srf(Rn, 5), rf(Zd, 0);
3244   }
3245 
3246   // SVE copy signed integer immediate to vector elements (predicated)
3247   void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, int imm8, bool isMerge) {
3248     starti;
3249     assert(T != Q, "invalid size");
3250     int sh = 0;
3251     if (imm8 <= 127 && imm8 >= -128) {
3252       sh = 0;
3253     } else if (T != B && imm8 <= 32512 && imm8 >= -32768 && (imm8 & 0xff) == 0) {
3254       sh = 1;
3255       imm8 = (imm8 >> 8);
3256     } else {
3257       guarantee(false, "invalid immediate");

3320   f(0b00100101, 31, 24), f(T, 23, 22), f(0b0, 21), sf(imm5, 20, 16),
3321   f((cond_op >> 1) & 0x7, 15, 13), pgrf(Pg, 10), rf(Zn, 5);
3322   f(cond_op & 0x1, 4), prf(Pd, 0);
3323 }
3324 
3325 // SVE unpack vector elements
3326 #define INSN(NAME, op) \
3327   void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn) { \
3328     starti;                                                          \
3329     assert(T != B && T != Q, "invalid size");                        \
3330     f(0b00000101, 31, 24), f(T, 23, 22), f(0b1100, 21, 18);          \
3331     f(op, 17, 16), f(0b001110, 15, 10), rf(Zn, 5), rf(Zd, 0);        \
3332   }
3333 
3334   INSN(sve_uunpkhi, 0b11); // Signed unpack and extend half of vector - high half
3335   INSN(sve_uunpklo, 0b10); // Signed unpack and extend half of vector - low half
3336   INSN(sve_sunpkhi, 0b01); // Unsigned unpack and extend half of vector - high half
3337   INSN(sve_sunpklo, 0b00); // Unsigned unpack and extend half of vector - low half
3338 #undef INSN
3339 












3340 // SVE permute vector elements
3341 #define INSN(NAME, op) \
3342   void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, FloatRegister Zm) { \
3343     starti;                                                                            \
3344     assert(T != Q, "invalid size");                                                    \
3345     f(0b00000101, 31, 24), f(T, 23, 22), f(0b1, 21), rf(Zm, 16);                       \
3346     f(0b01101, 15, 11), f(op, 10), rf(Zn, 5), rf(Zd, 0);                               \
3347   }
3348 
3349   INSN(sve_uzp1, 0b0); // Concatenate even elements from two vectors
3350   INSN(sve_uzp2, 0b1); // Concatenate odd elements from two vectors
3351 #undef INSN
3352 













3353 // Predicate counted loop (SVE) (32-bit variants are not included)
3354 #define INSN(NAME, decode)                                                \
3355   void NAME(PRegister Pd, SIMD_RegVariant T, Register Rn, Register Rm) {  \
3356     starti;                                                               \
3357     assert(T != Q, "invalid register variant");                           \
3358     f(0b00100101, 31, 24), f(T, 23, 22), f(1, 21),                        \
3359     zrf(Rm, 16), f(0, 15, 13), f(1, 12), f(decode >> 1, 11, 10),          \
3360     zrf(Rn, 5), f(decode & 1, 4), prf(Pd, 0);                             \
3361   }
3362 
3363   INSN(sve_whilelt, 0b010);  // While incrementing signed scalar less than scalar
3364   INSN(sve_whilele, 0b011);  // While incrementing signed scalar less than or equal to scalar
3365   INSN(sve_whilelo, 0b110);  // While incrementing unsigned scalar lower than scalar
3366   INSN(sve_whilels, 0b111);  // While incrementing unsigned scalar lower than or the same as scalar
3367 #undef INSN
3368 
3369   // SVE predicate reverse
3370   void sve_rev(PRegister Pd, SIMD_RegVariant T, PRegister Pn) {
3371     starti;
3372     assert(T != Q, "invalid size");

 566     i->sf(_offset / size, 21, 15);
 567     i->srf(_base, 5);
 568   }
 569 
 570   void encode_nontemporal_pair(Instruction_aarch64 *i) const {
 571     // Only base + offset is allowed
 572     i->f(0b000, 25, 23);
 573     unsigned size = i->get(31, 31);
 574     size = 4 << size;
 575     guarantee(_offset % size == 0, "bad offset");
 576     i->sf(_offset / size, 21, 15);
 577     i->srf(_base, 5);
 578     guarantee(_mode == Address::base_plus_offset,
 579               "Bad addressing mode for non-temporal op");
 580   }
 581 
 582   void lea(MacroAssembler *, Register) const;
 583 
 584   static bool offset_ok_for_immed(int64_t offset, uint shift);
 585 
 586   static bool offset_ok_for_sve_immed(int64_t offset, int shift, int vl /* sve vector length */) {
 587     if (offset % vl == 0) {
 588       // Convert address offset into sve imm offset (MUL VL).
 589       int sve_offset = offset / vl;
 590       if (((-(1 << (shift - 1))) <= sve_offset) && (sve_offset < (1 << (shift - 1)))) {
 591         // sve_offset can be encoded
 592         return true;
 593       }
 594     }
 595     return false;
 596   }
 597 };
 598 
 599 // Convience classes
 600 class RuntimeAddress: public Address {
 601 
 602   public:
 603 
 604   RuntimeAddress(address target) : Address(target, relocInfo::runtime_call_type) {}
 605 
 606 };

2948     starti;                                                                            \
2949     assert(T == S || T == D, "invalid register variant");                              \
2950     f(0b01100101, 31, 24), f(T, 23, 22), f(0, 21),                                     \
2951     rf(Zm, 16), f(0, 15, 13), f(opcode, 12, 10), rf(Zn, 5), rf(Zd, 0);                 \
2952   }
2953 
2954   INSN(sve_fadd, 0b000);
2955   INSN(sve_fmul, 0b010);
2956   INSN(sve_fsub, 0b001);
2957 #undef INSN
2958 
2959 private:
2960   void sve_predicate_reg_insn(unsigned op24, unsigned op13,
2961                               FloatRegister Zd_or_Vd, SIMD_RegVariant T,
2962                               PRegister Pg, FloatRegister Zn_or_Vn) {
2963     starti;
2964     f(op24, 31, 24), f(T, 23, 22), f(op13, 21, 13);
2965     pgrf(Pg, 10), rf(Zn_or_Vn, 5), rf(Zd_or_Vd, 0);
2966   }
2967 
2968   void sve_shift_imm_encoding(SIMD_RegVariant T, int shift, bool isSHR,
2969                               int& tszh, int& tszl_imm) {
2970     /* The encodings for the tszh:tszl:imm3 fields
2971      * for shift right is calculated as:
2972      *   0001 xxx       B, shift = 16  - UInt(tszh:tszl:imm3)
2973      *   001x xxx       H, shift = 32  - UInt(tszh:tszl:imm3)
2974      *   01xx xxx       S, shift = 64  - UInt(tszh:tszl:imm3)
2975      *   1xxx xxx       D, shift = 128 - UInt(tszh:tszl:imm3)
2976      * for shift left is calculated as:
2977      *   0001 xxx       B, shift = UInt(tszh:tszl:imm3) - 8
2978      *   001x xxx       H, shift = UInt(tszh:tszl:imm3) - 16
2979      *   01xx xxx       S, shift = UInt(tszh:tszl:imm3) - 32
2980      *   1xxx xxx       D, shift = UInt(tszh:tszl:imm3) - 64
2981      */
2982     assert(T != Q, "Invalid register variant");
2983     if (isSHR) {
2984       assert(((1 << (T + 3)) >= shift) && (shift > 0) , "Invalid shift value");
2985     } else {
2986       assert(((1 << (T + 3)) > shift) && (shift >= 0) , "Invalid shift value");
2987     }
2988     int cVal = (1 << ((T + 3) + (isSHR ? 1 : 0)));
2989     int encodedShift = isSHR ? cVal - shift : cVal + shift;
2990     tszh = encodedShift >> 5;
2991     tszl_imm = encodedShift & 0x1f;
2992   }
2993 
2994 public:
2995 
2996 // SVE integer arithmetic - predicate
2997 #define INSN(NAME, op1, op2)                                                                            \
2998   void NAME(FloatRegister Zdn_or_Zd_or_Vd, SIMD_RegVariant T, PRegister Pg, FloatRegister Znm_or_Vn) {  \
2999     assert(T != Q, "invalid register variant");                                                         \
3000     sve_predicate_reg_insn(op1, op2, Zdn_or_Zd_or_Vd, T, Pg, Znm_or_Vn);                                \
3001   }
3002 
3003   INSN(sve_abs,  0b00000100, 0b010110101); // vector abs, unary
3004   INSN(sve_add,  0b00000100, 0b000000000); // vector add
3005   INSN(sve_and,  0b00000100, 0b011010000); // vector and
3006   INSN(sve_andv, 0b00000100, 0b011010001); // bitwise and reduction to scalar
3007   INSN(sve_asr,  0b00000100, 0b010000100); // vector arithmetic shift right
3008   INSN(sve_cnt,  0b00000100, 0b011010101); // count non-zero bits
3009   INSN(sve_cpy,  0b00000101, 0b100000100); // copy scalar to each active vector element
3010   INSN(sve_eor,  0b00000100, 0b011001000); // vector eor
3011   INSN(sve_eorv, 0b00000100, 0b011001001); // bitwise xor reduction to scalar
3012   INSN(sve_lsl,  0b00000100, 0b010011100); // vector logical shift left
3013   INSN(sve_lsr,  0b00000100, 0b010001100); // vector logical shift right
3014   INSN(sve_mul,  0b00000100, 0b010000000); // vector mul
3015   INSN(sve_neg,  0b00000100, 0b010111101); // vector neg, unary
3016   INSN(sve_not,  0b00000100, 0b011110101); // bitwise invert vector, unary
3017   INSN(sve_orr,  0b00000100, 0b011000000); // vector or
3018   INSN(sve_orv,  0b00000100, 0b011000001); // bitwise or reduction to scalar
3019   INSN(sve_smax, 0b00000100, 0b001000000); // signed maximum vectors
3020   INSN(sve_smaxv, 0b00000100, 0b001000001); // signed maximum reduction to scalar
3021   INSN(sve_smin,  0b00000100, 0b001010000); // signed minimum vectors
3022   INSN(sve_sminv, 0b00000100, 0b001010001); // signed minimum reduction to scalar
3023   INSN(sve_sub,   0b00000100, 0b000001000); // vector sub
3024   INSN(sve_uaddv, 0b00000100, 0b000001001); // unsigned add reduction to scalar
3025 #undef INSN
3026 
3027 // SVE floating-point arithmetic - predicate
3028 #define INSN(NAME, op1, op2)                                                                          \
3029   void NAME(FloatRegister Zd_or_Zdn_or_Vd, SIMD_RegVariant T, PRegister Pg, FloatRegister Zn_or_Zm) { \
3030     assert(T == S || T == D, "invalid register variant");                                             \
3031     sve_predicate_reg_insn(op1, op2, Zd_or_Zdn_or_Vd, T, Pg, Zn_or_Zm);                               \
3032   }
3033 
3034   INSN(sve_fabs,    0b00000100, 0b011100101);
3035   INSN(sve_fadd,    0b01100101, 0b000000100);
3036   INSN(sve_fadda,   0b01100101, 0b011000001); // add strictly-ordered reduction to scalar Vd
3037   INSN(sve_fdiv,    0b01100101, 0b001101100);

3040   INSN(sve_fmin,    0b01100101, 0b000111100); // floating-point minimum
3041   INSN(sve_fminv,   0b01100101, 0b000111001); // floating-point minimum recursive reduction to scalar
3042   INSN(sve_fmul,    0b01100101, 0b000010100);
3043   INSN(sve_fneg,    0b00000100, 0b011101101);
3044   INSN(sve_frintm,  0b01100101, 0b000010101); // floating-point round to integral value, toward minus infinity
3045   INSN(sve_frintn,  0b01100101, 0b000000101); // floating-point round to integral value, nearest with ties to even
3046   INSN(sve_frintp,  0b01100101, 0b000001101); // floating-point round to integral value, toward plus infinity
3047   INSN(sve_fsqrt,   0b01100101, 0b001101101);
3048   INSN(sve_fsub,    0b01100101, 0b000001100);
3049 #undef INSN
3050 
3051   // SVE multiple-add/sub - predicated
3052 #define INSN(NAME, op0, op1, op2)                                                                     \
3053   void NAME(FloatRegister Zda, SIMD_RegVariant T, PRegister Pg, FloatRegister Zn, FloatRegister Zm) { \
3054     starti;                                                                                           \
3055     assert(T != Q, "invalid size");                                                                   \
3056     f(op0, 31, 24), f(T, 23, 22), f(op1, 21), rf(Zm, 16);                                             \
3057     f(op2, 15, 13), pgrf(Pg, 10), rf(Zn, 5), rf(Zda, 0);                                              \
3058   }
3059 
3060   INSN(sve_fmla,  0b01100101, 1, 0b000); // floating-point fused multiply-add, writing addend: Zda = Zda + Zn * Zm
3061   INSN(sve_fmls,  0b01100101, 1, 0b001); // floating-point fused multiply-subtract: Zda = Zda + -Zn * Zm
3062   INSN(sve_fnmla, 0b01100101, 1, 0b010); // floating-point negated fused multiply-add: Zda = -Zda + -Zn * Zm
3063   INSN(sve_fnmls, 0b01100101, 1, 0b011); // floating-point negated fused multiply-subtract: Zda = -Zda + Zn * Zm
3064   INSN(sve_fmad,  0b01100101, 1, 0b100); // floating-point fused multiply-add, writing multiplicand: Zda = Zm + Zda * Zn
3065   INSN(sve_mla,   0b00000100, 0, 0b010); // multiply-add: Zda = Zda + Zn*Zm
3066   INSN(sve_mls,   0b00000100, 0, 0b011); // multiply-subtract: Zda = Zda + -Zn*Zm
3067 #undef INSN
3068 
3069 // SVE bitwise logical - unpredicated
3070 #define INSN(NAME, opc)                                              \
3071   void NAME(FloatRegister Zd, FloatRegister Zn, FloatRegister Zm) {  \
3072     starti;                                                          \
3073     f(0b00000100, 31, 24), f(opc, 23, 22), f(1, 21),                 \
3074     rf(Zm, 16), f(0b001100, 15, 10), rf(Zn, 5), rf(Zd, 0);           \
3075   }
3076   INSN(sve_and, 0b00);
3077   INSN(sve_eor, 0b10);
3078   INSN(sve_orr, 0b01);
3079   INSN(sve_bic, 0b11);
3080 #undef INSN
3081 
3082 // SVE shift immediate - unpredicated
3083 #define INSN(NAME, opc, isSHR)                                                  \
3084   void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, int shift) { \
3085     starti;                                                                     \
3086     int tszh, tszl_imm;                                                         \
3087     sve_shift_imm_encoding(T, shift, isSHR, tszh, tszl_imm);                    \




















3088     f(0b00000100, 31, 24);                                                      \
3089     f(tszh, 23, 22), f(1,21), f(tszl_imm, 20, 16);                              \
3090     f(0b100, 15, 13), f(opc, 12, 10), rf(Zn, 5), rf(Zd, 0);                     \
3091   }
3092 
3093   INSN(sve_asr, 0b100, /* isSHR = */ true);
3094   INSN(sve_lsl, 0b111, /* isSHR = */ false);
3095   INSN(sve_lsr, 0b101, /* isSHR = */ true);
3096 #undef INSN
3097 
3098 // SVE bitwise shift by immediate (predicated)
3099 #define INSN(NAME, opc, isSHR)                                                  \
3100   void NAME(FloatRegister Zdn, SIMD_RegVariant T, PRegister Pg, int shift) {    \
3101     starti;                                                                     \
3102     int tszh, tszl_imm;                                                         \
3103     sve_shift_imm_encoding(T, shift, isSHR, tszh, tszl_imm);                    \
3104     f(0b00000100, 31, 24), f(tszh, 23, 22), f(0b00, 21, 20), f(opc, 19, 16);    \
3105     f(0b100, 15, 13), pgrf(Pg, 10), f(tszl_imm, 9, 5), rf(Zdn, 0);              \
3106   }
3107 
3108   INSN(sve_asr, 0b0000, /* isSHR = */ true);
3109   INSN(sve_lsl, 0b0011, /* isSHR = */ false);
3110   INSN(sve_lsr, 0b0001, /* isSHR = */ true);
3111 #undef INSN
3112 
3113 private:
3114 
3115   // Scalar base + immediate index
3116   void sve_ld_st1(FloatRegister Zt, Register Xn, int imm, PRegister Pg,
3117               SIMD_RegVariant T, int op1, int type, int op2) {
3118     starti;
3119     assert_cond(T >= type);
3120     f(op1, 31, 25), f(type, 24, 23), f(T, 22, 21);
3121     f(0, 20), sf(imm, 19, 16), f(op2, 15, 13);
3122     pgrf(Pg, 10), srf(Xn, 5), rf(Zt, 0);
3123   }
3124 
3125   // Scalar base + scalar index
3126   void sve_ld_st1(FloatRegister Zt, Register Xn, Register Xm, PRegister Pg,
3127               SIMD_RegVariant T, int op1, int type, int op2) {
3128     starti;
3129     assert_cond(T >= type);
3130     f(op1, 31, 25), f(type, 24, 23), f(T, 22, 21);
3131     rf(Xm, 16), f(op2, 15, 13);
3132     pgrf(Pg, 10), srf(Xn, 5), rf(Zt, 0);

3204     srf(Xn, 16), f(0b01010, 15, 11), sf(imm6, 10, 5), srf(Xd, 0); \
3205   }
3206 
3207   INSN(sve_addvl, 0b01); // Add multiple of vector register size to scalar register
3208   INSN(sve_addpl, 0b11); // Add multiple of predicate register size to scalar register
3209 #undef INSN
3210 
3211 // SVE inc/dec register by element count
3212 #define INSN(NAME, op) \
3213   void NAME(Register Xdn, SIMD_RegVariant T, unsigned imm4 = 1, int pattern = 0b11111) { \
3214     starti;                                                                              \
3215     assert(T != Q, "invalid size");                                                      \
3216     f(0b00000100,31, 24), f(T, 23, 22), f(0b11, 21, 20);                                 \
3217     f(imm4 - 1, 19, 16), f(0b11100, 15, 11), f(op, 10), f(pattern, 9, 5), rf(Xdn, 0);    \
3218   }
3219 
3220   INSN(sve_inc, 0);
3221   INSN(sve_dec, 1);
3222 #undef INSN
3223 
3224 // SVE predicate logical operations
3225 #define INSN(NAME, op1, op2, op3) \
3226   void NAME(PRegister Pd, PRegister Pg, PRegister Pn, PRegister Pm) { \
3227     starti;                                                           \
3228     f(0b00100101, 31, 24), f(op1, 23, 22), f(0b00, 21, 20);           \
3229     prf(Pm, 16), f(0b01, 15, 14), prf(Pg, 10), f(op2, 9);             \
3230     prf(Pn, 5), f(op3, 4), prf(Pd, 0);                                \
3231   }
3232 
3233   INSN(sve_and,  0b00, 0b0, 0b0);
3234   INSN(sve_ands, 0b01, 0b0, 0b0);
3235   INSN(sve_eor,  0b00, 0b1, 0b0);
3236   INSN(sve_eors, 0b01, 0b1, 0b0);
3237   INSN(sve_orr,  0b10, 0b0, 0b0);
3238   INSN(sve_orrs, 0b11, 0b0, 0b0);
3239   INSN(sve_bic,  0b00, 0b0, 0b1);
3240 #undef INSN
3241 
3242   // SVE increment register by predicate count
3243   void sve_incp(const Register rd, SIMD_RegVariant T, PRegister pg) {
3244     starti;
3245     assert(T != Q, "invalid size");
3246     f(0b00100101, 31, 24), f(T, 23, 22), f(0b1011001000100, 21, 9),
3247     prf(pg, 5), rf(rd, 0);
3248   }
3249 
3250   // SVE broadcast general-purpose register to vector elements (unpredicated)
3251   void sve_dup(FloatRegister Zd, SIMD_RegVariant T, Register Rn) {
3252     starti;
3253     assert(T != Q, "invalid size");
3254     f(0b00000101, 31, 24), f(T, 23, 22), f(0b100000001110, 21, 10);
3255     srf(Rn, 5), rf(Zd, 0);
3256   }
3257 
3258   // SVE broadcast signed immediate to vector elements (unpredicated)
3259   void sve_dup(FloatRegister Zd, SIMD_RegVariant T, int imm8) {
3260     starti;
3261     assert(T != Q, "invalid size");
3262     int sh = 0;
3263     if (imm8 <= 127 && imm8 >= -128) {
3264       sh = 0;
3265     } else if (T != B && imm8 <= 32512 && imm8 >= -32768 && (imm8 & 0xff) == 0) {
3266       sh = 1;
3267       imm8 = (imm8 >> 8);
3268     } else {
3269       guarantee(false, "invalid immediate");
3270     }
3271     f(0b00100101, 31, 24), f(T, 23, 22), f(0b11100011, 21, 14);
3272     f(sh, 13), sf(imm8, 12, 5), rf(Zd, 0);
3273   }
3274 
3275   // SVE predicate test
3276   void sve_ptest(PRegister Pg, PRegister Pn) {
3277     starti;
3278     f(0b001001010101000011, 31, 14), prf(Pg, 10), f(0, 9), prf(Pn, 5), f(0, 4, 0);
3279   }
3280 
3281   // SVE predicate initialize
3282   void sve_ptrue(PRegister pd, SIMD_RegVariant esize, int pattern = 0b11111) {
3283     starti;
3284     f(0b00100101, 31, 24), f(esize, 23, 22), f(0b011000111000, 21, 10);
3285     f(pattern, 9, 5), f(0b0, 4), prf(pd, 0);
3286   }
3287 
3288   // SVE predicate zero
3289   void sve_pfalse(PRegister pd) {
3290     starti;
3291     f(0b00100101, 31, 24), f(0b00, 23, 22), f(0b011000111001, 21, 10);
3292     f(0b000000, 9, 4), prf(pd, 0);
3293   }
3294 
3295 // SVE load/store predicate register
3296 #define INSN(NAME, op1)                                                  \
3297   void NAME(PRegister Pt, const Address &a)  {                           \
3298     starti;                                                              \
3299     assert(a.index() == noreg, "invalid address variant");               \
3300     f(op1, 31, 29), f(0b0010110, 28, 22), sf(a.offset() >> 3, 21, 16),   \
3301     f(0b000, 15, 13), f(a.offset() & 0x7, 12, 10), srf(a.base(), 5),     \
3302     f(0, 4), prf(Pt, 0);                                                 \
3303   }
3304 
3305   INSN(sve_ldr, 0b100); // LDR (predicate)
3306   INSN(sve_str, 0b111); // STR (predicate)
3307 #undef INSN
3308 
3309   // SVE move predicate register
3310   void sve_mov(PRegister Pd, PRegister Pn) {
3311     starti;
3312     f(0b001001011000, 31, 20), prf(Pn, 16), f(0b01, 15, 14), prf(Pn, 10);
3313     f(0, 9), prf(Pn, 5), f(0, 4), prf(Pd, 0);
3314   }
3315 
3316   // SVE copy general-purpose register to vector elements (predicated)
3317   void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, Register Rn) {
3318     starti;
3319     assert(T != Q, "invalid size");
3320     f(0b00000101, 31, 24), f(T, 23, 22), f(0b101000101, 21, 13);
3321     pgrf(Pg, 10), srf(Rn, 5), rf(Zd, 0);
3322   }
3323 
3324   // SVE copy signed integer immediate to vector elements (predicated)
3325   void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, int imm8, bool isMerge) {
3326     starti;
3327     assert(T != Q, "invalid size");
3328     int sh = 0;
3329     if (imm8 <= 127 && imm8 >= -128) {
3330       sh = 0;
3331     } else if (T != B && imm8 <= 32512 && imm8 >= -32768 && (imm8 & 0xff) == 0) {
3332       sh = 1;
3333       imm8 = (imm8 >> 8);
3334     } else {
3335       guarantee(false, "invalid immediate");

3398   f(0b00100101, 31, 24), f(T, 23, 22), f(0b0, 21), sf(imm5, 20, 16),
3399   f((cond_op >> 1) & 0x7, 15, 13), pgrf(Pg, 10), rf(Zn, 5);
3400   f(cond_op & 0x1, 4), prf(Pd, 0);
3401 }
3402 
3403 // SVE unpack vector elements
3404 #define INSN(NAME, op) \
3405   void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn) { \
3406     starti;                                                          \
3407     assert(T != B && T != Q, "invalid size");                        \
3408     f(0b00000101, 31, 24), f(T, 23, 22), f(0b1100, 21, 18);          \
3409     f(op, 17, 16), f(0b001110, 15, 10), rf(Zn, 5), rf(Zd, 0);        \
3410   }
3411 
3412   INSN(sve_uunpkhi, 0b11); // Signed unpack and extend half of vector - high half
3413   INSN(sve_uunpklo, 0b10); // Signed unpack and extend half of vector - low half
3414   INSN(sve_sunpkhi, 0b01); // Unsigned unpack and extend half of vector - high half
3415   INSN(sve_sunpklo, 0b00); // Unsigned unpack and extend half of vector - low half
3416 #undef INSN
3417 
3418 // SVE unpack predicate elements
3419 #define INSN(NAME, op) \
3420   void NAME(PRegister Pd, PRegister Pn) { \
3421     starti;                                                          \
3422     f(0b000001010011000, 31, 17), f(op, 16), f(0b0100000, 15, 9);    \
3423     prf(Pn, 5), f(0b0, 4), prf(Pd, 0);                               \
3424   }
3425 
3426   INSN(sve_punpkhi, 0b1); // Unpack and widen high half of predicate
3427   INSN(sve_punpklo, 0b0); // Unpack and widen low half of predicate
3428 #undef INSN
3429 
3430 // SVE permute vector elements
3431 #define INSN(NAME, op) \
3432   void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, FloatRegister Zm) { \
3433     starti;                                                                            \
3434     assert(T != Q, "invalid size");                                                    \
3435     f(0b00000101, 31, 24), f(T, 23, 22), f(0b1, 21), rf(Zm, 16);                       \
3436     f(0b01101, 15, 11), f(op, 10), rf(Zn, 5), rf(Zd, 0);                               \
3437   }
3438 
3439   INSN(sve_uzp1, 0b0); // Concatenate even elements from two vectors
3440   INSN(sve_uzp2, 0b1); // Concatenate odd elements from two vectors
3441 #undef INSN
3442 
3443 // SVE permute predicate elements
3444 #define INSN(NAME, op) \
3445   void NAME(PRegister Pd, SIMD_RegVariant T, PRegister Pn, PRegister Pm) {             \
3446     starti;                                                                            \
3447     assert(T != Q, "invalid size");                                                    \
3448     f(0b00000101, 31, 24), f(T, 23, 22), f(0b10, 21, 20), prf(Pm, 16);                 \
3449     f(0b01001, 15, 11), f(op, 10), f(0b0, 9), prf(Pn, 5), f(0b0, 4), prf(Pd, 0);       \
3450   }
3451 
3452   INSN(sve_uzp1, 0b0); // Concatenate even elements from two predicates
3453   INSN(sve_uzp2, 0b1); // Concatenate odd elements from two predicates
3454 #undef INSN
3455 
3456 // Predicate counted loop (SVE) (32-bit variants are not included)
3457 #define INSN(NAME, decode)                                                \
3458   void NAME(PRegister Pd, SIMD_RegVariant T, Register Rn, Register Rm) {  \
3459     starti;                                                               \
3460     assert(T != Q, "invalid register variant");                           \
3461     f(0b00100101, 31, 24), f(T, 23, 22), f(1, 21),                        \
3462     zrf(Rm, 16), f(0, 15, 13), f(1, 12), f(decode >> 1, 11, 10),          \
3463     zrf(Rn, 5), f(decode & 1, 4), prf(Pd, 0);                             \
3464   }
3465 
3466   INSN(sve_whilelt, 0b010);  // While incrementing signed scalar less than scalar
3467   INSN(sve_whilele, 0b011);  // While incrementing signed scalar less than or equal to scalar
3468   INSN(sve_whilelo, 0b110);  // While incrementing unsigned scalar lower than scalar
3469   INSN(sve_whilels, 0b111);  // While incrementing unsigned scalar lower than or the same as scalar
3470 #undef INSN
3471 
3472   // SVE predicate reverse
3473   void sve_rev(PRegister Pd, SIMD_RegVariant T, PRegister Pn) {
3474     starti;
3475     assert(T != Q, "invalid size");
< prev index next >