566 i->sf(_offset / size, 21, 15);
567 i->srf(_base, 5);
568 }
569
570 void encode_nontemporal_pair(Instruction_aarch64 *i) const {
571 // Only base + offset is allowed
572 i->f(0b000, 25, 23);
573 unsigned size = i->get(31, 31);
574 size = 4 << size;
575 guarantee(_offset % size == 0, "bad offset");
576 i->sf(_offset / size, 21, 15);
577 i->srf(_base, 5);
578 guarantee(_mode == Address::base_plus_offset,
579 "Bad addressing mode for non-temporal op");
580 }
581
582 void lea(MacroAssembler *, Register) const;
583
584 static bool offset_ok_for_immed(int64_t offset, uint shift);
585
586 static bool offset_ok_for_sve_immed(long offset, int shift, int vl /* sve vector length */) {
587 if (offset % vl == 0) {
588 // Convert address offset into sve imm offset (MUL VL).
589 int sve_offset = offset / vl;
590 if (((-(1 << (shift - 1))) <= sve_offset) && (sve_offset < (1 << (shift - 1)))) {
591 // sve_offset can be encoded
592 return true;
593 }
594 }
595 return false;
596 }
597 };
598
599 // Convience classes
600 class RuntimeAddress: public Address {
601
602 public:
603
604 RuntimeAddress(address target) : Address(target, relocInfo::runtime_call_type) {}
605
606 };
2948 starti; \
2949 assert(T == S || T == D, "invalid register variant"); \
2950 f(0b01100101, 31, 24), f(T, 23, 22), f(0, 21), \
2951 rf(Zm, 16), f(0, 15, 13), f(opcode, 12, 10), rf(Zn, 5), rf(Zd, 0); \
2952 }
2953
2954 INSN(sve_fadd, 0b000);
2955 INSN(sve_fmul, 0b010);
2956 INSN(sve_fsub, 0b001);
2957 #undef INSN
2958
2959 private:
2960 void sve_predicate_reg_insn(unsigned op24, unsigned op13,
2961 FloatRegister Zd_or_Vd, SIMD_RegVariant T,
2962 PRegister Pg, FloatRegister Zn_or_Vn) {
2963 starti;
2964 f(op24, 31, 24), f(T, 23, 22), f(op13, 21, 13);
2965 pgrf(Pg, 10), rf(Zn_or_Vn, 5), rf(Zd_or_Vd, 0);
2966 }
2967
2968 public:
2969
2970 // SVE integer arithmetic - predicate
2971 #define INSN(NAME, op1, op2) \
2972 void NAME(FloatRegister Zdn_or_Zd_or_Vd, SIMD_RegVariant T, PRegister Pg, FloatRegister Znm_or_Vn) { \
2973 assert(T != Q, "invalid register variant"); \
2974 sve_predicate_reg_insn(op1, op2, Zdn_or_Zd_or_Vd, T, Pg, Znm_or_Vn); \
2975 }
2976
2977 INSN(sve_abs, 0b00000100, 0b010110101); // vector abs, unary
2978 INSN(sve_add, 0b00000100, 0b000000000); // vector add
2979 INSN(sve_andv, 0b00000100, 0b011010001); // bitwise and reduction to scalar
2980 INSN(sve_asr, 0b00000100, 0b010000100); // vector arithmetic shift right
2981 INSN(sve_cnt, 0b00000100, 0b011010101) // count non-zero bits
2982 INSN(sve_cpy, 0b00000101, 0b100000100); // copy scalar to each active vector element
2983 INSN(sve_eorv, 0b00000100, 0b011001001); // bitwise xor reduction to scalar
2984 INSN(sve_lsl, 0b00000100, 0b010011100); // vector logical shift left
2985 INSN(sve_lsr, 0b00000100, 0b010001100); // vector logical shift right
2986 INSN(sve_mul, 0b00000100, 0b010000000); // vector mul
2987 INSN(sve_neg, 0b00000100, 0b010111101); // vector neg, unary
2988 INSN(sve_not, 0b00000100, 0b011110101); // bitwise invert vector, unary
2989 INSN(sve_orv, 0b00000100, 0b011000001); // bitwise or reduction to scalar
2990 INSN(sve_smax, 0b00000100, 0b001000000); // signed maximum vectors
2991 INSN(sve_smaxv, 0b00000100, 0b001000001); // signed maximum reduction to scalar
2992 INSN(sve_smin, 0b00000100, 0b001010000); // signed minimum vectors
2993 INSN(sve_sminv, 0b00000100, 0b001010001); // signed minimum reduction to scalar
2994 INSN(sve_sub, 0b00000100, 0b000001000); // vector sub
2995 INSN(sve_uaddv, 0b00000100, 0b000001001); // unsigned add reduction to scalar
2996 #undef INSN
2997
2998 // SVE floating-point arithmetic - predicate
2999 #define INSN(NAME, op1, op2) \
3000 void NAME(FloatRegister Zd_or_Zdn_or_Vd, SIMD_RegVariant T, PRegister Pg, FloatRegister Zn_or_Zm) { \
3001 assert(T == S || T == D, "invalid register variant"); \
3002 sve_predicate_reg_insn(op1, op2, Zd_or_Zdn_or_Vd, T, Pg, Zn_or_Zm); \
3003 }
3004
3005 INSN(sve_fabs, 0b00000100, 0b011100101);
3006 INSN(sve_fadd, 0b01100101, 0b000000100);
3007 INSN(sve_fadda, 0b01100101, 0b011000001); // add strictly-ordered reduction to scalar Vd
3008 INSN(sve_fdiv, 0b01100101, 0b001101100);
3011 INSN(sve_fmin, 0b01100101, 0b000111100); // floating-point minimum
3012 INSN(sve_fminv, 0b01100101, 0b000111001); // floating-point minimum recursive reduction to scalar
3013 INSN(sve_fmul, 0b01100101, 0b000010100);
3014 INSN(sve_fneg, 0b00000100, 0b011101101);
3015 INSN(sve_frintm, 0b01100101, 0b000010101); // floating-point round to integral value, toward minus infinity
3016 INSN(sve_frintn, 0b01100101, 0b000000101); // floating-point round to integral value, nearest with ties to even
3017 INSN(sve_frintp, 0b01100101, 0b000001101); // floating-point round to integral value, toward plus infinity
3018 INSN(sve_fsqrt, 0b01100101, 0b001101101);
3019 INSN(sve_fsub, 0b01100101, 0b000001100);
3020 #undef INSN
3021
3022 // SVE multiple-add/sub - predicated
3023 #define INSN(NAME, op0, op1, op2) \
3024 void NAME(FloatRegister Zda, SIMD_RegVariant T, PRegister Pg, FloatRegister Zn, FloatRegister Zm) { \
3025 starti; \
3026 assert(T != Q, "invalid size"); \
3027 f(op0, 31, 24), f(T, 23, 22), f(op1, 21), rf(Zm, 16); \
3028 f(op2, 15, 13), pgrf(Pg, 10), rf(Zn, 5), rf(Zda, 0); \
3029 }
3030
3031 INSN(sve_fmla, 0b01100101, 1, 0b000); // floating-point fused multiply-add: Zda = Zda + Zn * Zm
3032 INSN(sve_fmls, 0b01100101, 1, 0b001); // floating-point fused multiply-subtract: Zda = Zda + -Zn * Zm
3033 INSN(sve_fnmla, 0b01100101, 1, 0b010); // floating-point negated fused multiply-add: Zda = -Zda + -Zn * Zm
3034 INSN(sve_fnmls, 0b01100101, 1, 0b011); // floating-point negated fused multiply-subtract: Zda = -Zda + Zn * Zm
3035 INSN(sve_mla, 0b00000100, 0, 0b010); // multiply-add: Zda = Zda + Zn*Zm
3036 INSN(sve_mls, 0b00000100, 0, 0b011); // multiply-subtract: Zda = Zda + -Zn*Zm
3037 #undef INSN
3038
3039 // SVE bitwise logical - unpredicated
3040 #define INSN(NAME, opc) \
3041 void NAME(FloatRegister Zd, FloatRegister Zn, FloatRegister Zm) { \
3042 starti; \
3043 f(0b00000100, 31, 24), f(opc, 23, 22), f(1, 21), \
3044 rf(Zm, 16), f(0b001100, 15, 10), rf(Zn, 5), rf(Zd, 0); \
3045 }
3046 INSN(sve_and, 0b00);
3047 INSN(sve_eor, 0b10);
3048 INSN(sve_orr, 0b01);
3049 INSN(sve_bic, 0b11);
3050 #undef INSN
3051
3052 // SVE shift immediate - unpredicated
3053 #define INSN(NAME, opc, isSHR) \
3054 void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, int shift) { \
3055 starti; \
3056 /* The encodings for the tszh:tszl:imm3 fields (bits 23:22 20:19 18:16) \
3057 * for shift right is calculated as: \
3058 * 0001 xxx B, shift = 16 - UInt(tszh:tszl:imm3) \
3059 * 001x xxx H, shift = 32 - UInt(tszh:tszl:imm3) \
3060 * 01xx xxx S, shift = 64 - UInt(tszh:tszl:imm3) \
3061 * 1xxx xxx D, shift = 128 - UInt(tszh:tszl:imm3) \
3062 * for shift left is calculated as: \
3063 * 0001 xxx B, shift = UInt(tszh:tszl:imm3) - 8 \
3064 * 001x xxx H, shift = UInt(tszh:tszl:imm3) - 16 \
3065 * 01xx xxx S, shift = UInt(tszh:tszl:imm3) - 32 \
3066 * 1xxx xxx D, shift = UInt(tszh:tszl:imm3) - 64 \
3067 */ \
3068 assert(T != Q, "Invalid register variant"); \
3069 if (isSHR) { \
3070 assert(((1 << (T + 3)) >= shift) && (shift > 0) , "Invalid shift value"); \
3071 } else { \
3072 assert(((1 << (T + 3)) > shift) && (shift >= 0) , "Invalid shift value"); \
3073 } \
3074 int cVal = (1 << ((T + 3) + (isSHR ? 1 : 0))); \
3075 int encodedShift = isSHR ? cVal - shift : cVal + shift; \
3076 int tszh = encodedShift >> 5; \
3077 int tszl_imm = encodedShift & 0x1f; \
3078 f(0b00000100, 31, 24); \
3079 f(tszh, 23, 22), f(1,21), f(tszl_imm, 20, 16); \
3080 f(0b100, 15, 13), f(opc, 12, 10), rf(Zn, 5), rf(Zd, 0); \
3081 }
3082
3083 INSN(sve_asr, 0b100, /* isSHR = */ true);
3084 INSN(sve_lsl, 0b111, /* isSHR = */ false);
3085 INSN(sve_lsr, 0b101, /* isSHR = */ true);
3086 #undef INSN
3087
3088 private:
3089
3090 // Scalar base + immediate index
3091 void sve_ld_st1(FloatRegister Zt, Register Xn, int imm, PRegister Pg,
3092 SIMD_RegVariant T, int op1, int type, int op2) {
3093 starti;
3094 assert_cond(T >= type);
3095 f(op1, 31, 25), f(type, 24, 23), f(T, 22, 21);
3096 f(0, 20), sf(imm, 19, 16), f(op2, 15, 13);
3097 pgrf(Pg, 10), srf(Xn, 5), rf(Zt, 0);
3098 }
3099
3100 // Scalar base + scalar index
3101 void sve_ld_st1(FloatRegister Zt, Register Xn, Register Xm, PRegister Pg,
3102 SIMD_RegVariant T, int op1, int type, int op2) {
3103 starti;
3104 assert_cond(T >= type);
3105 f(op1, 31, 25), f(type, 24, 23), f(T, 22, 21);
3106 rf(Xm, 16), f(op2, 15, 13);
3107 pgrf(Pg, 10), srf(Xn, 5), rf(Zt, 0);
3179 srf(Xn, 16), f(0b01010, 15, 11), sf(imm6, 10, 5), srf(Xd, 0); \
3180 }
3181
3182 INSN(sve_addvl, 0b01); // Add multiple of vector register size to scalar register
3183 INSN(sve_addpl, 0b11); // Add multiple of predicate register size to scalar register
3184 #undef INSN
3185
3186 // SVE inc/dec register by element count
3187 #define INSN(NAME, op) \
3188 void NAME(Register Xdn, SIMD_RegVariant T, unsigned imm4 = 1, int pattern = 0b11111) { \
3189 starti; \
3190 assert(T != Q, "invalid size"); \
3191 f(0b00000100,31, 24), f(T, 23, 22), f(0b11, 21, 20); \
3192 f(imm4 - 1, 19, 16), f(0b11100, 15, 11), f(op, 10), f(pattern, 9, 5), rf(Xdn, 0); \
3193 }
3194
3195 INSN(sve_inc, 0);
3196 INSN(sve_dec, 1);
3197 #undef INSN
3198
3199 // SVE increment register by predicate count
3200 void sve_incp(const Register rd, SIMD_RegVariant T, PRegister pg) {
3201 starti;
3202 assert(T != Q, "invalid size");
3203 f(0b00100101, 31, 24), f(T, 23, 22), f(0b1011001000100, 21, 9),
3204 prf(pg, 5), rf(rd, 0);
3205 }
3206
3207 // SVE broadcast general-purpose register to vector elements (unpredicated)
3208 void sve_dup(FloatRegister Zd, SIMD_RegVariant T, Register Rn) {
3209 starti;
3210 assert(T != Q, "invalid size");
3211 f(0b00000101, 31, 24), f(T, 23, 22), f(0b100000001110, 21, 10);
3212 srf(Rn, 5), rf(Zd, 0);
3213 }
3214
3215 // SVE broadcast signed immediate to vector elements (unpredicated)
3216 void sve_dup(FloatRegister Zd, SIMD_RegVariant T, int imm8) {
3217 starti;
3218 assert(T != Q, "invalid size");
3219 int sh = 0;
3220 if (imm8 <= 127 && imm8 >= -128) {
3221 sh = 0;
3222 } else if (T != B && imm8 <= 32512 && imm8 >= -32768 && (imm8 & 0xff) == 0) {
3223 sh = 1;
3224 imm8 = (imm8 >> 8);
3225 } else {
3226 guarantee(false, "invalid immediate");
3227 }
3228 f(0b00100101, 31, 24), f(T, 23, 22), f(0b11100011, 21, 14);
3229 f(sh, 13), sf(imm8, 12, 5), rf(Zd, 0);
3230 }
3231
3232 void sve_ptrue(PRegister pd, SIMD_RegVariant esize, int pattern = 0b11111) {
3233 starti;
3234 f(0b00100101, 31, 24), f(esize, 23, 22), f(0b011000111000, 21, 10);
3235 f(pattern, 9, 5), f(0b0, 4), prf(pd, 0);
3236 }
3237
3238 // SVE copy general-purpose register to vector elements (predicated)
3239 void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, Register Rn) {
3240 starti;
3241 assert(T != Q, "invalid size");
3242 f(0b00000101, 31, 24), f(T, 23, 22), f(0b101000101, 21, 13);
3243 pgrf(Pg, 10), srf(Rn, 5), rf(Zd, 0);
3244 }
3245
3246 // SVE copy signed integer immediate to vector elements (predicated)
3247 void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, int imm8, bool isMerge) {
3248 starti;
3249 assert(T != Q, "invalid size");
3250 int sh = 0;
3251 if (imm8 <= 127 && imm8 >= -128) {
3252 sh = 0;
3253 } else if (T != B && imm8 <= 32512 && imm8 >= -32768 && (imm8 & 0xff) == 0) {
3254 sh = 1;
3255 imm8 = (imm8 >> 8);
3256 } else {
3257 guarantee(false, "invalid immediate");
3320 f(0b00100101, 31, 24), f(T, 23, 22), f(0b0, 21), sf(imm5, 20, 16),
3321 f((cond_op >> 1) & 0x7, 15, 13), pgrf(Pg, 10), rf(Zn, 5);
3322 f(cond_op & 0x1, 4), prf(Pd, 0);
3323 }
3324
3325 // SVE unpack vector elements
3326 #define INSN(NAME, op) \
3327 void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn) { \
3328 starti; \
3329 assert(T != B && T != Q, "invalid size"); \
3330 f(0b00000101, 31, 24), f(T, 23, 22), f(0b1100, 21, 18); \
3331 f(op, 17, 16), f(0b001110, 15, 10), rf(Zn, 5), rf(Zd, 0); \
3332 }
3333
3334 INSN(sve_uunpkhi, 0b11); // Signed unpack and extend half of vector - high half
3335 INSN(sve_uunpklo, 0b10); // Signed unpack and extend half of vector - low half
3336 INSN(sve_sunpkhi, 0b01); // Unsigned unpack and extend half of vector - high half
3337 INSN(sve_sunpklo, 0b00); // Unsigned unpack and extend half of vector - low half
3338 #undef INSN
3339
3340 // SVE permute vector elements
3341 #define INSN(NAME, op) \
3342 void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, FloatRegister Zm) { \
3343 starti; \
3344 assert(T != Q, "invalid size"); \
3345 f(0b00000101, 31, 24), f(T, 23, 22), f(0b1, 21), rf(Zm, 16); \
3346 f(0b01101, 15, 11), f(op, 10), rf(Zn, 5), rf(Zd, 0); \
3347 }
3348
3349 INSN(sve_uzp1, 0b0); // Concatenate even elements from two vectors
3350 INSN(sve_uzp2, 0b1); // Concatenate odd elements from two vectors
3351 #undef INSN
3352
3353 // Predicate counted loop (SVE) (32-bit variants are not included)
3354 #define INSN(NAME, decode) \
3355 void NAME(PRegister Pd, SIMD_RegVariant T, Register Rn, Register Rm) { \
3356 starti; \
3357 assert(T != Q, "invalid register variant"); \
3358 f(0b00100101, 31, 24), f(T, 23, 22), f(1, 21), \
3359 zrf(Rm, 16), f(0, 15, 13), f(1, 12), f(decode >> 1, 11, 10), \
3360 zrf(Rn, 5), f(decode & 1, 4), prf(Pd, 0); \
3361 }
3362
3363 INSN(sve_whilelt, 0b010); // While incrementing signed scalar less than scalar
3364 INSN(sve_whilele, 0b011); // While incrementing signed scalar less than or equal to scalar
3365 INSN(sve_whilelo, 0b110); // While incrementing unsigned scalar lower than scalar
3366 INSN(sve_whilels, 0b111); // While incrementing unsigned scalar lower than or the same as scalar
3367 #undef INSN
3368
3369 // SVE predicate reverse
3370 void sve_rev(PRegister Pd, SIMD_RegVariant T, PRegister Pn) {
3371 starti;
3372 assert(T != Q, "invalid size");
|
566 i->sf(_offset / size, 21, 15);
567 i->srf(_base, 5);
568 }
569
570 void encode_nontemporal_pair(Instruction_aarch64 *i) const {
571 // Only base + offset is allowed
572 i->f(0b000, 25, 23);
573 unsigned size = i->get(31, 31);
574 size = 4 << size;
575 guarantee(_offset % size == 0, "bad offset");
576 i->sf(_offset / size, 21, 15);
577 i->srf(_base, 5);
578 guarantee(_mode == Address::base_plus_offset,
579 "Bad addressing mode for non-temporal op");
580 }
581
582 void lea(MacroAssembler *, Register) const;
583
584 static bool offset_ok_for_immed(int64_t offset, uint shift);
585
586 static bool offset_ok_for_sve_immed(int64_t offset, int shift, int vl /* sve vector length */) {
587 if (offset % vl == 0) {
588 // Convert address offset into sve imm offset (MUL VL).
589 int sve_offset = offset / vl;
590 if (((-(1 << (shift - 1))) <= sve_offset) && (sve_offset < (1 << (shift - 1)))) {
591 // sve_offset can be encoded
592 return true;
593 }
594 }
595 return false;
596 }
597 };
598
599 // Convience classes
600 class RuntimeAddress: public Address {
601
602 public:
603
604 RuntimeAddress(address target) : Address(target, relocInfo::runtime_call_type) {}
605
606 };
2948 starti; \
2949 assert(T == S || T == D, "invalid register variant"); \
2950 f(0b01100101, 31, 24), f(T, 23, 22), f(0, 21), \
2951 rf(Zm, 16), f(0, 15, 13), f(opcode, 12, 10), rf(Zn, 5), rf(Zd, 0); \
2952 }
2953
2954 INSN(sve_fadd, 0b000);
2955 INSN(sve_fmul, 0b010);
2956 INSN(sve_fsub, 0b001);
2957 #undef INSN
2958
2959 private:
2960 void sve_predicate_reg_insn(unsigned op24, unsigned op13,
2961 FloatRegister Zd_or_Vd, SIMD_RegVariant T,
2962 PRegister Pg, FloatRegister Zn_or_Vn) {
2963 starti;
2964 f(op24, 31, 24), f(T, 23, 22), f(op13, 21, 13);
2965 pgrf(Pg, 10), rf(Zn_or_Vn, 5), rf(Zd_or_Vd, 0);
2966 }
2967
2968 void sve_shift_imm_encoding(SIMD_RegVariant T, int shift, bool isSHR,
2969 int& tszh, int& tszl_imm) {
2970 /* The encodings for the tszh:tszl:imm3 fields
2971 * for shift right is calculated as:
2972 * 0001 xxx B, shift = 16 - UInt(tszh:tszl:imm3)
2973 * 001x xxx H, shift = 32 - UInt(tszh:tszl:imm3)
2974 * 01xx xxx S, shift = 64 - UInt(tszh:tszl:imm3)
2975 * 1xxx xxx D, shift = 128 - UInt(tszh:tszl:imm3)
2976 * for shift left is calculated as:
2977 * 0001 xxx B, shift = UInt(tszh:tszl:imm3) - 8
2978 * 001x xxx H, shift = UInt(tszh:tszl:imm3) - 16
2979 * 01xx xxx S, shift = UInt(tszh:tszl:imm3) - 32
2980 * 1xxx xxx D, shift = UInt(tszh:tszl:imm3) - 64
2981 */
2982 assert(T != Q, "Invalid register variant");
2983 if (isSHR) {
2984 assert(((1 << (T + 3)) >= shift) && (shift > 0) , "Invalid shift value");
2985 } else {
2986 assert(((1 << (T + 3)) > shift) && (shift >= 0) , "Invalid shift value");
2987 }
2988 int cVal = (1 << ((T + 3) + (isSHR ? 1 : 0)));
2989 int encodedShift = isSHR ? cVal - shift : cVal + shift;
2990 tszh = encodedShift >> 5;
2991 tszl_imm = encodedShift & 0x1f;
2992 }
2993
2994 public:
2995
2996 // SVE integer arithmetic - predicate
2997 #define INSN(NAME, op1, op2) \
2998 void NAME(FloatRegister Zdn_or_Zd_or_Vd, SIMD_RegVariant T, PRegister Pg, FloatRegister Znm_or_Vn) { \
2999 assert(T != Q, "invalid register variant"); \
3000 sve_predicate_reg_insn(op1, op2, Zdn_or_Zd_or_Vd, T, Pg, Znm_or_Vn); \
3001 }
3002
3003 INSN(sve_abs, 0b00000100, 0b010110101); // vector abs, unary
3004 INSN(sve_add, 0b00000100, 0b000000000); // vector add
3005 INSN(sve_and, 0b00000100, 0b011010000); // vector and
3006 INSN(sve_andv, 0b00000100, 0b011010001); // bitwise and reduction to scalar
3007 INSN(sve_asr, 0b00000100, 0b010000100); // vector arithmetic shift right
3008 INSN(sve_cnt, 0b00000100, 0b011010101); // count non-zero bits
3009 INSN(sve_cpy, 0b00000101, 0b100000100); // copy scalar to each active vector element
3010 INSN(sve_eor, 0b00000100, 0b011001000); // vector eor
3011 INSN(sve_eorv, 0b00000100, 0b011001001); // bitwise xor reduction to scalar
3012 INSN(sve_lsl, 0b00000100, 0b010011100); // vector logical shift left
3013 INSN(sve_lsr, 0b00000100, 0b010001100); // vector logical shift right
3014 INSN(sve_mul, 0b00000100, 0b010000000); // vector mul
3015 INSN(sve_neg, 0b00000100, 0b010111101); // vector neg, unary
3016 INSN(sve_not, 0b00000100, 0b011110101); // bitwise invert vector, unary
3017 INSN(sve_orr, 0b00000100, 0b011000000); // vector or
3018 INSN(sve_orv, 0b00000100, 0b011000001); // bitwise or reduction to scalar
3019 INSN(sve_smax, 0b00000100, 0b001000000); // signed maximum vectors
3020 INSN(sve_smaxv, 0b00000100, 0b001000001); // signed maximum reduction to scalar
3021 INSN(sve_smin, 0b00000100, 0b001010000); // signed minimum vectors
3022 INSN(sve_sminv, 0b00000100, 0b001010001); // signed minimum reduction to scalar
3023 INSN(sve_sub, 0b00000100, 0b000001000); // vector sub
3024 INSN(sve_uaddv, 0b00000100, 0b000001001); // unsigned add reduction to scalar
3025 #undef INSN
3026
3027 // SVE floating-point arithmetic - predicate
3028 #define INSN(NAME, op1, op2) \
3029 void NAME(FloatRegister Zd_or_Zdn_or_Vd, SIMD_RegVariant T, PRegister Pg, FloatRegister Zn_or_Zm) { \
3030 assert(T == S || T == D, "invalid register variant"); \
3031 sve_predicate_reg_insn(op1, op2, Zd_or_Zdn_or_Vd, T, Pg, Zn_or_Zm); \
3032 }
3033
3034 INSN(sve_fabs, 0b00000100, 0b011100101);
3035 INSN(sve_fadd, 0b01100101, 0b000000100);
3036 INSN(sve_fadda, 0b01100101, 0b011000001); // add strictly-ordered reduction to scalar Vd
3037 INSN(sve_fdiv, 0b01100101, 0b001101100);
3040 INSN(sve_fmin, 0b01100101, 0b000111100); // floating-point minimum
3041 INSN(sve_fminv, 0b01100101, 0b000111001); // floating-point minimum recursive reduction to scalar
3042 INSN(sve_fmul, 0b01100101, 0b000010100);
3043 INSN(sve_fneg, 0b00000100, 0b011101101);
3044 INSN(sve_frintm, 0b01100101, 0b000010101); // floating-point round to integral value, toward minus infinity
3045 INSN(sve_frintn, 0b01100101, 0b000000101); // floating-point round to integral value, nearest with ties to even
3046 INSN(sve_frintp, 0b01100101, 0b000001101); // floating-point round to integral value, toward plus infinity
3047 INSN(sve_fsqrt, 0b01100101, 0b001101101);
3048 INSN(sve_fsub, 0b01100101, 0b000001100);
3049 #undef INSN
3050
3051 // SVE multiple-add/sub - predicated
3052 #define INSN(NAME, op0, op1, op2) \
3053 void NAME(FloatRegister Zda, SIMD_RegVariant T, PRegister Pg, FloatRegister Zn, FloatRegister Zm) { \
3054 starti; \
3055 assert(T != Q, "invalid size"); \
3056 f(op0, 31, 24), f(T, 23, 22), f(op1, 21), rf(Zm, 16); \
3057 f(op2, 15, 13), pgrf(Pg, 10), rf(Zn, 5), rf(Zda, 0); \
3058 }
3059
3060 INSN(sve_fmla, 0b01100101, 1, 0b000); // floating-point fused multiply-add, writing addend: Zda = Zda + Zn * Zm
3061 INSN(sve_fmls, 0b01100101, 1, 0b001); // floating-point fused multiply-subtract: Zda = Zda + -Zn * Zm
3062 INSN(sve_fnmla, 0b01100101, 1, 0b010); // floating-point negated fused multiply-add: Zda = -Zda + -Zn * Zm
3063 INSN(sve_fnmls, 0b01100101, 1, 0b011); // floating-point negated fused multiply-subtract: Zda = -Zda + Zn * Zm
3064 INSN(sve_fmad, 0b01100101, 1, 0b100); // floating-point fused multiply-add, writing multiplicand: Zda = Zm + Zda * Zn
3065 INSN(sve_mla, 0b00000100, 0, 0b010); // multiply-add: Zda = Zda + Zn*Zm
3066 INSN(sve_mls, 0b00000100, 0, 0b011); // multiply-subtract: Zda = Zda + -Zn*Zm
3067 #undef INSN
3068
3069 // SVE bitwise logical - unpredicated
3070 #define INSN(NAME, opc) \
3071 void NAME(FloatRegister Zd, FloatRegister Zn, FloatRegister Zm) { \
3072 starti; \
3073 f(0b00000100, 31, 24), f(opc, 23, 22), f(1, 21), \
3074 rf(Zm, 16), f(0b001100, 15, 10), rf(Zn, 5), rf(Zd, 0); \
3075 }
3076 INSN(sve_and, 0b00);
3077 INSN(sve_eor, 0b10);
3078 INSN(sve_orr, 0b01);
3079 INSN(sve_bic, 0b11);
3080 #undef INSN
3081
3082 // SVE shift immediate - unpredicated
3083 #define INSN(NAME, opc, isSHR) \
3084 void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, int shift) { \
3085 starti; \
3086 int tszh, tszl_imm; \
3087 sve_shift_imm_encoding(T, shift, isSHR, tszh, tszl_imm); \
3088 f(0b00000100, 31, 24); \
3089 f(tszh, 23, 22), f(1,21), f(tszl_imm, 20, 16); \
3090 f(0b100, 15, 13), f(opc, 12, 10), rf(Zn, 5), rf(Zd, 0); \
3091 }
3092
3093 INSN(sve_asr, 0b100, /* isSHR = */ true);
3094 INSN(sve_lsl, 0b111, /* isSHR = */ false);
3095 INSN(sve_lsr, 0b101, /* isSHR = */ true);
3096 #undef INSN
3097
3098 // SVE bitwise shift by immediate (predicated)
3099 #define INSN(NAME, opc, isSHR) \
3100 void NAME(FloatRegister Zdn, SIMD_RegVariant T, PRegister Pg, int shift) { \
3101 starti; \
3102 int tszh, tszl_imm; \
3103 sve_shift_imm_encoding(T, shift, isSHR, tszh, tszl_imm); \
3104 f(0b00000100, 31, 24), f(tszh, 23, 22), f(0b00, 21, 20), f(opc, 19, 16); \
3105 f(0b100, 15, 13), pgrf(Pg, 10), f(tszl_imm, 9, 5), rf(Zdn, 0); \
3106 }
3107
3108 INSN(sve_asr, 0b0000, /* isSHR = */ true);
3109 INSN(sve_lsl, 0b0011, /* isSHR = */ false);
3110 INSN(sve_lsr, 0b0001, /* isSHR = */ true);
3111 #undef INSN
3112
3113 private:
3114
3115 // Scalar base + immediate index
3116 void sve_ld_st1(FloatRegister Zt, Register Xn, int imm, PRegister Pg,
3117 SIMD_RegVariant T, int op1, int type, int op2) {
3118 starti;
3119 assert_cond(T >= type);
3120 f(op1, 31, 25), f(type, 24, 23), f(T, 22, 21);
3121 f(0, 20), sf(imm, 19, 16), f(op2, 15, 13);
3122 pgrf(Pg, 10), srf(Xn, 5), rf(Zt, 0);
3123 }
3124
3125 // Scalar base + scalar index
3126 void sve_ld_st1(FloatRegister Zt, Register Xn, Register Xm, PRegister Pg,
3127 SIMD_RegVariant T, int op1, int type, int op2) {
3128 starti;
3129 assert_cond(T >= type);
3130 f(op1, 31, 25), f(type, 24, 23), f(T, 22, 21);
3131 rf(Xm, 16), f(op2, 15, 13);
3132 pgrf(Pg, 10), srf(Xn, 5), rf(Zt, 0);
3204 srf(Xn, 16), f(0b01010, 15, 11), sf(imm6, 10, 5), srf(Xd, 0); \
3205 }
3206
3207 INSN(sve_addvl, 0b01); // Add multiple of vector register size to scalar register
3208 INSN(sve_addpl, 0b11); // Add multiple of predicate register size to scalar register
3209 #undef INSN
3210
3211 // SVE inc/dec register by element count
3212 #define INSN(NAME, op) \
3213 void NAME(Register Xdn, SIMD_RegVariant T, unsigned imm4 = 1, int pattern = 0b11111) { \
3214 starti; \
3215 assert(T != Q, "invalid size"); \
3216 f(0b00000100,31, 24), f(T, 23, 22), f(0b11, 21, 20); \
3217 f(imm4 - 1, 19, 16), f(0b11100, 15, 11), f(op, 10), f(pattern, 9, 5), rf(Xdn, 0); \
3218 }
3219
3220 INSN(sve_inc, 0);
3221 INSN(sve_dec, 1);
3222 #undef INSN
3223
3224 // SVE predicate logical operations
3225 #define INSN(NAME, op1, op2, op3) \
3226 void NAME(PRegister Pd, PRegister Pg, PRegister Pn, PRegister Pm) { \
3227 starti; \
3228 f(0b00100101, 31, 24), f(op1, 23, 22), f(0b00, 21, 20); \
3229 prf(Pm, 16), f(0b01, 15, 14), prf(Pg, 10), f(op2, 9); \
3230 prf(Pn, 5), f(op3, 4), prf(Pd, 0); \
3231 }
3232
3233 INSN(sve_and, 0b00, 0b0, 0b0);
3234 INSN(sve_ands, 0b01, 0b0, 0b0);
3235 INSN(sve_eor, 0b00, 0b1, 0b0);
3236 INSN(sve_eors, 0b01, 0b1, 0b0);
3237 INSN(sve_orr, 0b10, 0b0, 0b0);
3238 INSN(sve_orrs, 0b11, 0b0, 0b0);
3239 INSN(sve_bic, 0b00, 0b0, 0b1);
3240 #undef INSN
3241
3242 // SVE increment register by predicate count
3243 void sve_incp(const Register rd, SIMD_RegVariant T, PRegister pg) {
3244 starti;
3245 assert(T != Q, "invalid size");
3246 f(0b00100101, 31, 24), f(T, 23, 22), f(0b1011001000100, 21, 9),
3247 prf(pg, 5), rf(rd, 0);
3248 }
3249
3250 // SVE broadcast general-purpose register to vector elements (unpredicated)
3251 void sve_dup(FloatRegister Zd, SIMD_RegVariant T, Register Rn) {
3252 starti;
3253 assert(T != Q, "invalid size");
3254 f(0b00000101, 31, 24), f(T, 23, 22), f(0b100000001110, 21, 10);
3255 srf(Rn, 5), rf(Zd, 0);
3256 }
3257
3258 // SVE broadcast signed immediate to vector elements (unpredicated)
3259 void sve_dup(FloatRegister Zd, SIMD_RegVariant T, int imm8) {
3260 starti;
3261 assert(T != Q, "invalid size");
3262 int sh = 0;
3263 if (imm8 <= 127 && imm8 >= -128) {
3264 sh = 0;
3265 } else if (T != B && imm8 <= 32512 && imm8 >= -32768 && (imm8 & 0xff) == 0) {
3266 sh = 1;
3267 imm8 = (imm8 >> 8);
3268 } else {
3269 guarantee(false, "invalid immediate");
3270 }
3271 f(0b00100101, 31, 24), f(T, 23, 22), f(0b11100011, 21, 14);
3272 f(sh, 13), sf(imm8, 12, 5), rf(Zd, 0);
3273 }
3274
3275 // SVE predicate test
3276 void sve_ptest(PRegister Pg, PRegister Pn) {
3277 starti;
3278 f(0b001001010101000011, 31, 14), prf(Pg, 10), f(0, 9), prf(Pn, 5), f(0, 4, 0);
3279 }
3280
3281 // SVE predicate initialize
3282 void sve_ptrue(PRegister pd, SIMD_RegVariant esize, int pattern = 0b11111) {
3283 starti;
3284 f(0b00100101, 31, 24), f(esize, 23, 22), f(0b011000111000, 21, 10);
3285 f(pattern, 9, 5), f(0b0, 4), prf(pd, 0);
3286 }
3287
3288 // SVE predicate zero
3289 void sve_pfalse(PRegister pd) {
3290 starti;
3291 f(0b00100101, 31, 24), f(0b00, 23, 22), f(0b011000111001, 21, 10);
3292 f(0b000000, 9, 4), prf(pd, 0);
3293 }
3294
3295 // SVE load/store predicate register
3296 #define INSN(NAME, op1) \
3297 void NAME(PRegister Pt, const Address &a) { \
3298 starti; \
3299 assert(a.index() == noreg, "invalid address variant"); \
3300 f(op1, 31, 29), f(0b0010110, 28, 22), sf(a.offset() >> 3, 21, 16), \
3301 f(0b000, 15, 13), f(a.offset() & 0x7, 12, 10), srf(a.base(), 5), \
3302 f(0, 4), prf(Pt, 0); \
3303 }
3304
3305 INSN(sve_ldr, 0b100); // LDR (predicate)
3306 INSN(sve_str, 0b111); // STR (predicate)
3307 #undef INSN
3308
3309 // SVE move predicate register
3310 void sve_mov(PRegister Pd, PRegister Pn) {
3311 starti;
3312 f(0b001001011000, 31, 20), prf(Pn, 16), f(0b01, 15, 14), prf(Pn, 10);
3313 f(0, 9), prf(Pn, 5), f(0, 4), prf(Pd, 0);
3314 }
3315
3316 // SVE copy general-purpose register to vector elements (predicated)
3317 void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, Register Rn) {
3318 starti;
3319 assert(T != Q, "invalid size");
3320 f(0b00000101, 31, 24), f(T, 23, 22), f(0b101000101, 21, 13);
3321 pgrf(Pg, 10), srf(Rn, 5), rf(Zd, 0);
3322 }
3323
3324 // SVE copy signed integer immediate to vector elements (predicated)
3325 void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, int imm8, bool isMerge) {
3326 starti;
3327 assert(T != Q, "invalid size");
3328 int sh = 0;
3329 if (imm8 <= 127 && imm8 >= -128) {
3330 sh = 0;
3331 } else if (T != B && imm8 <= 32512 && imm8 >= -32768 && (imm8 & 0xff) == 0) {
3332 sh = 1;
3333 imm8 = (imm8 >> 8);
3334 } else {
3335 guarantee(false, "invalid immediate");
3398 f(0b00100101, 31, 24), f(T, 23, 22), f(0b0, 21), sf(imm5, 20, 16),
3399 f((cond_op >> 1) & 0x7, 15, 13), pgrf(Pg, 10), rf(Zn, 5);
3400 f(cond_op & 0x1, 4), prf(Pd, 0);
3401 }
3402
3403 // SVE unpack vector elements
3404 #define INSN(NAME, op) \
3405 void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn) { \
3406 starti; \
3407 assert(T != B && T != Q, "invalid size"); \
3408 f(0b00000101, 31, 24), f(T, 23, 22), f(0b1100, 21, 18); \
3409 f(op, 17, 16), f(0b001110, 15, 10), rf(Zn, 5), rf(Zd, 0); \
3410 }
3411
3412 INSN(sve_uunpkhi, 0b11); // Signed unpack and extend half of vector - high half
3413 INSN(sve_uunpklo, 0b10); // Signed unpack and extend half of vector - low half
3414 INSN(sve_sunpkhi, 0b01); // Unsigned unpack and extend half of vector - high half
3415 INSN(sve_sunpklo, 0b00); // Unsigned unpack and extend half of vector - low half
3416 #undef INSN
3417
3418 // SVE unpack predicate elements
3419 #define INSN(NAME, op) \
3420 void NAME(PRegister Pd, PRegister Pn) { \
3421 starti; \
3422 f(0b000001010011000, 31, 17), f(op, 16), f(0b0100000, 15, 9); \
3423 prf(Pn, 5), f(0b0, 4), prf(Pd, 0); \
3424 }
3425
3426 INSN(sve_punpkhi, 0b1); // Unpack and widen high half of predicate
3427 INSN(sve_punpklo, 0b0); // Unpack and widen low half of predicate
3428 #undef INSN
3429
3430 // SVE permute vector elements
3431 #define INSN(NAME, op) \
3432 void NAME(FloatRegister Zd, SIMD_RegVariant T, FloatRegister Zn, FloatRegister Zm) { \
3433 starti; \
3434 assert(T != Q, "invalid size"); \
3435 f(0b00000101, 31, 24), f(T, 23, 22), f(0b1, 21), rf(Zm, 16); \
3436 f(0b01101, 15, 11), f(op, 10), rf(Zn, 5), rf(Zd, 0); \
3437 }
3438
3439 INSN(sve_uzp1, 0b0); // Concatenate even elements from two vectors
3440 INSN(sve_uzp2, 0b1); // Concatenate odd elements from two vectors
3441 #undef INSN
3442
3443 // SVE permute predicate elements
3444 #define INSN(NAME, op) \
3445 void NAME(PRegister Pd, SIMD_RegVariant T, PRegister Pn, PRegister Pm) { \
3446 starti; \
3447 assert(T != Q, "invalid size"); \
3448 f(0b00000101, 31, 24), f(T, 23, 22), f(0b10, 21, 20), prf(Pm, 16); \
3449 f(0b01001, 15, 11), f(op, 10), f(0b0, 9), prf(Pn, 5), f(0b0, 4), prf(Pd, 0); \
3450 }
3451
3452 INSN(sve_uzp1, 0b0); // Concatenate even elements from two predicates
3453 INSN(sve_uzp2, 0b1); // Concatenate odd elements from two predicates
3454 #undef INSN
3455
3456 // Predicate counted loop (SVE) (32-bit variants are not included)
3457 #define INSN(NAME, decode) \
3458 void NAME(PRegister Pd, SIMD_RegVariant T, Register Rn, Register Rm) { \
3459 starti; \
3460 assert(T != Q, "invalid register variant"); \
3461 f(0b00100101, 31, 24), f(T, 23, 22), f(1, 21), \
3462 zrf(Rm, 16), f(0, 15, 13), f(1, 12), f(decode >> 1, 11, 10), \
3463 zrf(Rn, 5), f(decode & 1, 4), prf(Pd, 0); \
3464 }
3465
3466 INSN(sve_whilelt, 0b010); // While incrementing signed scalar less than scalar
3467 INSN(sve_whilele, 0b011); // While incrementing signed scalar less than or equal to scalar
3468 INSN(sve_whilelo, 0b110); // While incrementing unsigned scalar lower than scalar
3469 INSN(sve_whilels, 0b111); // While incrementing unsigned scalar lower than or the same as scalar
3470 #undef INSN
3471
3472 // SVE predicate reverse
3473 void sve_rev(PRegister Pd, SIMD_RegVariant T, PRegister Pn) {
3474 starti;
3475 assert(T != Q, "invalid size");
|