< prev index next >

src/hotspot/cpu/aarch64/aarch64_sve_ad.m4

Print this page

 127   bool op_sve_supported(int opcode, int vlen, BasicType bt) {
 128     int length_in_bytes = vlen * type2aelembytes(bt);
 129     switch (opcode) {
 130       case Op_MulAddVS2VI:
 131       // No multiply reduction instructions
 132       case Op_MulReductionVD:
 133       case Op_MulReductionVF:
 134       case Op_MulReductionVI:
 135       case Op_MulReductionVL:
 136       // Others
 137       case Op_ExtractC:
 138       case Op_ExtractUB:
 139         return false;
 140       // Vector API specific
 141       case Op_VectorLoadShuffle:
 142       case Op_VectorRearrange:
 143         return vlen >= 4 && length_in_bytes <= MaxVectorSize;
 144       case Op_LoadVector:
 145       case Op_StoreVector:
 146         return Matcher::vector_size_supported(bt, vlen);


 147       default:
 148         break;
 149     }
 150     // By default, we only support vector operations with no less than 8 bytes and 2 elements.
 151     return 8 <= length_in_bytes && length_in_bytes <= MaxVectorSize && vlen >= 2;
 152   }
 153 
 154   bool masked_op_sve_supported(int opcode, int vlen, BasicType bt) {
 155     if (opcode == Op_VectorRearrange) {
 156       return false;
 157     }
 158     return op_sve_supported(opcode, vlen, bt);
 159   }
 160 %}
 161 
 162 definitions %{
 163   int_def SVE_COST             (200, 200);
 164 %}
 165 
 166 dnl

1153 // vector mul - predicated
1154 BINARY_OP_PREDICATE(vmulB, MulVB, B, sve_mul)
1155 BINARY_OP_PREDICATE(vmulS, MulVS, H, sve_mul)
1156 BINARY_OP_PREDICATE(vmulI, MulVI, S, sve_mul)
1157 BINARY_OP_PREDICATE(vmulL, MulVL, D, sve_mul)
1158 BINARY_OP_PREDICATE(vmulF, MulVF, S, sve_fmul)
1159 BINARY_OP_PREDICATE(vmulD, MulVD, D, sve_fmul)
1160 
1161 // vector neg
1162 UNARY_OP_TRUE_PREDICATE(vnegI, NegVI, B/H/S, sve_neg)
1163 UNARY_OP_TRUE_PREDICATE(vnegL, NegVL, D, sve_neg)
1164 UNARY_OP_TRUE_PREDICATE(vnegF, NegVF, S, sve_fneg)
1165 UNARY_OP_TRUE_PREDICATE(vnegD, NegVD, D, sve_fneg)
1166 
1167 // vector neg - predicated
1168 UNARY_OP_PREDICATE(vnegI, NegVI, B/H/S, sve_neg)
1169 UNARY_OP_PREDICATE(vnegL, NegVL, D, sve_neg)
1170 UNARY_OP_PREDICATE(vnegF, NegVF, S, sve_fneg)
1171 UNARY_OP_PREDICATE(vnegD, NegVD, D, sve_fneg)
1172 
1173 // popcount vector























1174 
1175 instruct vpopcountI(vReg dst, vReg src) %{
1176   predicate(UseSVE > 0);
1177   match(Set dst (PopCountVI src));
1178   format %{ "sve_cnt $dst, $src\t# vector (sve) (S)\n\t" %}










1179   ins_encode %{
1180      __ sve_cnt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg));




1181   %}
1182   ins_pipe(pipe_slow);
1183 %}
1184 























1185 // vector blend
1186 
1187 instruct vblend(vReg dst, vReg src1, vReg src2, pRegGov pg) %{
1188   predicate(UseSVE > 0);
1189   match(Set dst (VectorBlend (Binary src1 src2) pg));
1190   ins_cost(SVE_COST);
1191   format %{ "sve_sel $dst, $pg, $src2, $src1\t# vector blend (sve)" %}
1192   ins_encode %{
1193     Assembler::SIMD_RegVariant size =
1194                __ elemType_to_regVariant(Matcher::vector_element_basic_type(this));
1195     __ sve_sel(as_FloatRegister($dst$$reg), size, as_PRegister($pg$$reg),
1196                as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
1197   %}
1198   ins_pipe(pipe_slow);
1199 %}
1200 
1201 // vector store mask
1202 
1203 instruct vstoremaskB(vReg dst, pRegGov src, immI_1 size) %{
1204   predicate(UseSVE > 0);

3163                ptrue, as_FloatRegister($src$$reg), 0);
3164   %}
3165   ins_pipe(pipe_slow);
3166 %}
3167 
3168 instruct vloadmask_extend(pRegGov dst, vReg src, vReg tmp, rFlagsReg cr) %{
3169   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() != T_BYTE);
3170   match(Set dst (VectorLoadMask src));
3171   effect(TEMP tmp, KILL cr);
3172   ins_cost(3 * SVE_COST);
3173   format %{ "vloadmask $dst, $src\t# vector load mask (sve) (H/S/D)" %}
3174   ins_encode %{
3175     BasicType bt = Matcher::vector_element_basic_type(this);
3176     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3177     __ sve_vector_extend(as_FloatRegister($tmp$$reg), size, as_FloatRegister($src$$reg), __ B);
3178     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
3179   %}
3180   ins_pipe(pipe_slow);
3181 %}
3182 


































































































3183 instruct vmask_gen(pRegGov pg, iRegL len, rFlagsReg cr) %{
3184   predicate(UseSVE > 0);
3185   match(Set pg (VectorMaskGen len));
3186   effect(KILL cr);
3187   ins_cost(SVE_COST);
3188   format %{ "sve_whilelo $pg, zr, $len\t # sve" %}
3189   ins_encode %{
3190     BasicType bt = Matcher::vector_element_basic_type(this);
3191     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3192     __ sve_whilelo(as_PRegister($pg$$reg), size, zr, as_Register($len$$reg));
3193   %}
3194   ins_pipe(pipe_slow);
3195 %}













































































 127   bool op_sve_supported(int opcode, int vlen, BasicType bt) {
 128     int length_in_bytes = vlen * type2aelembytes(bt);
 129     switch (opcode) {
 130       case Op_MulAddVS2VI:
 131       // No multiply reduction instructions
 132       case Op_MulReductionVD:
 133       case Op_MulReductionVF:
 134       case Op_MulReductionVI:
 135       case Op_MulReductionVL:
 136       // Others
 137       case Op_ExtractC:
 138       case Op_ExtractUB:
 139         return false;
 140       // Vector API specific
 141       case Op_VectorLoadShuffle:
 142       case Op_VectorRearrange:
 143         return vlen >= 4 && length_in_bytes <= MaxVectorSize;
 144       case Op_LoadVector:
 145       case Op_StoreVector:
 146         return Matcher::vector_size_supported(bt, vlen);
 147       case Op_ExpandV:
 148         if (UseSVE < 2 || is_subword_type(bt)) return false;
 149       default:
 150         break;
 151     }
 152     // By default, we only support vector operations with no less than 8 bytes and 2 elements.
 153     return 8 <= length_in_bytes && length_in_bytes <= MaxVectorSize && vlen >= 2;
 154   }
 155 
 156   bool masked_op_sve_supported(int opcode, int vlen, BasicType bt) {
 157     if (opcode == Op_VectorRearrange) {
 158       return false;
 159     }
 160     return op_sve_supported(opcode, vlen, bt);
 161   }
 162 %}
 163 
 164 definitions %{
 165   int_def SVE_COST             (200, 200);
 166 %}
 167 
 168 dnl

1155 // vector mul - predicated
1156 BINARY_OP_PREDICATE(vmulB, MulVB, B, sve_mul)
1157 BINARY_OP_PREDICATE(vmulS, MulVS, H, sve_mul)
1158 BINARY_OP_PREDICATE(vmulI, MulVI, S, sve_mul)
1159 BINARY_OP_PREDICATE(vmulL, MulVL, D, sve_mul)
1160 BINARY_OP_PREDICATE(vmulF, MulVF, S, sve_fmul)
1161 BINARY_OP_PREDICATE(vmulD, MulVD, D, sve_fmul)
1162 
1163 // vector neg
1164 UNARY_OP_TRUE_PREDICATE(vnegI, NegVI, B/H/S, sve_neg)
1165 UNARY_OP_TRUE_PREDICATE(vnegL, NegVL, D, sve_neg)
1166 UNARY_OP_TRUE_PREDICATE(vnegF, NegVF, S, sve_fneg)
1167 UNARY_OP_TRUE_PREDICATE(vnegD, NegVD, D, sve_fneg)
1168 
1169 // vector neg - predicated
1170 UNARY_OP_PREDICATE(vnegI, NegVI, B/H/S, sve_neg)
1171 UNARY_OP_PREDICATE(vnegL, NegVL, D, sve_neg)
1172 UNARY_OP_PREDICATE(vnegF, NegVF, S, sve_fneg)
1173 UNARY_OP_PREDICATE(vnegD, NegVD, D, sve_fneg)
1174 
1175 dnl
1176 dnl VPOPCOUNT($1,          $2  )
1177 dnl VPOPCOUNT(name_suffix, size)
1178 define(`VPOPCOUNT', `
1179 instruct vpopcount$1(vReg dst, vReg src) %{
1180   predicate(UseSVE > 0 &&
1181             !n->as_Vector()->is_predicated_vector()`'ifelse($1, `L', ` &&
1182             n->bottom_type()->is_vect()->element_basic_type() == T_LONG', `'));
1183   match(Set dst (PopCountV$1 src));
1184   ins_cost(SVE_COST);
1185   format %{ "sve_cnt $dst, $src\t# vector (sve) ($2)" %}
1186   ins_encode %{
1187     assert(UsePopCountInstruction, "unsupported");dnl
1188 ifelse($1, `I', `
1189     BasicType bt = Matcher::vector_element_basic_type(this);', `')
1190     __ sve_cnt(as_FloatRegister($dst$$reg), ifelse($1, `I', `__ elemType_to_regVariant(bt)', `__ D'),
1191          ptrue, as_FloatRegister($src$$reg));
1192   %}
1193   ins_pipe(pipe_slow);
1194 %}')dnl
1195 dnl
1196 // vector popcount
1197 VPOPCOUNT(I, B/H/S)
1198 VPOPCOUNT(L, D)
1199 
1200 // If the PopCountVL is generated by auto-vectorization, the dst basic
1201 // type is T_INT. And once we have unified the type definition for
1202 // Vector API and auto-vectorization, this rule can be merged with
1203 // "vpopcountL" rule.
1204 instruct vpopcountLI(vReg dst, vReg src, vReg vtmp) %{
1205   predicate(UseSVE > 0 &&
1206             !n->as_Vector()->is_predicated_vector() &&
1207             n->bottom_type()->is_vect()->element_basic_type() == T_INT);
1208   match(Set dst (PopCountVL src));
1209   effect(TEMP_DEF dst, TEMP vtmp);
1210   ins_cost(3 * SVE_COST);
1211   format %{ "sve_cnt $dst, $src\n\t"
1212             "sve_dup $vtmp, #0\n\t"
1213             "sve_uzp1 $dst, $dst, $vtmp\t# vector (sve) (S)" %}
1214   ins_encode %{
1215     assert(UsePopCountInstruction, "unsupported");
1216     __ sve_cnt(as_FloatRegister($dst$$reg), __ D,
1217          ptrue, as_FloatRegister($src$$reg));
1218     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
1219          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($vtmp$$reg));
1220   %}
1221   ins_pipe(pipe_slow);
1222 %}
1223 
1224 dnl
1225 dnl VPOPCOUNT_PREDICATE($1,          $2  )
1226 dnl VPOPCOUNT_PREDICATE(name_suffix, size)
1227 define(`VPOPCOUNT_PREDICATE', `
1228 instruct vpopcount$1_masked(vReg dst_src, pRegGov pg) %{
1229   predicate(UseSVE > 0`'ifelse($1, `L', ` &&
1230             n->bottom_type()->is_vect()->element_basic_type() == T_LONG', `'));
1231   match(Set dst_src (PopCountV$1 dst_src pg));
1232   ins_cost(SVE_COST);
1233   format %{ "sve_cnt $dst_src, $pg, $dst_src\t# vector (sve) ($2)" %}
1234   ins_encode %{
1235     assert(UsePopCountInstruction, "unsupported");dnl
1236 ifelse($1, `I', `
1237     BasicType bt = Matcher::vector_element_basic_type(this);', `')
1238     __ sve_cnt(as_FloatRegister($dst_src$$reg), ifelse($1, `I', `__ elemType_to_regVariant(bt)', `__ D'),
1239          as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
1240   %}
1241   ins_pipe(pipe_slow);
1242 %}')dnl
1243 // vector popcount - predicated
1244 VPOPCOUNT_PREDICATE(I, B/H/S)
1245 VPOPCOUNT_PREDICATE(L, D)
1246 
1247 // vector blend
1248 
1249 instruct vblend(vReg dst, vReg src1, vReg src2, pRegGov pg) %{
1250   predicate(UseSVE > 0);
1251   match(Set dst (VectorBlend (Binary src1 src2) pg));
1252   ins_cost(SVE_COST);
1253   format %{ "sve_sel $dst, $pg, $src2, $src1\t# vector blend (sve)" %}
1254   ins_encode %{
1255     Assembler::SIMD_RegVariant size =
1256                __ elemType_to_regVariant(Matcher::vector_element_basic_type(this));
1257     __ sve_sel(as_FloatRegister($dst$$reg), size, as_PRegister($pg$$reg),
1258                as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
1259   %}
1260   ins_pipe(pipe_slow);
1261 %}
1262 
1263 // vector store mask
1264 
1265 instruct vstoremaskB(vReg dst, pRegGov src, immI_1 size) %{
1266   predicate(UseSVE > 0);

3225                ptrue, as_FloatRegister($src$$reg), 0);
3226   %}
3227   ins_pipe(pipe_slow);
3228 %}
3229 
3230 instruct vloadmask_extend(pRegGov dst, vReg src, vReg tmp, rFlagsReg cr) %{
3231   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() != T_BYTE);
3232   match(Set dst (VectorLoadMask src));
3233   effect(TEMP tmp, KILL cr);
3234   ins_cost(3 * SVE_COST);
3235   format %{ "vloadmask $dst, $src\t# vector load mask (sve) (H/S/D)" %}
3236   ins_encode %{
3237     BasicType bt = Matcher::vector_element_basic_type(this);
3238     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3239     __ sve_vector_extend(as_FloatRegister($tmp$$reg), size, as_FloatRegister($src$$reg), __ B);
3240     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
3241   %}
3242   ins_pipe(pipe_slow);
3243 %}
3244 
3245 // ---------------------------- Compress/Expand Operations ---------------------------
3246 
3247 instruct mcompress(pReg dst, pReg pg, rFlagsReg cr) %{
3248   predicate(UseSVE > 0);
3249   match(Set dst (CompressM pg));
3250   effect(KILL cr);
3251   ins_cost(2 * SVE_COST);
3252   format %{ "sve_cntp rscratch1, $pg\n\t"
3253             "sve_whilelo $dst, zr, rscratch1\t# mask compress (B/H/S/D)" %}
3254   ins_encode %{
3255     BasicType bt = Matcher::vector_element_basic_type(this);
3256     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3257     __ sve_cntp(rscratch1, size, ptrue, as_PRegister($pg$$reg));
3258     __ sve_whilelo(as_PRegister($dst$$reg), size, zr, rscratch1);
3259   %}
3260   ins_pipe(pipe_slow);
3261 %}
3262 
3263 instruct vcompress(vReg dst, vReg src, pRegGov pg) %{
3264   predicate(UseSVE > 0 &&
3265             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
3266              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT ||
3267              n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
3268              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
3269   match(Set dst (CompressV src pg));
3270   ins_cost(SVE_COST);
3271   format %{ "sve_compact $dst, $src, $pg\t# vector compress (S/D)" %}
3272   ins_encode %{
3273     BasicType bt = Matcher::vector_element_basic_type(this);
3274     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3275     __ sve_compact(as_FloatRegister($dst$$reg), size, as_FloatRegister($src$$reg), as_PRegister($pg$$reg));
3276   %}
3277   ins_pipe(pipe_slow);
3278 %}
3279 
3280 instruct vcompressB(vReg dst, vReg src, pReg pg, vReg vtmp1, vReg vtmp2, vReg vtmp3, vReg vtmp4,
3281                     pReg ptmp, pRegGov pgtmp) %{
3282   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
3283   effect(TEMP_DEF dst, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP ptmp, TEMP pgtmp);
3284   match(Set dst (CompressV src pg));
3285   ins_cost(13 * SVE_COST);
3286   format %{ "sve_compact $dst, $src, $pg\t# vector compress (B)" %}
3287   ins_encode %{
3288     __ sve_compress_byte(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
3289                          as_FloatRegister($vtmp1$$reg),as_FloatRegister($vtmp2$$reg),
3290                          as_FloatRegister($vtmp3$$reg),as_FloatRegister($vtmp4$$reg),
3291                          as_PRegister($ptmp$$reg), as_PRegister($pgtmp$$reg));
3292   %}
3293   ins_pipe(pipe_slow);
3294 %}
3295 
3296 instruct vcompressS(vReg dst, vReg src, pReg pg, vReg vtmp1, vReg vtmp2, pRegGov pgtmp) %{
3297   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
3298   effect(TEMP_DEF dst, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp);
3299   match(Set dst (CompressV src pg));
3300   ins_cost(38 * SVE_COST);
3301   format %{ "sve_compact $dst, $src, $pg\t# vector compress (H)" %}
3302   ins_encode %{
3303     __ sve_compress_short(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
3304                           as_FloatRegister($vtmp1$$reg),as_FloatRegister($vtmp2$$reg), as_PRegister($pgtmp$$reg));
3305   %}
3306   ins_pipe(pipe_slow);
3307 %}
3308 
3309 instruct vexpand(vReg dst, vReg src, pRegGov pg) %{
3310   match(Set dst (ExpandV src pg));
3311   effect(TEMP_DEF dst);
3312   ins_cost(4 * SVE_COST);
3313   format %{ "sve_dup $dst, S/D, 0\n\t"
3314             "sve_histcnt $dst, S/D, $pg, $dst, $dst\n\t"
3315             "sve_sub $dst, S/D, 1\n\t"
3316             "sve_tbl $dst, S/D, $src, $dst\t# vector expand (S/D)" %}
3317   ins_encode %{
3318     // Example input:   src   = 1 2 3 4 5 6 7 8
3319     //                  pg    = 1 0 0 1 1 0 1 1
3320     // Expected result: dst   = 4 0 0 5 6 0 7 8
3321 
3322     // The basic idea is to use TBL which can shuffle the elements in the given
3323     // vector flexibly. HISTCNT + SUB is used to generate the second source input
3324     // for TBL whose value is used to select the indexed element from src vector.
3325 
3326     BasicType bt = Matcher::vector_element_basic_type(this);
3327     assert(UseSVE == 2 && !is_subword_type(bt), "unsupported");
3328     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3329     // dst = 0 0 0 0 0 0 0 0
3330     __ sve_dup(as_FloatRegister($dst$$reg), size, 0);
3331     // dst = 5 0 0 4 3 0 2 1
3332     __ sve_histcnt(as_FloatRegister($dst$$reg), size, as_PRegister($pg$$reg),
3333                    as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg));
3334     // dst = 4 -1 -1 3 2 -1 1 0
3335     __ sve_sub(as_FloatRegister($dst$$reg), size, 1);
3336     // dst = 4 0 0 5 6 0 7 8
3337     __ sve_tbl(as_FloatRegister($dst$$reg), size, as_FloatRegister($src$$reg),
3338                as_FloatRegister($dst$$reg));
3339   %}
3340   ins_pipe(pipe_slow);
3341 %}
3342 
3343 instruct vmask_gen(pRegGov pg, iRegL len, rFlagsReg cr) %{
3344   predicate(UseSVE > 0);
3345   match(Set pg (VectorMaskGen len));
3346   effect(KILL cr);
3347   ins_cost(SVE_COST);
3348   format %{ "sve_whilelo $pg, zr, $len\t # sve" %}
3349   ins_encode %{
3350     BasicType bt = Matcher::vector_element_basic_type(this);
3351     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3352     __ sve_whilelo(as_PRegister($pg$$reg), size, zr, as_Register($len$$reg));
3353   %}
3354   ins_pipe(pipe_slow);
3355 %}
3356 
3357 dnl
3358 dnl BITWISE_UNARY($1,        $2,      $3  )
3359 dnl BITWISE_UNARY(insn_name, op_name, insn)
3360 define(`BITWISE_UNARY', `
3361 instruct $1(vReg dst, vReg src) %{
3362   predicate(UseSVE > 0 &&
3363             !n->as_Vector()->is_predicated_vector());
3364   match(Set dst ($2 src));
3365   ins_cost(ifelse($2, `CountTrailingZerosV', `2 * ', `')SVE_COST);
3366   format %{ ifelse($2, `CountTrailingZerosV', `"sve_rbit $dst, $src\n\t"
3367             "$3  $dst, $dst', `"$3 $dst, $src')\t# vector (sve)" %}
3368   ins_encode %{
3369     BasicType bt = Matcher::vector_element_basic_type(this);
3370     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);dnl
3371 ifelse($2, `CountTrailingZerosV', `
3372     __ sve_rbit(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($src$$reg));', `')dnl
3373 ifelse($2, `ReverseBytesV', `
3374     if (bt == T_BYTE) {
3375       if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
3376         __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
3377       }
3378     } else {
3379       __ $3(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($src$$reg));
3380     }', `
3381     __ $3(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($ifelse($2, `CountTrailingZerosV', dst, src)$$reg));')
3382   %}
3383   ins_pipe(pipe_slow);
3384 %}')dnl
3385 dnl
3386 dnl BITWISE_UNARY_PREDICATE($1,        $2,      $3  )
3387 dnl BITWISE_UNARY_PREDICATE(insn_name, op_name, insn)
3388 define(`BITWISE_UNARY_PREDICATE', `
3389 // The dst and src should use the same register to make sure the
3390 // inactive lanes in dst save the same elements as src.
3391 instruct $1_masked(vReg dst_src, pRegGov pg) %{
3392   predicate(UseSVE > 0);
3393   match(Set dst_src ($2 dst_src pg));
3394   ins_cost(ifelse($2, `CountTrailingZerosV', `2 * ', `')SVE_COST);
3395   format %{ ifelse($2, `CountTrailingZerosV', `"sve_rbit $dst_src, $pg, $dst_src\n\t"
3396             "$3  $dst_src, $pg, $dst_src', `"$3 $dst_src, $pg, $dst_src')\t# vector (sve)" %}
3397   ins_encode %{
3398     BasicType bt = Matcher::vector_element_basic_type(this);
3399     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);dnl
3400 ifelse($2, `CountTrailingZerosV', `
3401     __ sve_rbit(as_FloatRegister($dst_src$$reg), size,
3402         as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));', `')dnl
3403 ifelse($2, `ReverseBytesV', `
3404     if (bt == T_BYTE) {
3405       // do nothing
3406     } else {
3407       __ $3(as_FloatRegister($dst_src$$reg), size,
3408           as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
3409     }', `
3410     __ $3(as_FloatRegister($dst_src$$reg), size,
3411         as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));')
3412   %}
3413   ins_pipe(pipe_slow);
3414 %}')dnl
3415 dnl
3416 // ------------------------------ CountLeadingZerosV ------------------------------
3417 BITWISE_UNARY(vcountLeadingZeros, CountLeadingZerosV, sve_clz)
3418 BITWISE_UNARY_PREDICATE(vcountLeadingZeros, CountLeadingZerosV, sve_clz)
3419 
3420 // ------------------------------ CountTrailingZerosV -----------------------------
3421 BITWISE_UNARY(vcountTrailingZeros, CountTrailingZerosV, sve_clz)
3422 BITWISE_UNARY_PREDICATE(vcountTrailingZeros, CountTrailingZerosV, sve_clz)
3423 
3424 // ---------------------------------- ReverseV ------------------------------------
3425 BITWISE_UNARY(vreverse, ReverseV, sve_rbit)
3426 BITWISE_UNARY_PREDICATE(vreverse, ReverseV, sve_rbit)
3427 
3428 // -------------------------------- ReverseBytesV ---------------------------------
3429 BITWISE_UNARY(vreverseBytes, ReverseBytesV, sve_revb)
3430 BITWISE_UNARY_PREDICATE(vreverseBytes, ReverseBytesV, sve_revb)
3431 
< prev index next >