< prev index next >

src/hotspot/cpu/aarch64/aarch64_sve_ad.m4

Print this page

  66   constraint(ALLOC_IN_RC(ptr_reg));
  67   match(AddP reg off);
  68   op_cost(0);
  69   format %{ "[$reg, $off]" %}
  70   interface(MEMORY_INTER) %{
  71     base($reg);
  72     `index'(0xffffffff);
  73     scale(0x0);
  74     disp($off);
  75   %}
  76 %}')dnl
  77 OPERAND_VMEMORYA_INDIRECT_OFFSET(I, 4)
  78 OPERAND_VMEMORYA_INDIRECT_OFFSET(L, 4)
  79 
  80 // The indOff of vmemA is valid only when the vector element (load to/store from)
  81 // size equals to memory element (load from/store to) size.
  82 opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);
  83 
  84 source_hpp %{
  85   bool op_sve_supported(int opcode, int vlen, BasicType bt);

  86 %}
  87 
  88 source %{
  89 
  90   typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
  91                                                              PRegister Pg, const Address &adr);
  92 
  93   // Predicated load/store, with optional ptrue to all elements of given predicate register.
  94   static void loadStoreA_predicated(C2_MacroAssembler masm, bool is_store, FloatRegister reg,
  95                                     PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
  96                                     int opcode, Register base, int index, int size, int disp) {
  97     sve_mem_insn_predicate insn;
  98     int mesize = type2aelembytes(mem_elem_bt);
  99     if (index == -1) {
 100       assert(size == 0, "unsupported address mode: scale size = %d", size);
 101       switch(mesize) {
 102       case 1:
 103         insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
 104         break;
 105       case 2:

 122       ShouldNotReachHere();
 123     }
 124   }
 125 
 126   bool op_sve_supported(int opcode, int vlen, BasicType bt) {
 127     int length_in_bytes = vlen * type2aelembytes(bt);
 128     switch (opcode) {
 129       case Op_MulAddVS2VI:
 130       // No multiply reduction instructions
 131       case Op_MulReductionVD:
 132       case Op_MulReductionVF:
 133       case Op_MulReductionVI:
 134       case Op_MulReductionVL:
 135       // Others
 136       case Op_ExtractC:
 137       case Op_ExtractUB:
 138         return false;
 139       // Vector API specific
 140       case Op_VectorLoadShuffle:
 141       case Op_VectorRearrange:
 142         if (vlen < 4 || length_in_bytes > MaxVectorSize) {
 143           return false;
 144         } else {
 145           return true;
 146         }
 147       case Op_LoadVector:
 148       case Op_StoreVector:
 149         return Matcher::vector_size_supported(bt, vlen);
 150       default:
 151         break;
 152     }
 153     // By default, we only support vector operations with no less than 8 bytes and 2 elements.
 154     return 8 <= length_in_bytes && length_in_bytes <= MaxVectorSize && vlen >= 2;
 155   }








 156 %}
 157 
 158 definitions %{
 159   int_def SVE_COST             (200, 200);
 160 %}
 161 
 162 dnl
 163 dnl ELEMENT_SHORT_CHART($1, $2)
 164 dnl ELEMENT_SHORT_CHART(etype, node)
 165 define(`ELEMENT_SHORT_CHAR',`ifelse(`$1', `T_SHORT',
 166   `($2->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
 167             ($2->bottom_type()->is_vect()->element_basic_type() == T_CHAR))',
 168    `($2->bottom_type()->is_vect()->element_basic_type() == $1)')')dnl
 169 dnl
 170 
 171 // All SVE instructions
 172 
 173 // vector load/store
 174 
 175 // Unpredicated vector load/store

 214   match(Set ifelse(load, $3, dst (LoadVector mem), mem (StoreVector mem src)));
 215   ins_cost(4 * INSN_COST);
 216   format %{ "$1   ifelse(load, $3, `$dst,$mem', `$mem,$src')\t# vector ($6 bits)" %}
 217   ins_encode( `aarch64_enc_'ifelse(load, $3, ldr, str)v$2($7, mem) );
 218   ins_pipe(v$3`_reg_mem'ifelse(eval($4 * 8), 128, 128, 64));
 219 %}')dnl
 220 dnl        $1    $2 $3     $4  $5 $6   $7
 221 VLoadStore(ldrh, H, load,  2,  D, 16,  dst)
 222 VLoadStore(strh, H, store, 2,  D, 16,  src)
 223 VLoadStore(ldrs, S, load,  4,  D, 32,  dst)
 224 VLoadStore(strs, S, store, 4,  D, 32,  src)
 225 VLoadStore(ldrd, D, load,  8,  D, 64,  dst)
 226 VLoadStore(strd, D, store, 8,  D, 64,  src)
 227 VLoadStore(ldrq, Q, load, 16,  X, 128, dst)
 228 VLoadStore(strq, Q, store, 16, X, 128, src)
 229 
 230 // Predicated vector load/store, based on the vector length of the node.
 231 // Only load/store values in the range of the memory_size. This is needed
 232 // when the memory_size is lower than the hardware supported max vector size.
 233 // And this might happen for Vector API mask vector load/store.
 234 instruct loadV_partial(vReg dst, vmemA mem, pRegGov pTmp, rFlagsReg cr) %{
 235   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() > 16 &&
 236             n->as_LoadVector()->memory_size() < MaxVectorSize);
 237   match(Set dst (LoadVector mem));
 238   effect(TEMP pTmp, KILL cr);
 239   ins_cost(6 * SVE_COST);
 240   format %{ "sve_whilelo_zr_imm $pTmp, vector_length\n\t"
 241             "sve_ldr $dst, $pTmp, $mem\t# load vector predicated" %}
 242   ins_encode %{
 243     BasicType bt = Matcher::vector_element_basic_type(this);
 244     __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), __ elemType_to_regVariant(bt),
 245                           Matcher::vector_length(this));
 246     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 247     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg,
 248                           as_PRegister($pTmp$$reg), bt, bt, $mem->opcode(),
 249                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 250   %}
 251   ins_pipe(pipe_slow);
 252 %}
 253 
 254 instruct storeV_partial(vReg src, vmemA mem, pRegGov pTmp, rFlagsReg cr) %{
 255   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() > 16 &&
 256             n->as_StoreVector()->memory_size() < MaxVectorSize);
 257   match(Set mem (StoreVector mem src));
 258   effect(TEMP pTmp, KILL cr);
 259   ins_cost(5 * SVE_COST);
 260   format %{ "sve_whilelo_zr_imm $pTmp, vector_length\n\t"
 261             "sve_str $src, $pTmp, $mem\t# store vector predicated" %}
 262   ins_encode %{
 263     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 264     __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), __ elemType_to_regVariant(bt),
 265                           Matcher::vector_length(this, $src));
 266     FloatRegister src_reg = as_FloatRegister($src$$reg);
 267     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg,
 268                           as_PRegister($pTmp$$reg), bt, bt, $mem->opcode(),
 269                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 270   %}
 271   ins_pipe(pipe_slow);
 272 %}dnl












































































































































 273 



















 274 
 275 // vector reinterpret
 276 
 277 instruct reinterpret(vReg dst) %{
 278   predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() ==
 279                           n->in(1)->bottom_type()->is_vect()->length_in_bytes());  // src == dst
 280   match(Set dst (VectorReinterpret dst));
 281   ins_cost(0);
 282   format %{ "# reinterpret $dst\t# do nothing" %}
 283   ins_encode %{
 284     // empty
 285   %}
 286   ins_pipe(pipe_class_empty);
 287 %}
 288 
 289 instruct reinterpretResize(vReg dst, vReg src, pRegGov pTmp, rFlagsReg cr) %{
 290   predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() !=
 291                           n->in(1)->bottom_type()->is_vect()->length_in_bytes());  // src != dst
 292   match(Set dst (VectorReinterpret src));
 293   effect(TEMP_DEF dst, TEMP pTmp, KILL cr);
 294   ins_cost(3 * SVE_COST);
 295   format %{ "reinterpretResize $dst, $src\t# vector (sve)" %}
 296   ins_encode %{
 297     uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
 298     uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
 299     uint length_in_bytes_resize = length_in_bytes_src < length_in_bytes_dst ?
 300                                   length_in_bytes_src : length_in_bytes_dst;
 301     assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
 302            "invalid vector length");
 303     __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), __ B, length_in_bytes_resize);
 304     __ sve_dup(as_FloatRegister($dst$$reg), __ B, 0);
 305     __ sve_sel(as_FloatRegister($dst$$reg), __ B, as_PRegister($pTmp$$reg),
 306                as_FloatRegister($src$$reg), as_FloatRegister($dst$$reg));
 307   %}
 308   ins_pipe(pipe_slow);
 309 %}


































 310 dnl
 311 dnl UNARY_OP_TRUE_PREDICATE_ETYPE($1,        $2,      $3,           $4,   $5,          %6  )
 312 dnl UNARY_OP_TRUE_PREDICATE_ETYPE(insn_name, op_name, element_type, size, min_vec_len, insn)
 313 define(`UNARY_OP_TRUE_PREDICATE_ETYPE', `
 314 instruct $1(vReg dst, vReg src) %{
 315   predicate(UseSVE > 0 &&
 316             n->bottom_type()->is_vect()->element_basic_type() == $3);
 317   match(Set dst ($2 src));
 318   ins_cost(SVE_COST);
 319   format %{ "$6 $dst, $src\t# vector (sve) ($4)" %}
 320   ins_encode %{
 321     __ $6(as_FloatRegister($dst$$reg), __ $4,
 322          ptrue, as_FloatRegister($src$$reg));
 323   %}
 324   ins_pipe(pipe_slow);
 325 %}')dnl
 326 dnl
 327 
 328 // vector abs
 329 UNARY_OP_TRUE_PREDICATE_ETYPE(vabsB, AbsVB, T_BYTE,   B, 16, sve_abs)
 330 UNARY_OP_TRUE_PREDICATE_ETYPE(vabsS, AbsVS, T_SHORT,  H, 8,  sve_abs)
 331 UNARY_OP_TRUE_PREDICATE_ETYPE(vabsI, AbsVI, T_INT,    S, 4,  sve_abs)
 332 UNARY_OP_TRUE_PREDICATE_ETYPE(vabsL, AbsVL, T_LONG,   D, 2,  sve_abs)
 333 UNARY_OP_TRUE_PREDICATE_ETYPE(vabsF, AbsVF, T_FLOAT,  S, 4,  sve_fabs)
 334 UNARY_OP_TRUE_PREDICATE_ETYPE(vabsD, AbsVD, T_DOUBLE, D, 2,  sve_fabs)
 335 dnl
 336 dnl BINARY_OP_UNPREDICATED($1,        $2       $3,   $4           $5  )
 337 dnl BINARY_OP_UNPREDICATED(insn_name, op_name, size, min_vec_len, insn)
 338 define(`BINARY_OP_UNPREDICATED', `
























 339 instruct $1(vReg dst, vReg src1, vReg src2) %{
 340   predicate(UseSVE > 0);
 341   match(Set dst ($2 src1 src2));
 342   ins_cost(SVE_COST);
 343   format %{ "$5 $dst, $src1, $src2\t # vector (sve) ($3)" %}
 344   ins_encode %{
 345     __ $5(as_FloatRegister($dst$$reg), __ $3,
 346          as_FloatRegister($src1$$reg),
 347          as_FloatRegister($src2$$reg));
 348   %}
 349   ins_pipe(pipe_slow);
 350 %}')dnl
 351 

















 352 // vector add
 353 BINARY_OP_UNPREDICATED(vaddB, AddVB, B, 16, sve_add)
 354 BINARY_OP_UNPREDICATED(vaddS, AddVS, H, 8,  sve_add)
 355 BINARY_OP_UNPREDICATED(vaddI, AddVI, S, 4,  sve_add)
 356 BINARY_OP_UNPREDICATED(vaddL, AddVL, D, 2,  sve_add)
 357 BINARY_OP_UNPREDICATED(vaddF, AddVF, S, 4,  sve_fadd)
 358 BINARY_OP_UNPREDICATED(vaddD, AddVD, D, 2,  sve_fadd)
 359 dnl
 360 dnl BINARY_OP_UNSIZED($1,        $2,      $3,          $4  )
 361 dnl BINARY_OP_UNSIZED(insn_name, op_name, min_vec_len, insn)









 362 define(`BINARY_OP_UNSIZED', `
 363 instruct $1(vReg dst, vReg src1, vReg src2) %{
 364   predicate(UseSVE > 0);
 365   match(Set dst ($2 src1 src2));
 366   ins_cost(SVE_COST);
 367   format %{ "$4  $dst, $src1, $src2\t# vector (sve)" %}
 368   ins_encode %{
 369     __ $4(as_FloatRegister($dst$$reg),
 370          as_FloatRegister($src1$$reg),
 371          as_FloatRegister($src2$$reg));
 372   %}
 373   ins_pipe(pipe_slow);
 374 %}')dnl
 375 
 376 // vector and
 377 BINARY_OP_UNSIZED(vand, AndV, 16, sve_and)
 378 
 379 // vector or
 380 BINARY_OP_UNSIZED(vor, OrV, 16, sve_orr)
 381 
 382 // vector xor
 383 BINARY_OP_UNSIZED(vxor, XorV, 16, sve_eor)



























 384 
 385 // vector not
 386 dnl
 387 define(`MATCH_RULE', `ifelse($1, I,
 388 `match(Set dst (XorV src (ReplicateB m1)));
 389   match(Set dst (XorV src (ReplicateS m1)));
 390   match(Set dst (XorV src (ReplicateI m1)));',
 391 `match(Set dst (XorV src (ReplicateL m1)));')')dnl
 392 dnl
 393 define(`VECTOR_NOT', `
 394 instruct vnot$1`'(vReg dst, vReg src, imm$1_M1 m1) %{
 395   predicate(UseSVE > 0);
 396   MATCH_RULE($1)
 397   ins_cost(SVE_COST);
 398   format %{ "sve_not $dst, $src\t# vector (sve) $2" %}
 399   ins_encode %{
 400     __ sve_not(as_FloatRegister($dst$$reg), __ D,
 401                ptrue, as_FloatRegister($src$$reg));
 402   %}
 403   ins_pipe(pipe_slow);
 404 %}')dnl
 405 dnl        $1,$2
 406 VECTOR_NOT(I, B/H/S)
 407 VECTOR_NOT(L, D)
 408 undefine(MATCH_RULE)
 409 
 410 // vector and_not
 411 dnl
 412 define(`MATCH_RULE', `ifelse($1, I,
 413 `match(Set dst (AndV src1 (XorV src2 (ReplicateB m1))));
 414   match(Set dst (AndV src1 (XorV src2 (ReplicateS m1))));
 415   match(Set dst (AndV src1 (XorV src2 (ReplicateI m1))));',
 416 `match(Set dst (AndV src1 (XorV src2 (ReplicateL m1))));')')dnl
 417 dnl
 418 define(`VECTOR_AND_NOT', `
 419 instruct vand_not$1`'(vReg dst, vReg src1, vReg src2, imm$1_M1 m1) %{
 420   predicate(UseSVE > 0);
 421   MATCH_RULE($1)
 422   ins_cost(SVE_COST);
 423   format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) $2" %}
 424   ins_encode %{
 425     __ sve_bic(as_FloatRegister($dst$$reg),
 426                as_FloatRegister($src1$$reg),
 427                as_FloatRegister($src2$$reg));
 428   %}
 429   ins_pipe(pipe_slow);
 430 %}')dnl
 431 dnl            $1,$2
 432 VECTOR_AND_NOT(I, B/H/S)
 433 VECTOR_AND_NOT(L, D)
 434 undefine(MATCH_RULE)
 435 dnl
 436 dnl VDIVF($1,          $2  , $3         )
 437 dnl VDIVF(name_suffix, size, min_vec_len)
 438 define(`VDIVF', `
 439 instruct vdiv$1(vReg dst_src1, vReg src2) %{
 440   predicate(UseSVE > 0);
 441   match(Set dst_src1 (DivV$1 dst_src1 src2));
 442   ins_cost(SVE_COST);
 443   format %{ "sve_fdiv  $dst_src1, $dst_src1, $src2\t# vector (sve) ($2)" %}
 444   ins_encode %{
 445     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ $2,
 446          ptrue, as_FloatRegister($src2$$reg));
 447   %}
 448   ins_pipe(pipe_slow);
 449 %}')dnl
 450 
 451 // vector float div
 452 VDIVF(F, S, 4)
 453 VDIVF(D, D, 2)
 454 
 455 // vector min/max


 456 
 457 instruct vmin(vReg dst_src1, vReg src2) %{




 458   predicate(UseSVE > 0);
 459   match(Set dst_src1 (MinV dst_src1 src2));
 460   ins_cost(SVE_COST);
 461   format %{ "sve_min $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
 462   ins_encode %{
 463     BasicType bt = Matcher::vector_element_basic_type(this);
 464     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
 465     if (is_floating_point_type(bt)) {
 466       __ sve_fmin(as_FloatRegister($dst_src1$$reg), size,
 467                   ptrue, as_FloatRegister($src2$$reg));
 468     } else {
 469       assert(is_integral_type(bt), "Unsupported type");
 470       __ sve_smin(as_FloatRegister($dst_src1$$reg), size,
 471                   ptrue, as_FloatRegister($src2$$reg));
 472     }
 473   %}
 474   ins_pipe(pipe_slow);
 475 %}




 476 
 477 instruct vmax(vReg dst_src1, vReg src2) %{




 478   predicate(UseSVE > 0);
 479   match(Set dst_src1 (MaxV dst_src1 src2));
 480   ins_cost(SVE_COST);
 481   format %{ "sve_max $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
 482   ins_encode %{
 483     BasicType bt = Matcher::vector_element_basic_type(this);
 484     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
 485     if (is_floating_point_type(bt)) {
 486       __ sve_fmax(as_FloatRegister($dst_src1$$reg), size,
 487                   ptrue, as_FloatRegister($src2$$reg));
 488     } else {
 489       assert(is_integral_type(bt), "Unsupported type");
 490       __ sve_smax(as_FloatRegister($dst_src1$$reg), size,
 491                   ptrue, as_FloatRegister($src2$$reg));
 492     }
 493   %}
 494   ins_pipe(pipe_slow);
 495 %}




 496 
 497 dnl
 498 dnl VFMLA($1           $2    $3         )
 499 dnl VFMLA(name_suffix, size, min_vec_len)
 500 define(`VFMLA', `
 501 // dst_src1 = dst_src1 + src2 * src3
 502 instruct vfmla$1(vReg dst_src1, vReg src2, vReg src3) %{
 503   predicate(UseFMA && UseSVE > 0);
 504   match(Set dst_src1 (FmaV$1 dst_src1 (Binary src2 src3)));
 505   ins_cost(SVE_COST);
 506   format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
 507   ins_encode %{
 508     __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ $2,
 509          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 510   %}
 511   ins_pipe(pipe_slow);
 512 %}')dnl
 513 dnl
 514 // vector fmla
 515 VFMLA(F, S, 4)
 516 VFMLA(D, D, 2)
 517 





















 518 dnl
 519 dnl VFMLS($1           $2    $3         )
 520 dnl VFMLS(name_suffix, size, min_vec_len)
 521 define(`VFMLS', `
 522 // dst_src1 = dst_src1 + -src2 * src3
 523 // dst_src1 = dst_src1 + src2 * -src3
 524 instruct vfmls$1(vReg dst_src1, vReg src2, vReg src3) %{
 525   predicate(UseFMA && UseSVE > 0);
 526   match(Set dst_src1 (FmaV$1 dst_src1 (Binary (NegV$1 src2) src3)));
 527   match(Set dst_src1 (FmaV$1 dst_src1 (Binary src2 (NegV$1 src3))));
 528   ins_cost(SVE_COST);
 529   format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
 530   ins_encode %{
 531     __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ $2,
 532          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 533   %}
 534   ins_pipe(pipe_slow);
 535 %}')dnl
 536 dnl
 537 // vector fmls

 628 VMLS(B, B, 16)
 629 VMLS(S, H, 8)
 630 VMLS(I, S, 4)
 631 VMLS(L, D, 2)
 632 
 633 dnl
 634 dnl BINARY_OP_TRUE_PREDICATE($1,        $2,      $3,   $4,          $5  )
 635 dnl BINARY_OP_TRUE_PREDICATE(insn_name, op_name, size, min_vec_len, insn)
 636 define(`BINARY_OP_TRUE_PREDICATE', `
 637 instruct $1(vReg dst_src1, vReg src2) %{
 638   predicate(UseSVE > 0);
 639   match(Set dst_src1 ($2 dst_src1 src2));
 640   ins_cost(SVE_COST);
 641   format %{ "$5 $dst_src1, $dst_src1, $src2\t # vector (sve) ($3)" %}
 642   ins_encode %{
 643     __ $5(as_FloatRegister($dst_src1$$reg), __ $3,
 644          ptrue, as_FloatRegister($src2$$reg));
 645   %}
 646   ins_pipe(pipe_slow);
 647 %}')dnl
 648 
 649 // vector mul
 650 BINARY_OP_TRUE_PREDICATE(vmulB, MulVB, B, 16, sve_mul)
 651 BINARY_OP_TRUE_PREDICATE(vmulS, MulVS, H, 8,  sve_mul)
 652 BINARY_OP_TRUE_PREDICATE(vmulI, MulVI, S, 4,  sve_mul)
 653 BINARY_OP_TRUE_PREDICATE(vmulL, MulVL, D, 2,  sve_mul)
 654 BINARY_OP_UNPREDICATED(vmulF, MulVF, S, 4, sve_fmul)
 655 BINARY_OP_UNPREDICATED(vmulD, MulVD, D, 2, sve_fmul)








 656 
 657 dnl
 658 dnl UNARY_OP_TRUE_PREDICATE($1,        $2,      $3,   $4,            $5  )
 659 dnl UNARY_OP_TRUE_PREDICATE(insn_name, op_name, size, min_vec_bytes, insn)
 660 define(`UNARY_OP_TRUE_PREDICATE', `
 661 instruct $1(vReg dst, vReg src) %{
 662   predicate(UseSVE > 0);
 663   match(Set dst ($2 src));
 664   ins_cost(SVE_COST);
 665   format %{ "$5 $dst, $src\t# vector (sve) ($3)" %}
 666   ins_encode %{
 667     __ $5(as_FloatRegister($dst$$reg), __ $3,
 668          ptrue, as_FloatRegister($src$$reg));
 669   %}
 670   ins_pipe(pipe_slow);
 671 %}')dnl
 672 dnl
 673 // vector fneg
 674 UNARY_OP_TRUE_PREDICATE(vnegF, NegVF, S, 16, sve_fneg)
 675 UNARY_OP_TRUE_PREDICATE(vnegD, NegVD, D, 16, sve_fneg)




 676 
 677 // popcount vector
 678 
 679 instruct vpopcountI(vReg dst, vReg src) %{
 680   predicate(UseSVE > 0);
 681   match(Set dst (PopCountVI src));
 682   format %{ "sve_cnt $dst, $src\t# vector (sve) (S)\n\t" %}
 683   ins_encode %{
 684      __ sve_cnt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg));
 685   %}
 686   ins_pipe(pipe_slow);
 687 %}
 688 
 689 // vector mask compare
 690 
 691 instruct vmaskcmp(vReg dst, vReg src1, vReg src2, immI cond, pRegGov pTmp, rFlagsReg cr) %{
 692   predicate(UseSVE > 0);
 693   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
 694   effect(TEMP pTmp, KILL cr);
 695   ins_cost(2 * SVE_COST);
 696   format %{ "sve_cmp $pTmp, $src1, $src2\n\t"
 697             "sve_cpy $dst, $pTmp, -1\t# vector mask cmp (sve)" %}
 698   ins_encode %{
 699     BasicType bt = Matcher::vector_element_basic_type(this);
 700     __ sve_compare(as_PRegister($pTmp$$reg), bt, ptrue, as_FloatRegister($src1$$reg),
 701                    as_FloatRegister($src2$$reg), (int)$cond$$constant);
 702     __ sve_cpy(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
 703                as_PRegister($pTmp$$reg), -1, false);
 704   %}
 705   ins_pipe(pipe_slow);
 706 %}
 707 
 708 // vector blend
 709 
 710 instruct vblend(vReg dst, vReg src1, vReg src2, vReg src3, pRegGov pTmp, rFlagsReg cr) %{
 711   predicate(UseSVE > 0);
 712   match(Set dst (VectorBlend (Binary src1 src2) src3));
 713   effect(TEMP pTmp, KILL cr);
 714   ins_cost(2 * SVE_COST);
 715   format %{ "sve_cmpeq $pTmp, $src3, -1\n\t"
 716             "sve_sel $dst, $pTmp, $src2, $src1\t# vector blend (sve)" %}
 717   ins_encode %{
 718     Assembler::SIMD_RegVariant size =
 719       __ elemType_to_regVariant(Matcher::vector_element_basic_type(this));
 720     __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), size,
 721                ptrue, as_FloatRegister($src3$$reg), -1);
 722     __ sve_sel(as_FloatRegister($dst$$reg), size, as_PRegister($pTmp$$reg),
 723                as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
 724   %}
 725   ins_pipe(pipe_slow);
 726 %}
 727 
 728 // vector blend with compare
 729 
 730 instruct vblend_maskcmp(vReg dst, vReg src1, vReg src2, vReg src3,
 731                         vReg src4, pRegGov pTmp, immI cond, rFlagsReg cr) %{
 732   predicate(UseSVE > 0);
 733   match(Set dst (VectorBlend (Binary src1 src2) (VectorMaskCmp (Binary src3 src4) cond)));
 734   effect(TEMP pTmp, KILL cr);
 735   ins_cost(2 * SVE_COST);
 736   format %{ "sve_cmp $pTmp, $src3, $src4\t# vector cmp (sve)\n\t"
 737             "sve_sel $dst, $pTmp, $src2, $src1\t# vector blend (sve)" %}
 738   ins_encode %{
 739     BasicType bt = Matcher::vector_element_basic_type(this);
 740     __ sve_compare(as_PRegister($pTmp$$reg), bt, ptrue, as_FloatRegister($src3$$reg),
 741                    as_FloatRegister($src4$$reg), (int)$cond$$constant);
 742     __ sve_sel(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
 743                as_PRegister($pTmp$$reg), as_FloatRegister($src2$$reg),
 744                as_FloatRegister($src1$$reg));
 745   %}
 746   ins_pipe(pipe_slow);
 747 %}
 748 
 749 // vector load mask
 750 
 751 instruct vloadmaskB(vReg dst, vReg src) %{
 752   predicate(UseSVE > 0 &&
 753             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
 754   match(Set dst (VectorLoadMask src));

 755   ins_cost(SVE_COST);
 756   format %{ "sve_neg $dst, $src\t# vector load mask (B)" %}
 757   ins_encode %{
 758     __ sve_neg(as_FloatRegister($dst$$reg), __ B, ptrue, as_FloatRegister($src$$reg));
 759   %}
 760   ins_pipe(pipe_slow);
 761 %}
 762 
 763 instruct vloadmaskS(vReg dst, vReg src) %{
 764   predicate(UseSVE > 0 &&
 765             n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
 766   match(Set dst (VectorLoadMask src));
 767   ins_cost(2 * SVE_COST);
 768   format %{ "sve_uunpklo $dst, H, $src\n\t"
 769             "sve_neg $dst, $dst\t# vector load mask (B to H)" %}
 770   ins_encode %{
 771     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
 772     __ sve_neg(as_FloatRegister($dst$$reg), __ H, ptrue, as_FloatRegister($dst$$reg));
 773   %}
 774   ins_pipe(pipe_slow);
 775 %}
 776 
 777 instruct vloadmaskI(vReg dst, vReg src) %{
 778   predicate(UseSVE > 0 &&
 779             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
 780              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
 781   match(Set dst (VectorLoadMask src));

 782   ins_cost(3 * SVE_COST);
 783   format %{ "sve_uunpklo $dst, H, $src\n\t"
 784             "sve_uunpklo $dst, S, $dst\n\t"
 785             "sve_neg $dst, $dst\t# vector load mask (B to S)" %}
 786   ins_encode %{
 787     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
 788     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($dst$$reg));
 789     __ sve_neg(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($dst$$reg));
 790   %}
 791   ins_pipe(pipe_slow);
 792 %}
 793 
 794 instruct vloadmaskL(vReg dst, vReg src) %{
 795   predicate(UseSVE > 0 &&
 796             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
 797              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
 798   match(Set dst (VectorLoadMask src));
 799   ins_cost(4 * SVE_COST);
 800   format %{ "sve_uunpklo $dst, H, $src\n\t"
 801             "sve_uunpklo $dst, S, $dst\n\t"
 802             "sve_uunpklo $dst, D, $dst\n\t"
 803             "sve_neg $dst, $dst\t# vector load mask (B to D)" %}
 804   ins_encode %{
 805     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
 806     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($dst$$reg));
 807     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($dst$$reg));
 808     __ sve_neg(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg));
 809   %}
 810   ins_pipe(pipe_slow);
 811 %}
 812 
 813 // vector store mask
 814 
 815 instruct vstoremaskB(vReg dst, vReg src, immI_1 size) %{
 816   predicate(UseSVE > 0);
 817   match(Set dst (VectorStoreMask src size));
 818   ins_cost(SVE_COST);
 819   format %{ "sve_neg $dst, $src\t# vector store mask (B)" %}
 820   ins_encode %{
 821     __ sve_neg(as_FloatRegister($dst$$reg), __ B, ptrue,
 822                as_FloatRegister($src$$reg));
 823   %}
 824   ins_pipe(pipe_slow);
 825 %}
 826 
 827 instruct vstoremaskS(vReg dst, vReg src, vReg tmp, immI_2 size) %{
 828   predicate(UseSVE > 0);
 829   match(Set dst (VectorStoreMask src size));
 830   effect(TEMP_DEF dst, TEMP tmp);
 831   ins_cost(3 * SVE_COST);
 832   format %{ "sve_dup $tmp, H, 0\n\t"
 833             "sve_uzp1 $dst, B, $src, $tmp\n\t"
 834             "sve_neg $dst, B, $dst\t# vector store mask (sve) (H to B)" %}
 835   ins_encode %{
 836     __ sve_dup(as_FloatRegister($tmp$$reg), __ H, 0);
 837     __ sve_uzp1(as_FloatRegister($dst$$reg), __ B,
 838                 as_FloatRegister($src$$reg), as_FloatRegister($tmp$$reg));
 839     __ sve_neg(as_FloatRegister($dst$$reg), __ B, ptrue,
 840                as_FloatRegister($dst$$reg));
 841 
 842   %}
 843   ins_pipe(pipe_slow);
 844 %}
 845 
 846 instruct vstoremaskI(vReg dst, vReg src, vReg tmp, immI_4 size) %{
 847   predicate(UseSVE > 0);
 848   match(Set dst (VectorStoreMask src size));
 849   effect(TEMP_DEF dst, TEMP tmp);
 850   ins_cost(4 * SVE_COST);
 851   format %{ "sve_dup $tmp, S, 0\n\t"
 852             "sve_uzp1 $dst, H, $src, $tmp\n\t"
 853             "sve_uzp1 $dst, B, $dst, $tmp\n\t"
 854             "sve_neg $dst, B, $dst\t# vector store mask (sve) (S to B)" %}
 855   ins_encode %{
 856     __ sve_dup(as_FloatRegister($tmp$$reg), __ S, 0);
 857     __ sve_uzp1(as_FloatRegister($dst$$reg), __ H,
 858                 as_FloatRegister($src$$reg), as_FloatRegister($tmp$$reg));
 859     __ sve_uzp1(as_FloatRegister($dst$$reg), __ B,
 860                 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
 861     __ sve_neg(as_FloatRegister($dst$$reg), __ B, ptrue,
 862                as_FloatRegister($dst$$reg));
 863   %}
 864   ins_pipe(pipe_slow);
 865 %}
 866 
 867 instruct vstoremaskL(vReg dst, vReg src, vReg tmp, immI_8 size) %{
 868   predicate(UseSVE > 0);
 869   match(Set dst (VectorStoreMask src size));
 870   effect(TEMP_DEF dst, TEMP tmp);
 871   ins_cost(5 * SVE_COST);
 872   format %{ "sve_dup $tmp, D, 0\n\t"
 873             "sve_uzp1 $dst, S, $src, $tmp\n\t"
 874             "sve_uzp1 $dst, H, $dst, $tmp\n\t"
 875             "sve_uzp1 $dst, B, $dst, $tmp\n\t"
 876             "sve_neg $dst, B, $dst\t# vector store mask (sve) (D to B)" %}
 877   ins_encode %{
 878     __ sve_dup(as_FloatRegister($tmp$$reg), __ D, 0);
 879     __ sve_uzp1(as_FloatRegister($dst$$reg), __ S,
 880                 as_FloatRegister($src$$reg), as_FloatRegister($tmp$$reg));
 881     __ sve_uzp1(as_FloatRegister($dst$$reg), __ H,
 882                 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
 883     __ sve_uzp1(as_FloatRegister($dst$$reg), __ B,
 884                 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
 885     __ sve_neg(as_FloatRegister($dst$$reg), __ B, ptrue,
 886                as_FloatRegister($dst$$reg));
 887   %}
 888   ins_pipe(pipe_slow);
 889 %}
 890 dnl
 891 dnl
 892 dnl VLOADMASK_LOADV($1,    $2  )
 893 dnl VLOADMASK_LOADV(esize, cond)
 894 define(`VLOADMASK_LOADV', `
 895 instruct vloadmask_loadV_$1(vReg dst, ifelse($1, `byte', vmemA, indirect) mem) %{
 896   predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() == MaxVectorSize &&
 897             type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) $2);
 898   match(Set dst (VectorLoadMask (LoadVector mem)));
 899   ins_cost(5 * SVE_COST);
 900   format %{ "sve_ld1b $dst, $mem\n\t"
 901             "sve_neg $dst, $dst\t# load vector mask (sve)" %}
 902   ins_encode %{
 903     FloatRegister dst_reg = as_FloatRegister($dst$$reg);

 904     BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
 905     Assembler::SIMD_RegVariant to_vect_variant = __ elemType_to_regVariant(to_vect_bt);
 906     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg, ptrue,
 907                           T_BOOLEAN, to_vect_bt, $mem->opcode(),

 908                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 909     __ sve_neg(dst_reg, to_vect_variant, ptrue, dst_reg);
 910   %}
 911   ins_pipe(pipe_slow);
 912 %}')dnl
 913 dnl
 914 define(`ARGLIST',
 915 `ifelse($1, `byte', vmemA, indirect) mem, vReg src, vReg tmp, ifelse($1, `byte', immI_1, immI_gt_1) esize')
 916 dnl
 917 dnl STOREV_VSTOREMASK($1,  )
 918 dnl STOREV_VSTOREMASK(esize)
 919 define(`STOREV_VSTOREMASK', `
 920 instruct storeV_vstoremask_$1(ARGLIST($1)) %{
 921   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() *
 922                           n->as_StoreVector()->in(MemNode::ValueIn)->in(2)->get_int() == MaxVectorSize);
 923   match(Set mem (StoreVector mem (VectorStoreMask src esize)));
 924   effect(TEMP tmp);
 925   ins_cost(5 * SVE_COST);
 926   format %{ "sve_neg $tmp, $src\n\t"
 927             "sve_st1b $tmp, $mem\t# store vector mask (sve)" %}
 928   ins_encode %{
 929     BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
 930     assert(type2aelembytes(from_vect_bt) == (int)$esize$$constant, "unsupported type.");
 931     Assembler::SIMD_RegVariant from_vect_variant = __ elemBytes_to_regVariant($esize$$constant);
 932     __ sve_neg(as_FloatRegister($tmp$$reg), from_vect_variant, ptrue,
 933                as_FloatRegister($src$$reg));
 934     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($tmp$$reg),
 935                           ptrue, T_BOOLEAN, from_vect_bt, $mem->opcode(),
 936                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 937   %}
 938   ins_pipe(pipe_slow);
 939 %}')dnl
 940 undefine(ARGLIST)dnl
 941 dnl
 942 // load/store mask vector
 943 VLOADMASK_LOADV(byte, == 1)
 944 VLOADMASK_LOADV(non_byte, > 1)
 945 STOREV_VSTOREMASK(byte)
 946 STOREV_VSTOREMASK(non_byte)
 947 
 948 // vector add reduction
 949 
 950 instruct reduce_addI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp) %{
 951   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
 952   match(Set dst (AddReductionVI src1 src2));
 953   effect(TEMP_DEF dst, TEMP vtmp);
































 954   ins_cost(SVE_COST);
 955   format %{ "sve_reduce_addI $dst, $src1, $src2\t# addB/S/I reduction (sve) (may extend)" %}
 956   ins_encode %{
 957     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
 958     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
 959     __ sve_uaddv(as_FloatRegister($vtmp$$reg), variant, ptrue, as_FloatRegister($src2$$reg));
 960     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
 961     __ addw($dst$$Register, $dst$$Register, $src1$$Register);
 962     if (bt == T_BYTE) {
 963       __ sxtb($dst$$Register, $dst$$Register);
 964     } else if (bt == T_SHORT) {
 965       __ sxth($dst$$Register, $dst$$Register);
 966     } else {
 967       assert(bt == T_INT, "unsupported type");
 968     }
 969   %}
 970   ins_pipe(pipe_slow);
 971 %}
 972 
 973 instruct reduce_addI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,


























 974                              pRegGov ptmp, rFlagsReg cr) %{
 975   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
 976   match(Set dst (AddReductionVI src1 src2));





 977   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
 978   ins_cost(SVE_COST);
 979   format %{ "sve_reduce_addI $dst, $src1, $src2\t# addI reduction partial (sve) (may extend)" %}
 980   ins_encode %{
 981     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
 982     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
 983     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
 984                           Matcher::vector_length(this, $src2));
 985     __ sve_uaddv(as_FloatRegister($vtmp$$reg), variant,
 986                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
 987     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
 988     __ addw($dst$$Register, $dst$$Register, $src1$$Register);
 989     if (bt == T_BYTE) {
 990       __ sxtb($dst$$Register, $dst$$Register);
 991     } else if (bt == T_SHORT) {
 992       __ sxth($dst$$Register, $dst$$Register);
 993     } else {
 994       assert(bt == T_INT, "unsupported type");
 995     }
 996   %}
 997   ins_pipe(pipe_slow);
 998 %}
 999 
1000 instruct reduce_addL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp) %{
1001   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1002   match(Set dst (AddReductionVL src1 src2));
1003   effect(TEMP_DEF dst, TEMP vtmp);
1004   ins_cost(SVE_COST);
1005   format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction (sve)" %}
1006   ins_encode %{
1007     __ sve_uaddv(as_FloatRegister($vtmp$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
1008     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1009     __ add($dst$$Register, $dst$$Register, $src1$$Register);
1010   %}
1011   ins_pipe(pipe_slow);
1012 %}
1013 
1014 instruct reduce_addL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,



1015                              pRegGov ptmp, rFlagsReg cr) %{
1016   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1017   match(Set dst (AddReductionVL src1 src2));





1018   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1019   ins_cost(SVE_COST);
1020   format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction partial (sve)" %}
1021   ins_encode %{
1022     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1023                           Matcher::vector_length(this, $src2));
1024     __ sve_uaddv(as_FloatRegister($vtmp$$reg), __ D,
1025                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1026     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1027     __ add($dst$$Register, $dst$$Register, $src1$$Register);
1028   %}
1029   ins_pipe(pipe_slow);
1030 %}
1031 
1032 dnl
1033 dnl REDUCE_ADDF($1,        $2,      $3,      $4  )
1034 dnl REDUCE_ADDF(insn_name, op_name, reg_dst, size)
1035 define(`REDUCE_ADDF', `
1036 instruct $1($3 src1_dst, vReg src2) %{
1037   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1038   match(Set src1_dst (AddReductionV$2 src1_dst src2));

1039   ins_cost(SVE_COST);
1040   format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) ($4)" %}
1041   ins_encode %{
1042     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1043          ptrue, as_FloatRegister($src2$$reg));
1044   %}
1045   ins_pipe(pipe_slow);
1046 %}')dnl
1047 dnl
1048 dnl
1049 dnl REDUCE_ADDF_PARTIAL($1,        $2,     $3,      $4  )
1050 dnl REDUCE_ADDF_PARTIAL(insn_name, suffix, reg_dst, size)
1051 define(`REDUCE_ADDF_PARTIAL', `
1052 instruct $1($3 src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
1053   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1054   match(Set src1_dst (AddReductionV$2 src1_dst src2));

1055   ins_cost(SVE_COST);
1056   effect(TEMP ptmp, KILL cr);
1057   format %{ "sve_reduce_add$2 $src1_dst, $src1_dst, $src2\t# add$2 reduction partial (sve) ($4)" %}
1058   ins_encode %{
1059     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
1060                           Matcher::vector_length(this, $src2));
1061     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1062                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1063   %}
1064   ins_pipe(pipe_slow);
1065 %}')dnl
1066 dnl
1067 REDUCE_ADDF(reduce_addF, F, vRegF, S)
1068 REDUCE_ADDF_PARTIAL(reduce_addF_partial, F, vRegF, S)
1069 REDUCE_ADDF(reduce_addD, D, vRegD, D)
1070 REDUCE_ADDF_PARTIAL(reduce_addD_partial, D, vRegD, D)
1071 
1072 // vector and reduction
1073 
1074 instruct reduce_andI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp) %{
1075   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1076             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1077   match(Set dst (AndReductionV src1 src2));
1078   effect(TEMP_DEF dst, TEMP vtmp);

1079   ins_cost(SVE_COST);
1080   format %{ "sve_reduce_andI $dst, $src1, $src2\t# andB/S/I reduction (sve) (may extend)" %}
1081   ins_encode %{
1082     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1083     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1084     __ sve_andv(as_FloatRegister($vtmp$$reg), variant, ptrue, as_FloatRegister($src2$$reg));
1085     __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1086     __ andw($dst$$Register, $dst$$Register, $src1$$Register);
1087     if (bt == T_BYTE) {
1088       __ sxtb($dst$$Register, $dst$$Register);
1089     } else if (bt == T_SHORT) {
1090       __ sxth($dst$$Register, $dst$$Register);
1091     } else {
1092       assert(bt == T_INT, "unsupported type");
1093     }
1094   %}
1095   ins_pipe(pipe_slow);
1096 %}
1097 
1098 instruct reduce_andI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1099                              pRegGov ptmp, rFlagsReg cr) %{
1100   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1101             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1102   match(Set dst (AndReductionV src1 src2));
1103   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);






1104   ins_cost(SVE_COST);
1105   format %{ "sve_reduce_andI $dst, $src1, $src2\t# andI reduction partial (sve) (may extend)" %}























1106   ins_encode %{
1107     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1108     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1109     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1110                           Matcher::vector_length(this, $src2));
1111     __ sve_andv(as_FloatRegister($vtmp$$reg), variant,
1112                 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1113     __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1114     __ andw($dst$$Register, $dst$$Register, $src1$$Register);
1115     if (bt == T_BYTE) {
1116       __ sxtb($dst$$Register, $dst$$Register);
1117     } else if (bt == T_SHORT) {
1118       __ sxth($dst$$Register, $dst$$Register);
1119     } else {
1120       assert(bt == T_INT, "unsupported type");
1121     }
1122   %}
1123   ins_pipe(pipe_slow);
1124 %}
1125 
1126 instruct reduce_andL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp) %{
1127   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1128             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1129   match(Set dst (AndReductionV src1 src2));
1130   effect(TEMP_DEF dst, TEMP vtmp);
1131   ins_cost(SVE_COST);
1132   format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction (sve)" %}
1133   ins_encode %{
1134     __ sve_andv(as_FloatRegister($vtmp$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
1135     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1136     __ andr($dst$$Register, $dst$$Register, $src1$$Register);
1137   %}
1138   ins_pipe(pipe_slow);
1139 %}
1140 
1141 instruct reduce_andL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1142                              pRegGov ptmp, rFlagsReg cr) %{
1143   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1144             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1145   match(Set dst (AndReductionV src1 src2));







1146   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1147   ins_cost(SVE_COST);
1148   format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction partial (sve)" %}
1149   ins_encode %{
1150     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1151                           Matcher::vector_length(this, $src2));
1152     __ sve_andv(as_FloatRegister($vtmp$$reg), __ D,
1153                 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1154     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1155     __ andr($dst$$Register, $dst$$Register, $src1$$Register);

1156   %}
1157   ins_pipe(pipe_slow);
1158 %}
1159 
1160 // vector or reduction
1161 
1162 instruct reduce_orI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp) %{
1163   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1164             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1165   match(Set dst (OrReductionV src1 src2));
1166   effect(TEMP_DEF dst, TEMP vtmp);
1167   ins_cost(SVE_COST);
1168   format %{ "sve_reduce_orI $dst, $src1, $src2\t# orB/S/I reduction (sve) (may extend)" %}
1169   ins_encode %{
1170     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1171     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1172     __ sve_orv(as_FloatRegister($vtmp$$reg), variant, ptrue, as_FloatRegister($src2$$reg));
1173     __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1174     __ orrw($dst$$Register, $dst$$Register, $src1$$Register);
1175     if (bt == T_BYTE) {
1176       __ sxtb($dst$$Register, $dst$$Register);
1177     } else if (bt == T_SHORT) {
1178       __ sxth($dst$$Register, $dst$$Register);
1179     } else {
1180       assert(bt == T_INT, "unsupported type");
1181     }
1182   %}
1183   ins_pipe(pipe_slow);
1184 %}
1185 
1186 instruct reduce_orI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1187                              pRegGov ptmp, rFlagsReg cr) %{
1188   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1189             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1190   match(Set dst (OrReductionV src1 src2));
1191   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);


1192   ins_cost(SVE_COST);
1193   format %{ "sve_reduce_orI $dst, $src1, $src2\t# orI reduction partial (sve) (may extend)" %}
1194   ins_encode %{
1195     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1196     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1197     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1198                           Matcher::vector_length(this, $src2));
1199     __ sve_orv(as_FloatRegister($vtmp$$reg), variant,
1200                as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1201     __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1202     __ orrw($dst$$Register, $dst$$Register, $src1$$Register);
1203     if (bt == T_BYTE) {
1204       __ sxtb($dst$$Register, $dst$$Register);
1205     } else if (bt == T_SHORT) {
1206       __ sxth($dst$$Register, $dst$$Register);
1207     } else {
1208       assert(bt == T_INT, "unsupported type");
1209     }
1210   %}
1211   ins_pipe(pipe_slow);
1212 %}

1213 
1214 instruct reduce_orL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp) %{
1215   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1216             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1217   match(Set dst (OrReductionV src1 src2));
1218   effect(TEMP_DEF dst, TEMP vtmp);
1219   ins_cost(SVE_COST);
1220   format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction (sve)" %}
1221   ins_encode %{
1222     __ sve_orv(as_FloatRegister($vtmp$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
1223     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1224     __ orr($dst$$Register, $dst$$Register, $src1$$Register);
1225   %}
1226   ins_pipe(pipe_slow);
1227 %}





1228 
1229 instruct reduce_orL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1230                              pRegGov ptmp, rFlagsReg cr) %{
1231   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1232             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1233   match(Set dst (OrReductionV src1 src2));
1234   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1235   ins_cost(SVE_COST);
1236   format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction partial (sve)" %}
1237   ins_encode %{
1238     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1239                           Matcher::vector_length(this, $src2));
1240     __ sve_orv(as_FloatRegister($vtmp$$reg), __ D,
1241                as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1242     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1243     __ orr($dst$$Register, $dst$$Register, $src1$$Register);
1244   %}
1245   ins_pipe(pipe_slow);
1246 %}





1247 
1248 // vector xor reduction




1249 
1250 instruct reduce_eorI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp) %{
1251   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1252             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1253   match(Set dst (XorReductionV src1 src2));
1254   effect(TEMP_DEF dst, TEMP vtmp);












1255   ins_cost(SVE_COST);
1256   format %{ "sve_reduce_eorI $dst, $src1, $src2\t# xorB/H/I reduction (sve) (may extend)" %}
1257   ins_encode %{
1258     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1259     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1260     __ sve_eorv(as_FloatRegister($vtmp$$reg), variant, ptrue, as_FloatRegister($src2$$reg));
1261     __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1262     __ eorw($dst$$Register, $dst$$Register, $src1$$Register);
1263     if (bt == T_BYTE) {
1264       __ sxtb($dst$$Register, $dst$$Register);
1265     } else if (bt == T_SHORT) {
1266       __ sxth($dst$$Register, $dst$$Register);
1267     } else {
1268       assert(bt == T_INT, "unsupported type");
1269     }
1270   %}
1271   ins_pipe(pipe_slow);
1272 %}
1273 
1274 instruct reduce_eorI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,






















1275                              pRegGov ptmp, rFlagsReg cr) %{
1276   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1277             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1278   match(Set dst (XorReductionV src1 src2));


1279   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1280   ins_cost(SVE_COST);
1281   format %{ "sve_reduce_eorI $dst, $src1, $src2\t# xorI reduction partial (sve) (may extend)" %}
1282   ins_encode %{
1283     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1284     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1285     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1286                           Matcher::vector_length(this, $src2));
1287     __ sve_eorv(as_FloatRegister($vtmp$$reg), variant,
1288                 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1289     __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1290     __ eorw($dst$$Register, $dst$$Register, $src1$$Register);
1291     if (bt == T_BYTE) {
1292       __ sxtb($dst$$Register, $dst$$Register);
1293     } else if (bt == T_SHORT) {
1294       __ sxth($dst$$Register, $dst$$Register);
1295     } else {
1296       assert(bt == T_INT, "unsupported type");
1297     }
1298   %}
1299   ins_pipe(pipe_slow);
1300 %}
1301 
1302 instruct reduce_eorL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp) %{
1303   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1304             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1305   match(Set dst (XorReductionV src1 src2));
1306   effect(TEMP_DEF dst, TEMP vtmp);
1307   ins_cost(SVE_COST);
1308   format %{ "sve_reduce_eorL $dst, $src1, $src2\t# xorL reduction (sve)" %}
1309   ins_encode %{
1310     __ sve_eorv(as_FloatRegister($vtmp$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
1311     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1312     __ eor($dst$$Register, $dst$$Register, $src1$$Register);
1313   %}
1314   ins_pipe(pipe_slow);
1315 %}
1316 
1317 instruct reduce_eorL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1318                              pRegGov ptmp, rFlagsReg cr) %{
1319   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1320             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1321   match(Set dst (XorReductionV src1 src2));

1322   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1323   ins_cost(SVE_COST);
1324   format %{ "sve_reduce_eorL $dst, $src1, $src2\t# xorL reduction partial (sve)" %}
1325   ins_encode %{
1326     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1327                           Matcher::vector_length(this, $src2));
1328     __ sve_eorv(as_FloatRegister($vtmp$$reg), __ D,
1329                 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1330     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1331     __ eor($dst$$Register, $dst$$Register, $src1$$Register);
1332   %}
1333   ins_pipe(pipe_slow);
1334 %}
1335 
1336 dnl
1337 dnl REDUCE_MAXMIN_I($1,      $2,      $3 )
1338 dnl REDUCE_MAXMIN_I(min_max, op_mame, cmp)
1339 define(`REDUCE_MAXMIN_I', `
1340 instruct reduce_$1I(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp) %{
1341   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1342             (n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
1343              n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
1344              n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT));
1345   match(Set dst ($2 src1 src2));
1346   effect(TEMP_DEF dst, TEMP vtmp);

1347   ins_cost(SVE_COST);
1348   format %{ "sve_reduce_$1I $dst, $src1, $src2\t# reduce $1B/S/I (sve)" %}
1349   ins_encode %{
1350     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1351     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1352     __ sve_s$1v(as_FloatRegister($vtmp$$reg), variant, ptrue, as_FloatRegister($src2$$reg));
1353     __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1354     __ cmpw($dst$$Register, $src1$$Register);
1355     __ cselw(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($src1$$reg), Assembler::$3);
1356   %}
1357   ins_pipe(pipe_slow);
1358 %}')dnl
1359 dnl
1360 dnl REDUCE_MAXMIN_L($1,      $2,      $3 )
1361 dnl REDUCE_MAXMIN_L(min_max, op_name, cmp)
1362 define(`REDUCE_MAXMIN_L', `
1363 instruct reduce_$1L(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp) %{
1364   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1365             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1366   match(Set dst ($2 src1 src2));
1367   effect(TEMP_DEF dst, TEMP vtmp);


1368   ins_cost(SVE_COST);
1369   format %{ "sve_reduce_$1L $dst, $src1, $src2\t# reduce $1L partial (sve)" %}
1370   ins_encode %{
1371     __ sve_s$1v(as_FloatRegister($vtmp$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
1372     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1373     __ cmp($dst$$Register, $src1$$Register);
1374     __ csel(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($src1$$reg), Assembler::$3);
1375   %}
1376   ins_pipe(pipe_slow);
1377 %}')dnl
1378 dnl
1379 dnl REDUCE_MAXMIN_I_PARTIAL($1,      $2,      $3 )
1380 dnl REDUCE_MAXMIN_I_PARTIAL(min_max, op_mame, cmp)
1381 define(`REDUCE_MAXMIN_I_PARTIAL', `
1382 instruct reduce_$1I_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1383                              pRegGov ptmp, rFlagsReg cr) %{
1384   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1385             (n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
1386              n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
1387              n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT));
1388   match(Set dst ($2 src1 src2));
1389   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1390   ins_cost(SVE_COST);
1391   format %{ "sve_reduce_$1I $dst, $src1, $src2\t# reduce $1I partial (sve)" %}
1392   ins_encode %{
1393     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1394     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1395     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1396                           Matcher::vector_length(this, $src2));
1397     __ sve_s$1v(as_FloatRegister($vtmp$$reg), variant,
1398                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1399     __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1400     __ cmpw($dst$$Register, $src1$$Register);
1401     __ cselw(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($src1$$reg), Assembler::$3);
1402   %}
1403   ins_pipe(pipe_slow);
1404 %}')dnl
1405 dnl
1406 dnl REDUCE_MAXMIN_L_PARTIAL($1,      $2,      $3 )
1407 dnl REDUCE_MAXMIN_L_PARTIAL(min_max, op_name, cmp)
1408 define(`REDUCE_MAXMIN_L_PARTIAL', `
1409 instruct reduce_$1L_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1410                              pRegGov ptmp, rFlagsReg cr) %{
1411   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1412             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1413   match(Set dst ($2 src1 src2));

1414   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1415   ins_cost(SVE_COST);
1416   format %{ "sve_reduce_$1L $dst, $src1, $src2\t# reduce $1L partial (sve)" %}
1417   ins_encode %{
1418     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1419                           Matcher::vector_length(this, $src2));
1420     __ sve_s$1v(as_FloatRegister($vtmp$$reg), __ D,
1421                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1422     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1423     __ cmp($dst$$Register, $src1$$Register);
1424     __ csel(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($src1$$reg), Assembler::$3);
1425   %}
1426   ins_pipe(pipe_slow);
1427 %}')dnl
1428 dnl
1429 dnl REDUCE_FMINMAX($1,      $2,          $3,           $4,   $5         )
1430 dnl REDUCE_FMINMAX(min_max, name_suffix, element_type, size, reg_src_dst)
1431 define(`REDUCE_FMINMAX', `
1432 instruct reduce_$1$2($5 dst, $5 src1, vReg src2) %{
1433   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&

1434             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1435   match(Set dst (translit($1, `m', `M')ReductionV src1 src2));
1436   ins_cost(INSN_COST);
1437   effect(TEMP_DEF dst);
1438   format %{ "sve_f$1v $dst, $src2 # vector (sve) ($4)\n\t"
1439             "f$1s $dst, $dst, $src1\t# $1 reduction $2" %}
1440   ins_encode %{
1441     __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4,
1442          ptrue, as_FloatRegister($src2$$reg));
1443     __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1444   %}
1445   ins_pipe(pipe_slow);
1446 %}')dnl
1447 dnl
1448 dnl
1449 dnl REDUCE_FMINMAX_PARTIAL($1,      $2,          $3,           $4,   $5         )
1450 dnl REDUCE_FMINMAX_PARTIAL(min_max, name_suffix, element_type, size, reg_src_dst)
1451 define(`REDUCE_FMINMAX_PARTIAL', `
1452 instruct reduce_$1$2_partial($5 dst, $5 src1, vReg src2,
1453                              pRegGov ptmp, rFlagsReg cr) %{
1454   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&

1455             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1456   match(Set dst (translit($1, `m', `M')ReductionV src1 src2));
1457   ins_cost(INSN_COST);
1458   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
1459   format %{ "sve_reduce_$1$2 $dst, $src1, $src2\t# reduce $1 $4 partial (sve)" %}
1460   ins_encode %{
1461     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
1462                           Matcher::vector_length(this, $src2));








































1463     __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4,
1464          as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1465     __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1466   %}
1467   ins_pipe(pipe_slow);
1468 %}')dnl
1469 
1470 // vector max reduction
1471 REDUCE_MAXMIN_I(max, MaxReductionV, GT)
1472 REDUCE_MAXMIN_I_PARTIAL(max, MaxReductionV, GT)
1473 REDUCE_MAXMIN_L(max, MaxReductionV, GT)
1474 REDUCE_MAXMIN_L_PARTIAL(max, MaxReductionV, GT)
1475 REDUCE_FMINMAX(max, F, T_FLOAT,  S, vRegF)
1476 REDUCE_FMINMAX_PARTIAL(max, F, T_FLOAT,  S, vRegF)
1477 REDUCE_FMINMAX(max, D, T_DOUBLE, D, vRegD)
1478 REDUCE_FMINMAX_PARTIAL(max, D, T_DOUBLE, D, vRegD)
1479 










1480 // vector min reduction
1481 REDUCE_MAXMIN_I(min, MinReductionV, LT)
1482 REDUCE_MAXMIN_I_PARTIAL(min, MinReductionV, LT)
1483 REDUCE_MAXMIN_L(min, MinReductionV, LT)
1484 REDUCE_MAXMIN_L_PARTIAL(min, MinReductionV, LT)
1485 REDUCE_FMINMAX(min, F, T_FLOAT,  S, vRegF)
1486 REDUCE_FMINMAX_PARTIAL(min, F, T_FLOAT,  S, vRegF)
1487 REDUCE_FMINMAX(min, D, T_DOUBLE, D, vRegD)
1488 REDUCE_FMINMAX_PARTIAL(min, D, T_DOUBLE, D, vRegD)
1489 










1490 // vector Math.rint, floor, ceil
1491 
1492 instruct vroundD(vReg dst, vReg src, immI rmode) %{
1493   predicate(UseSVE > 0 &&
1494             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
1495   match(Set dst (RoundDoubleModeV src rmode));
1496   format %{ "sve_frint $dst, $src, $rmode\t# vector (sve) (D)" %}
1497   ins_encode %{
1498     switch ($rmode$$constant) {
1499       case RoundDoubleModeNode::rmode_rint:
1500         __ sve_frintn(as_FloatRegister($dst$$reg), __ D,
1501              ptrue, as_FloatRegister($src$$reg));
1502         break;
1503       case RoundDoubleModeNode::rmode_floor:
1504         __ sve_frintm(as_FloatRegister($dst$$reg), __ D,
1505              ptrue, as_FloatRegister($src$$reg));
1506         break;
1507       case RoundDoubleModeNode::rmode_ceil:
1508         __ sve_frintp(as_FloatRegister($dst$$reg), __ D,
1509              ptrue, as_FloatRegister($src$$reg));

1647 VSHIFT_TRUE_PREDICATE(vlsrS, URShiftVS, H,  8, sve_lsr)
1648 VSHIFT_TRUE_PREDICATE(vlsrI, URShiftVI, S,  4, sve_lsr)
1649 VSHIFT_TRUE_PREDICATE(vlsrL, URShiftVL, D,  2, sve_lsr)
1650 VSHIFT_IMM_UNPREDICATED(vasrB_imm, RShiftVB,  RShiftCntV, B, 16, sve_asr)
1651 VSHIFT_IMM_UNPREDICATED(vasrS_imm, RShiftVS,  RShiftCntV, H,  8, sve_asr)
1652 VSHIFT_IMM_UNPREDICATED(vasrI_imm, RShiftVI,  RShiftCntV, S,  4, sve_asr)
1653 VSHIFT_IMM_UNPREDICATED(vasrL_imm, RShiftVL,  RShiftCntV, D,  2, sve_asr)
1654 VSHIFT_IMM_UNPREDICATED(vlsrB_imm, URShiftVB, RShiftCntV, B, 16, sve_lsr)
1655 VSHIFT_IMM_UNPREDICATED(vlsrS_imm, URShiftVS, RShiftCntV, H,  8, sve_lsr)
1656 VSHIFT_IMM_UNPREDICATED(vlsrI_imm, URShiftVI, RShiftCntV, S,  4, sve_lsr)
1657 VSHIFT_IMM_UNPREDICATED(vlsrL_imm, URShiftVL, RShiftCntV, D,  2, sve_lsr)
1658 VSHIFT_IMM_UNPREDICATED(vlslB_imm, LShiftVB,  LShiftCntV, B, 16, sve_lsl)
1659 VSHIFT_IMM_UNPREDICATED(vlslS_imm, LShiftVS,  LShiftCntV, H,  8, sve_lsl)
1660 VSHIFT_IMM_UNPREDICATED(vlslI_imm, LShiftVI,  LShiftCntV, S,  4, sve_lsl)
1661 VSHIFT_IMM_UNPREDICATED(vlslL_imm, LShiftVL,  LShiftCntV, D,  2, sve_lsl)
1662 VSHIFT_COUNT(vshiftcntB, B, 16, T_BYTE)
1663 VSHIFT_COUNT(vshiftcntS, H,  8, T_SHORT)
1664 VSHIFT_COUNT(vshiftcntI, S,  4, T_INT)
1665 VSHIFT_COUNT(vshiftcntL, D,  2, T_LONG)
1666 
1667 // vector sqrt
1668 UNARY_OP_TRUE_PREDICATE(vsqrtF, SqrtVF, S, 16, sve_fsqrt)
1669 UNARY_OP_TRUE_PREDICATE(vsqrtD, SqrtVD, D, 16, sve_fsqrt)







































1670 
1671 // vector sub
1672 BINARY_OP_UNPREDICATED(vsubB, SubVB, B, 16, sve_sub)
1673 BINARY_OP_UNPREDICATED(vsubS, SubVS, H, 8, sve_sub)
1674 BINARY_OP_UNPREDICATED(vsubI, SubVI, S, 4, sve_sub)
1675 BINARY_OP_UNPREDICATED(vsubL, SubVL, D, 2, sve_sub)
1676 BINARY_OP_UNPREDICATED(vsubF, SubVF, S, 4, sve_fsub)
1677 BINARY_OP_UNPREDICATED(vsubD, SubVD, D, 2, sve_fsub)
1678 
1679 // vector mask cast


1680 
1681 instruct vmaskcast(vReg dst) %{
1682   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->length() == n->in(1)->bottom_type()->is_vect()->length() &&



















1683             n->bottom_type()->is_vect()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
1684   match(Set dst (VectorMaskCast dst));
1685   ins_cost(0);
1686   format %{ "vmaskcast $dst\t# empty (sve)" %}
1687   ins_encode %{
1688     // empty
1689   %}
1690   ins_pipe(pipe_class_empty);
1691 %}
1692 

































1693 // ------------------------------ Vector cast -------------------------------
1694 dnl
1695 dnl
1696 define(`VECTOR_CAST_EXTEND1', `
1697 instruct vcvt$1to$2`'(vReg dst, vReg src)
1698 %{
1699   predicate(UseSVE > 0 &&
1700             n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1701   match(Set dst (VectorCast$1`'2X src));
1702   ins_cost(SVE_COST);
1703   format %{ "sve_$3  $dst, $4, $src\t# convert $1 to $2 vector" %}
1704   ins_encode %{
1705     __ sve_$3(as_FloatRegister($dst$$reg), __ $4, as_FloatRegister($src$$reg));
1706   %}
1707   ins_pipe(pipe_slow);
1708 %}')dnl

1709 dnl

1710 dnl
1711 define(`VECTOR_CAST_EXTEND2', `
1712 instruct vcvt$1to$2`'(vReg dst, vReg src)
1713 %{
1714   predicate(UseSVE > 0 &&
1715             n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1716   match(Set dst (VectorCast$1`'2X src));
1717   ins_cost(2 * SVE_COST);
1718   format %{ "sve_$3  $dst, $4, $src\n\t"
1719             "sve_$3  $dst, $5, $dst\t# convert $1 to $2 vector" %}
1720   ins_encode %{
1721     __ sve_$3(as_FloatRegister($dst$$reg), __ $4, as_FloatRegister($src$$reg));
1722     __ sve_$3(as_FloatRegister($dst$$reg), __ $5, as_FloatRegister($dst$$reg));




1723   %}
1724   ins_pipe(pipe_slow);
1725 %}')dnl
1726 dnl
1727 dnl
1728 define(`VECTOR_CAST_EXTEND3', `
1729 instruct vcvt$1to$2`'(vReg dst, vReg src)
1730 %{
1731   predicate(UseSVE > 0 &&
1732             n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1733   match(Set dst (VectorCast$1`'2X src));
1734   ins_cost(3 * SVE_COST);
1735   format %{ "sve_$3  $dst, $4, $src\n\t"
1736             "sve_$3  $dst, $5, $dst\n\t"
1737             "sve_$3  $dst, $6, $dst\t# convert $1 to $2 vector" %}
1738   ins_encode %{
1739     __ sve_$3(as_FloatRegister($dst$$reg), __ $4, as_FloatRegister($src$$reg));
1740     __ sve_$3(as_FloatRegister($dst$$reg), __ $5, as_FloatRegister($dst$$reg));
1741     __ sve_$3(as_FloatRegister($dst$$reg), __ $6, as_FloatRegister($dst$$reg));
1742   %}
1743   ins_pipe(pipe_slow);
1744 %}')dnl
1745 dnl
1746 dnl
1747 define(`VECTOR_CAST_NARROW1', `
1748 instruct vcvt$1to$2`'(vReg dst, vReg src, vReg tmp)
1749 %{
1750   predicate(UseSVE > 0 &&
1751             n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1752   match(Set dst (VectorCast$1`'2X src));
1753   effect(TEMP tmp);
1754   ins_cost(2 * SVE_COST);
1755   format %{ "sve_$3  $tmp, $4, 0\n\t"
1756             "sve_$5  $dst, $4, $src, tmp\t# convert $1 to $2 vector" %}
1757   ins_encode %{
1758     __ sve_$3(as_FloatRegister($tmp$$reg), __ $4, 0);
1759     __ sve_$5(as_FloatRegister($dst$$reg), __ $4, as_FloatRegister($src$$reg), as_FloatRegister($tmp$$reg));




1760   %}
1761   ins_pipe(pipe_slow);
1762 %}')dnl
1763 dnl
1764 dnl
1765 define(`VECTOR_CAST_NARROW2', `
1766 instruct vcvt$1to$2`'(vReg dst, vReg src, vReg tmp)
1767 %{
1768   predicate(UseSVE > 0 &&
1769             n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1770   match(Set dst (VectorCast$1`'2X src));
1771   effect(TEMP_DEF dst, TEMP tmp);
1772   ins_cost(3 * SVE_COST);
1773   format %{ "sve_$3  $tmp, $4, 0\n\t"
1774             "sve_$5  $dst, $4, $src, tmp\n\t"
1775             "sve_$5  $dst, $6, $dst, tmp\n\t# convert $1 to $2 vector" %}
1776   ins_encode %{
1777     __ sve_$3(as_FloatRegister($tmp$$reg), __ $4, 0);
1778     __ sve_$5(as_FloatRegister($dst$$reg), __ $4, as_FloatRegister($src$$reg), as_FloatRegister($tmp$$reg));
1779     __ sve_$5(as_FloatRegister($dst$$reg), __ $6, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
1780   %}
1781   ins_pipe(pipe_slow);
1782 %}')dnl
1783 dnl
1784 dnl
1785 define(`VECTOR_CAST_NARROW3', `
1786 instruct vcvt$1to$2`'(vReg dst, vReg src, vReg tmp)
1787 %{
1788   predicate(UseSVE > 0 &&
1789             n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1790   match(Set dst (VectorCast$1`'2X src));
1791   effect(TEMP_DEF dst, TEMP tmp);
1792   ins_cost(4 * SVE_COST);
1793   format %{ "sve_$3  $tmp, $4, 0\n\t"
1794             "sve_$5  $dst, $4, $src, tmp\n\t"
1795             "sve_$5  $dst, $6, $dst, tmp\n\t"
1796             "sve_$5  $dst, $7, $dst, tmp\n\t# convert $1 to $2 vector" %}
1797   ins_encode %{
1798     __ sve_$3(as_FloatRegister($tmp$$reg), __ $4, 0);
1799     __ sve_$5(as_FloatRegister($dst$$reg), __ $4, as_FloatRegister($src$$reg), as_FloatRegister($tmp$$reg));
1800     __ sve_$5(as_FloatRegister($dst$$reg), __ $6, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
1801     __ sve_$5(as_FloatRegister($dst$$reg), __ $7, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
1802   %}
1803   ins_pipe(pipe_slow);
1804 %}')dnl
1805 dnl
1806 dnl
1807 define(`VECTOR_CAST_I2F_EXTEND2', `
1808 instruct vcvt$1to$2`'(vReg dst, vReg src)
1809 %{
1810   predicate(UseSVE > 0 &&
1811             n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1812   match(Set dst (VectorCast$1`'2X src));
1813   ins_cost(3 * SVE_COST);
1814   format %{ "sve_$3  $dst, $4, $src\n\t"
1815             "sve_$3  $dst, $5, $dst\n\t"
1816             "sve_$6  $dst, $5, $dst, $5\t# convert $1 to $2 vector" %}
1817   ins_encode %{
1818     __ sve_$3(as_FloatRegister($dst$$reg), __ $4, as_FloatRegister($src$$reg));
1819     __ sve_$3(as_FloatRegister($dst$$reg), __ $5, as_FloatRegister($dst$$reg));
1820     __ sve_$6(as_FloatRegister($dst$$reg), __ $5, ptrue, as_FloatRegister($dst$$reg), __ $5);
1821   %}
1822   ins_pipe(pipe_slow);
1823 %}')dnl
1824 dnl
1825 dnl
1826 define(`VECTOR_CAST_I2F_EXTEND3', `
1827 instruct vcvt$1to$2`'(vReg dst, vReg src)


1828 %{
1829   predicate(UseSVE > 0 &&
1830             n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1831   match(Set dst (VectorCast$1`'2X src));
1832   ins_cost(4 * SVE_COST);
1833   format %{ "sve_$3  $dst, $4, $src\n\t"
1834             "sve_$3  $dst, $5, $dst\n\t"
1835             "sve_$3  $dst, $6, $dst\n\t"
1836             "sve_$7  $dst, $6, $dst, $6\t# convert $1 to $2 vector" %}
1837   ins_encode %{
1838     __ sve_$3(as_FloatRegister($dst$$reg), __ $4, as_FloatRegister($src$$reg));
1839     __ sve_$3(as_FloatRegister($dst$$reg), __ $5, as_FloatRegister($dst$$reg));
1840     __ sve_$3(as_FloatRegister($dst$$reg), __ $6, as_FloatRegister($dst$$reg));
1841     __ sve_$7(as_FloatRegister($dst$$reg), __ $6, ptrue, as_FloatRegister($dst$$reg), __ $6);
1842   %}
1843   ins_pipe(pipe_slow);
1844 %}')dnl
1845 dnl
1846 dnl
1847 define(`VECTOR_CAST_X2F_NARROW1', `
1848 instruct vcvt$1to$2`'(vReg dst, vReg src, vReg tmp)
1849 %{
1850   predicate(UseSVE > 0 &&
1851             n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1852   match(Set dst (VectorCast$1`'2X src));
1853   effect(TEMP_DEF dst, TEMP tmp);
1854   ins_cost(3 * SVE_COST);
1855   format %{ "sve_$3  $dst, $4, $src, $5\n\t"
1856             "sve_$6  $tmp, $7, 0\n\t"
1857             "sve_$8  $dst, $7, $dst, $tmp\t# convert $1 to $2 vector" %}
1858   ins_encode %{
1859     __ sve_$3(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src$$reg), __ $5);
1860     __ sve_$6(as_FloatRegister($tmp$$reg), __ $7, 0);
1861     __ sve_$8(as_FloatRegister($dst$$reg), __ $7, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));

1862   %}
1863   ins_pipe(pipe_slow);
1864 %}')dnl
1865 dnl
1866 dnl
1867 define(`VECTOR_CAST_X2X', `
1868 instruct vcvt$1to$2`'(vReg dst, vReg src)
1869 %{
1870   predicate(UseSVE > 0 &&
1871             n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1872   match(Set dst (VectorCast$1`'2X src));
1873   ins_cost(SVE_COST);
1874   format %{ "sve_$3  $dst, $4, $src, $4\t# convert $1 to $2 vector" %}

1875   ins_encode %{
1876     __ sve_$3(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src$$reg), __ $4);



1877   %}
1878   ins_pipe(pipe_slow);
1879 %}')dnl
1880 dnl
1881 dnl
1882 define(`VECTOR_CAST_X2F_EXTEND1', `
1883 instruct vcvt$1to$2`'(vReg dst, vReg src)


1884 %{
1885   predicate(UseSVE > 0 &&
1886             n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1887   match(Set dst (VectorCast$1`'2X src));
1888   ins_cost(2 * SVE_COST);
1889   format %{ "sve_$3  $dst, $4, $src\n\t"
1890             "sve_$5  $dst, $4, $dst, $6\t# convert $1 to $2 vector" %}

1891   ins_encode %{
1892     __ sve_$3(as_FloatRegister($dst$$reg), __ $4, as_FloatRegister($src$$reg));
1893     __ sve_$5(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($dst$$reg), __ $6);



1894   %}
1895   ins_pipe(pipe_slow);
1896 %}')dnl
1897 dnl
1898 dnl
1899 define(`VECTOR_CAST_F2X_NARROW1', `
1900 instruct vcvt$1to$2`'(vReg dst, vReg src, vReg tmp)
1901 %{
1902   predicate(UseSVE > 0 &&
1903             n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1904   match(Set dst (VectorCast$1`'2X src));
1905   effect(TEMP_DEF dst, TEMP tmp);
1906   ins_cost(3 * SVE_COST);
1907   format %{ "sve_$3  $dst, $4, $src, $4\n\t"
1908             "sve_$5  $tmp, $6, 0\n\t"
1909             "sve_$7  $dst, $6, $dst, tmp\t# convert $1 to $2 vector" %}
1910   ins_encode %{
1911     __ sve_$3(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src$$reg), __ $4);
1912     __ sve_$5(as_FloatRegister($tmp$$reg), __ $6, 0);
1913     __ sve_$7(as_FloatRegister($dst$$reg), __ $6, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));


1914   %}
1915   ins_pipe(pipe_slow);
1916 %}')dnl
1917 dnl
1918 dnl
1919 define(`VECTOR_CAST_F2X_NARROW2', `
1920 instruct vcvt$1to$2`'(vReg dst, vReg src, vReg tmp)
1921 %{
1922   predicate(UseSVE > 0 &&
1923             n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1924   match(Set dst (VectorCast$1`'2X src));
1925   effect(TEMP_DEF dst, TEMP tmp);
1926   ins_cost(4 * SVE_COST);
1927   format %{ "sve_$3  $dst, $4, $src, $4\n\t"
1928             "sve_$5  $tmp, $6, 0\n\t"
1929             "sve_$7  $dst, $6, $dst, tmp\n\t"
1930             "sve_$7  $dst, $8, $dst, tmp\n\t# convert $1 to $2 vector" %}
1931   ins_encode %{
1932     __ sve_$3(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src$$reg), __ $4);
1933     __ sve_$5(as_FloatRegister($tmp$$reg), __ $6, 0);
1934     __ sve_$7(as_FloatRegister($dst$$reg), __ $6, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
1935     __ sve_$7(as_FloatRegister($dst$$reg), __ $8, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
1936   %}
1937   ins_pipe(pipe_slow);
1938 %}')dnl
1939 dnl
1940 dnl
1941 define(`VECTOR_CAST_F2X_EXTEND1', `
1942 instruct vcvt$1to$2`'(vReg dst, vReg src)
1943 %{
1944   predicate(UseSVE > 0 &&
1945             n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1946   match(Set dst (VectorCast$1`'2X src));
1947   ins_cost(2 * SVE_COST);
1948   format %{ "sve_$3  $dst, $4, $src, $4\n\t"
1949             "sve_$5  $dst, $6, $dst\t# convert $1 to $2 vector" %}


1950   ins_encode %{
1951     __ sve_$3(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src$$reg), __ $4);
1952     __ sve_$5(as_FloatRegister($dst$$reg), __ $6, as_FloatRegister($dst$$reg));



1953   %}
1954   ins_pipe(pipe_slow);
1955 %}')dnl
1956 dnl
1957 dnl
1958 define(`VECTOR_CAST_F2X_NARROW3', `
1959 instruct vcvt$1to$2`'(vReg dst, vReg src, vReg tmp)


1960 %{
1961   predicate(UseSVE > 0 &&
1962             n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1963   match(Set dst (VectorCast$1`'2X src));
1964   effect(TEMP_DEF dst, TEMP tmp);
1965   ins_cost(5 * SVE_COST);
1966   format %{ "sve_$3  $dst, $4, $src, $4\n\t"
1967             "sve_$5  $tmp, $6, 0\n\t"
1968             "sve_$7  $dst, $6, $dst, tmp\n\t"
1969             "sve_$7  $dst, $8, $dst, tmp\n\t"
1970             "sve_$7  $dst, $9, $dst, tmp\n\t# convert $1 to $2 vector" %}
1971   ins_encode %{
1972     __ sve_$3(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src$$reg), __ $4);
1973     __ sve_$5(as_FloatRegister($tmp$$reg), __ $6, 0);
1974     __ sve_$7(as_FloatRegister($dst$$reg), __ $6, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
1975     __ sve_$7(as_FloatRegister($dst$$reg), __ $8, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
1976     __ sve_$7(as_FloatRegister($dst$$reg), __ $9, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
1977   %}
1978   ins_pipe(pipe_slow);
1979 %}')dnl
1980 dnl
1981 VECTOR_CAST_EXTEND1(B, S, sunpklo, H)
1982 VECTOR_CAST_EXTEND2(B, I, sunpklo, H, S)
1983 VECTOR_CAST_EXTEND3(B, L, sunpklo, H, S, D)
1984 VECTOR_CAST_I2F_EXTEND2(B, F, sunpklo, H, S, scvtf)
1985 VECTOR_CAST_I2F_EXTEND3(B, D, sunpklo, H, S, D, scvtf)
1986 dnl
1987 VECTOR_CAST_NARROW1(S, B, dup, B, uzp1)
1988 VECTOR_CAST_EXTEND1(S, I, sunpklo, S)
1989 VECTOR_CAST_EXTEND2(S, L, sunpklo, S, D)
1990 VECTOR_CAST_X2F_EXTEND1(S, F, sunpklo, S, scvtf, S)
1991 VECTOR_CAST_I2F_EXTEND2(S, D, sunpklo, S, D, scvtf)
1992 dnl
1993 VECTOR_CAST_NARROW2(I, B, dup, H, uzp1, B)
1994 VECTOR_CAST_NARROW1(I, S, dup, H, uzp1)
1995 VECTOR_CAST_EXTEND1(I, L, sunpklo, D)
1996 VECTOR_CAST_X2X(I, F, scvtf, S)
1997 VECTOR_CAST_X2F_EXTEND1(I, D, sunpklo, D, scvtf, D)
1998 dnl
1999 VECTOR_CAST_NARROW3(L, B, dup, S, uzp1, H, B)
2000 VECTOR_CAST_NARROW2(L, S, dup, S, uzp1, H)
2001 VECTOR_CAST_NARROW1(L, I, dup, S, uzp1)
2002 VECTOR_CAST_X2F_NARROW1(L, F, scvtf, S, D, dup, S, uzp1)
2003 VECTOR_CAST_X2X(L, D, scvtf, D)
2004 dnl
2005 VECTOR_CAST_F2X_NARROW2(F, B, fcvtzs, S, dup, H, uzp1, B)
2006 VECTOR_CAST_F2X_NARROW1(F, S, fcvtzs, S, dup, H, uzp1)
2007 VECTOR_CAST_X2X(F, I, fcvtzs, S)
2008 VECTOR_CAST_F2X_EXTEND1(F, L, fcvtzs, S, sunpklo, D)
2009 VECTOR_CAST_X2F_EXTEND1(F, D, sunpklo, D, fcvt, S)
2010 dnl
2011 VECTOR_CAST_F2X_NARROW3(D, B, fcvtzs, D, dup, S, uzp1, H, B)
2012 VECTOR_CAST_F2X_NARROW2(D, S, fcvtzs, D, dup, S, uzp1, H)
2013 VECTOR_CAST_F2X_NARROW1(D, I, fcvtzs, D, dup, S, uzp1)
2014 VECTOR_CAST_X2X(D, L, fcvtzs, D)
2015 VECTOR_CAST_X2F_NARROW1(D, F, fcvt, S, D, dup, S, uzp1)
2016 dnl
2017 dnl
2018 // ------------------------------ Vector extract ---------------------------------
2019 define(`VECTOR_EXTRACT_SXT', `
2020 instruct extract$1`'($2 dst, vReg src, immI idx, pRegGov pTmp, rFlagsReg cr)
2021 %{
2022   predicate(UseSVE > 0);
2023   match(Set dst (Extract$1 src idx));
2024   effect(TEMP pTmp, KILL cr);
2025   ins_cost(2 * SVE_COST);
2026   format %{ "sve_extract $dst, $3, $pTmp, $src, $idx\n\t"
2027             "sbfmw $dst, $dst, 0U, $5\t# extract from vector($1)" %}
2028   ins_encode %{
2029     __ sve_extract(as_$4($dst$$reg), __ $3, as_PRegister($pTmp$$reg),
2030                    as_FloatRegister($src$$reg), (int)($idx$$constant));
2031     __ sbfmw(as_$4($dst$$reg), as_$4($dst$$reg), 0U, $5);
2032   %}
2033   ins_pipe(pipe_slow);
2034 %}')dnl
2035 dnl                $1 $2         $3 $4        $5
2036 VECTOR_EXTRACT_SXT(B, iRegINoSp, B, Register, 7U)
2037 VECTOR_EXTRACT_SXT(S, iRegINoSp, H, Register, 15U)
2038 
2039 dnl
2040 define(`VECTOR_EXTRACT', `
2041 instruct extract$1`'($2 dst, vReg src, immI idx, pRegGov pTmp, rFlagsReg cr)
2042 %{
2043   predicate(UseSVE > 0);
2044   match(Set dst (Extract$1 src idx));
2045   effect(TEMP pTmp, KILL cr);
2046   ins_cost(2 * SVE_COST);
2047   format %{ "sve_extract $dst, $3, $pTmp, $src, $idx\t# extract from vector($1)" %}
2048   ins_encode %{
2049     __ sve_extract(as_$4($dst$$reg), __ $3, as_PRegister($pTmp$$reg),
2050                    as_FloatRegister($src$$reg), (int)($idx$$constant));
2051   %}
2052   ins_pipe(pipe_slow);
2053 %}')dnl
2054 dnl            $1 $2         $3 $4
2055 VECTOR_EXTRACT(I, iRegINoSp, S, Register)
2056 VECTOR_EXTRACT(L, iRegLNoSp, D, Register)
2057 VECTOR_EXTRACT(F, vRegF,     S, FloatRegister)
2058 VECTOR_EXTRACT(D, vRegD,     D, FloatRegister)
2059 
2060 // ------------------------------- VectorTest ----------------------------------
2061 dnl
2062 dnl VTEST($1,      $2,   $3,  $4  )
2063 dnl VTEST(op_name, pred, imm, cond)
2064 define(`VTEST', `
2065 instruct vtest_$1`'(iRegINoSp dst, vReg src1, vReg src2, pReg pTmp, rFlagsReg cr)
2066 %{
2067   predicate(UseSVE > 0 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2068             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::$2);



















2069   match(Set dst (VectorTest src1 src2));
2070   effect(TEMP pTmp, KILL cr);
2071   ins_cost(SVE_COST);
2072   format %{ "sve_cmpeq $pTmp, $src1, $3\n\t"
2073             "csetw $dst, $4\t# VectorTest (sve) - $1" %}
2074   ins_encode %{
2075     // "src2" is not used for sve.
2076     BasicType bt = Matcher::vector_element_basic_type(this, $src1);
2077     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2078     __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), size,
2079                ptrue, as_FloatRegister($src1$$reg), $3);
2080     __ csetw(as_Register($dst$$reg), Assembler::$4);
2081   %}
2082   ins_pipe(pipe_slow);
2083 %}')dnl
2084 dnl
2085 VTEST(alltrue, overflow, 0, EQ)
2086 VTEST(anytrue, ne,      -1, NE)
2087 dnl
2088 dnl
2089 dnl VTEST_PARTIAL($1,      $2,   $3,  $4  )
2090 dnl VTEST_PARTIAL(op_name, pred, imm, cond)
2091 define(`VTEST_PARTIAL', `
2092 instruct vtest_$1_partial`'(iRegINoSp dst, vReg src1, vReg src2, pRegGov pTmp, rFlagsReg cr)
2093 %{
2094   predicate(UseSVE > 0 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&

2095             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::$2);
2096   match(Set dst (VectorTest src1 src2));
2097   effect(TEMP pTmp, KILL cr);
2098   ins_cost(SVE_COST);
2099   format %{ "vtest_$1_partial $dst, $src1, $src2\t# VectorTest partial (sve) - $1" %}
2100   ins_encode %{
2101     // "src2" is not used for sve.
2102     BasicType bt = Matcher::vector_element_basic_type(this, $src1);
2103     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2104     __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), size,
2105                           Matcher::vector_length(this, $src1));
2106     __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), size,
2107                as_PRegister($pTmp$$reg), as_FloatRegister($src1$$reg), $3);
2108     __ csetw(as_Register($dst$$reg), Assembler::$4);
2109   %}
2110   ins_pipe(pipe_slow);
2111 %}')dnl
2112 dnl
2113 VTEST_PARTIAL(alltrue, overflow, 0, EQ)
2114 VTEST_PARTIAL(anytrue, ne,      -1, NE)
2115 
2116 // ------------------------------ Vector insert ---------------------------------
2117 
2118 instruct insertI_small(vReg dst, vReg src, iRegIorL2I val, immI idx, pRegGov pTmp, rFlagsReg cr)
2119 %{
2120   predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
2121             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2122              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2123              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2124   match(Set dst (VectorInsert (Binary src val) idx));
2125   effect(TEMP_DEF dst, TEMP pTmp, KILL cr);
2126   ins_cost(4 * SVE_COST);
2127   format %{ "sve_index $dst, -16, 1\t# (B/S/I)\n\t"
2128             "sve_cmpeq $pTmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2129             "sve_orr $dst, $src, $src\n\t"
2130             "sve_cpy $dst, $pTmp, $val\t# insert into vector (B/S/I)" %}
2131   ins_encode %{
2132     BasicType bt = Matcher::vector_element_basic_type(this, $src);
2133     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2134     __ sve_index(as_FloatRegister($dst$$reg), size, -16, 1);
2135     __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), size, ptrue,
2136                as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2137     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2138     __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pTmp$$reg), as_Register($val$$reg));
2139   %}
2140   ins_pipe(pipe_slow);
2141 %}
2142 
2143 instruct insertF_small(vReg dst, vReg src, vRegF val, immI idx, pRegGov pTmp, rFlagsReg cr)
2144 %{
2145   predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
2146             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2147   match(Set dst (VectorInsert (Binary src val) idx));
2148   effect(TEMP_DEF dst, TEMP pTmp, KILL cr);
2149   ins_cost(4 * SVE_COST);
2150   format %{ "sve_index $dst, S, -16, 1\n\t"
2151             "sve_cmpeq $pTmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2152             "sve_orr $dst, $src, $src\n\t"
2153             "sve_cpy $dst, $pTmp, $val\t# insert into vector (F)" %}
2154   ins_encode %{
2155     __ sve_index(as_FloatRegister($dst$$reg), __ S, -16, 1);
2156     __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), __ S, ptrue,
2157                as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2158     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2159     __ sve_cpy(as_FloatRegister($dst$$reg), __ S, as_PRegister($pTmp$$reg), as_FloatRegister($val$$reg));
2160   %}
2161   ins_pipe(pipe_slow);
2162 %}
2163 
2164 instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp1, pRegGov pTmp, rFlagsReg cr)
2165 %{
2166   predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
2167             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2168              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2169              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2170   match(Set dst (VectorInsert (Binary src val) idx));
2171   effect(TEMP_DEF dst, TEMP tmp1, TEMP pTmp, KILL cr);
2172   ins_cost(5 * SVE_COST);
2173   format %{ "sve_index $tmp1, 0, 1\t# (B/S/I)\n\t"
2174             "sve_dup $dst, $idx\t# (B/S/I)\n\t"
2175             "sve_cmpeq $pTmp, $tmp1, $dst\n\t"
2176             "sve_orr $dst, $src, $src\n\t"
2177             "sve_cpy $dst, $pTmp, $val\t# insert into vector (B/S/I)" %}
2178   ins_encode %{
2179     BasicType bt = Matcher::vector_element_basic_type(this, $src);
2180     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2181     __ sve_index(as_FloatRegister($tmp1$$reg), size, 0, 1);
2182     __ sve_dup(as_FloatRegister($dst$$reg), size, (int)($idx$$constant));
2183     __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), size, ptrue,
2184                as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
2185     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2186     __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pTmp$$reg), as_Register($val$$reg));
2187   %}
2188   ins_pipe(pipe_slow);
2189 %}
2190 dnl
2191 dnl
2192 define(`VECTOR_INSERT_D', `
2193 instruct insert$1`'(vReg dst, vReg src, $2 val, immI idx, pRegGov pTmp, rFlagsReg cr)
2194 %{
2195   predicate(UseSVE > 0 &&
2196             n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($1));
2197   match(Set dst (VectorInsert (Binary src val) idx));
2198   effect(TEMP_DEF dst, TEMP pTmp, KILL cr);
2199   ins_cost(4 * SVE_COST);
2200   format %{ "sve_index $dst, $3, -16, 1\n\t"
2201             "sve_cmpeq $pTmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2202             "sve_orr $dst, $src, $src\n\t"
2203             "sve_cpy $dst, $pTmp, $val\t# insert into vector ($1)" %}
2204   ins_encode %{
2205     __ sve_index(as_FloatRegister($dst$$reg), __ $3, -16, 1);
2206     __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), __ $3, ptrue,
2207                as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2208     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2209     __ sve_cpy(as_FloatRegister($dst$$reg), __ $3, as_PRegister($pTmp$$reg), as_$4($val$$reg));
2210   %}
2211   ins_pipe(pipe_slow);
2212 %}')dnl
2213 dnl             $1 $2     $3 $4
2214 VECTOR_INSERT_D(L, iRegL, D, Register)
2215 VECTOR_INSERT_D(D, vRegD, D, FloatRegister)
2216 
2217 instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pTmp, rFlagsReg cr)
2218 %{
2219   predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
2220             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2221   match(Set dst (VectorInsert (Binary src val) idx));
2222   effect(TEMP_DEF dst, TEMP tmp1, TEMP pTmp, KILL cr);
2223   ins_cost(5 * SVE_COST);
2224   format %{ "sve_index $tmp1, S, 0, 1\n\t"
2225             "sve_dup $dst, S, $idx\n\t"
2226             "sve_cmpeq $pTmp, $tmp1, $dst\n\t"
2227             "sve_orr $dst, $src, $src\n\t"
2228             "sve_cpy $dst, $pTmp, $val\t# insert into vector (F)" %}
2229   ins_encode %{
2230     __ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
2231     __ sve_dup(as_FloatRegister($dst$$reg), __ S, (int)($idx$$constant));
2232     __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), __ S, ptrue,
2233                as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
2234     __ sve_orr(as_FloatRegister($dst$$reg),
2235                as_FloatRegister($src$$reg),
2236                as_FloatRegister($src$$reg));
2237     __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
2238                as_PRegister($pTmp$$reg), as_FloatRegister($val$$reg));
2239   %}
2240   ins_pipe(pipe_slow);
2241 %}
2242 
2243 // ------------------------------ Vector shuffle -------------------------------
2244 
2245 instruct loadshuffleB(vReg dst, vReg src)
2246 %{
2247   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2248   match(Set dst (VectorLoadShuffle src));
2249   ins_cost(SVE_COST);
2250   format %{ "sve_orr $dst, $src, $src\t# vector load shuffle (B)" %}
2251   ins_encode %{
2252     if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
2253       __ sve_orr(as_FloatRegister($dst$$reg),
2254                  as_FloatRegister($src$$reg),
2255                  as_FloatRegister($src$$reg));





2256     }
2257   %}
2258   ins_pipe(pipe_slow);
2259 %}
2260 
2261 instruct loadshuffleS(vReg dst, vReg src)
2262 %{
2263   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
2264   match(Set dst (VectorLoadShuffle src));
2265   ins_cost(SVE_COST);
2266   format %{ "sve_uunpklo $dst, $src\t# vector load shuffle (B to H)" %}
2267   ins_encode %{
2268     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
2269   %}
2270   ins_pipe(pipe_slow);
2271 %}
2272 
2273 instruct loadshuffleI(vReg dst, vReg src)
2274 %{
2275   predicate(UseSVE > 0 &&
2276            (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2277             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2278   match(Set dst (VectorLoadShuffle src));
2279   ins_cost(2 * SVE_COST);
2280   format %{ "sve_uunpklo $dst, H, $src\n\t"
2281             "sve_uunpklo $dst, S, $dst\t# vector load shuffle (B to S)" %}
2282   ins_encode %{
2283     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
2284     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($dst$$reg));
2285   %}
2286   ins_pipe(pipe_slow);
2287 %}
2288 
2289 instruct loadshuffleL(vReg dst, vReg src)
2290 %{
2291   predicate(UseSVE > 0 &&
2292            (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2293             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2294   match(Set dst (VectorLoadShuffle src));
2295   ins_cost(3 * SVE_COST);
2296   format %{ "sve_uunpklo $dst, H, $src\n\t"
2297             "sve_uunpklo $dst, S, $dst\n\t"
2298             "sve_uunpklo $dst, D, $dst\t# vector load shuffle (B to D)" %}
2299   ins_encode %{
2300     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
2301     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($dst$$reg));
2302     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($dst$$reg));
2303   %}
2304   ins_pipe(pipe_slow);
2305 %}
2306 
2307 // ------------------------------ Vector rearrange -------------------------------
2308 
2309 instruct rearrange(vReg dst, vReg src, vReg shuffle)
2310 %{
2311   predicate(UseSVE > 0);
2312   match(Set dst (VectorRearrange src shuffle));
2313   ins_cost(SVE_COST);
2314   format %{ "sve_tbl $dst, $src, $shuffle\t# vector rearrange" %}
2315   ins_encode %{
2316     BasicType bt = Matcher::vector_element_basic_type(this, $src);
2317     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2318     __ sve_tbl(as_FloatRegister($dst$$reg), size,
2319                as_FloatRegister($src$$reg), as_FloatRegister($shuffle$$reg));
2320   %}
2321   ins_pipe(pipe_slow);
2322 %}
2323 
2324 // ------------------------------ Vector Load Gather ---------------------------------
2325 
2326 instruct gatherI(vReg dst, indirect mem, vReg idx) %{
2327   predicate(UseSVE > 0 &&
2328             n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
2329             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2330              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2331   match(Set dst (LoadVectorGather mem idx));
2332   ins_cost(SVE_COST);
2333   format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (I/F)" %}
2334   ins_encode %{
2335     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), ptrue,
2336                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2337   %}
2338   ins_pipe(pipe_slow);
2339 %}
2340 
2341 instruct gatherL(vReg dst, indirect mem, vReg idx) %{
2342   predicate(UseSVE > 0 &&
2343             n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
2344             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2345              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2346   match(Set dst (LoadVectorGather mem idx));
2347   ins_cost(2 * SVE_COST);
2348   format %{ "sve_uunpklo $idx, $idx\n\t"
2349             "load_vector_gather $dst, $mem, $idx\t# vector load gather (L/D)" %}
2350   ins_encode %{
2351     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2352     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), ptrue, as_Register($mem$$base), as_FloatRegister($idx$$reg));

2353   %}
2354   ins_pipe(pipe_slow);
2355 %}
2356 
2357 // ------------------------------ Vector Load Gather Partial-------------------------------
2358 
2359 instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov pTmp, rFlagsReg cr) %{
2360   predicate(UseSVE > 0 &&
2361             n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
2362             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2363              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2364   match(Set dst (LoadVectorGather mem idx));
2365   effect(TEMP pTmp, KILL cr);
2366   ins_cost(2 * SVE_COST + INSN_COST);
2367   format %{ "sve_whilelo_zr_imm $pTmp, vector_length\n\t"
2368             "load_vector_gather $dst, $pTmp, $mem, $idx\t# vector load gather partial (I/F)" %}
2369   ins_encode %{
2370     __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), __ S,
2371                           Matcher::vector_length(this));
2372     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($pTmp$$reg),
2373                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2374   %}
2375   ins_pipe(pipe_slow);
2376 %}
2377 
2378 instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov pTmp, rFlagsReg cr) %{
2379   predicate(UseSVE > 0 &&
2380             n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
2381             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2382              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2383   match(Set dst (LoadVectorGather mem idx));
2384   effect(TEMP pTmp, KILL cr);
2385   ins_cost(3 * SVE_COST + INSN_COST);
2386   format %{ "sve_whilelo_zr_imm $pTmp, vector_length\n\t"
2387             "sve_uunpklo $idx, $idx\n\t"
2388             "load_vector_gather $dst, $pTmp, $mem, $idx\t# vector load gather partial (L/D)" %}




















































2389   ins_encode %{
2390     __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), __ D,
2391                           Matcher::vector_length(this));





















2392     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2393     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($pTmp$$reg),
2394                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2395   %}
2396   ins_pipe(pipe_slow);
2397 %}
2398 
2399 // ------------------------------ Vector Store Scatter -------------------------------
2400 
2401 instruct scatterI(indirect mem, vReg src, vReg idx) %{
2402   predicate(UseSVE > 0 &&
2403             n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
2404             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2405              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2406   match(Set mem (StoreVectorScatter mem (Binary src idx)));
2407   ins_cost(SVE_COST);
2408   format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (I/F)" %}
2409   ins_encode %{
2410     __ sve_st1w_scatter(as_FloatRegister($src$$reg), ptrue,
2411                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2412   %}
2413   ins_pipe(pipe_slow);
2414 %}
2415 
2416 instruct scatterL(indirect mem, vReg src, vReg idx) %{
2417   predicate(UseSVE > 0 &&
2418             n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
2419             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2420              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2421   match(Set mem (StoreVectorScatter mem (Binary src idx)));
2422   ins_cost(2 * SVE_COST);
2423   format %{ "sve_uunpklo $idx, $idx\n\t"
2424             "store_vector_scatter $mem, $idx, $src\t# vector store scatter (L/D)" %}
2425   ins_encode %{
2426     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D,
2427                    as_FloatRegister($idx$$reg));
2428     __ sve_st1d_scatter(as_FloatRegister($src$$reg), ptrue,
2429                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2430   %}
2431   ins_pipe(pipe_slow);
2432 %}
2433 
2434 // ------------------------------ Vector Store Scatter Partial-------------------------------
2435 
2436 instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov pTmp, rFlagsReg cr) %{
2437   predicate(UseSVE > 0 &&
2438             n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
2439             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2440              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2441   match(Set mem (StoreVectorScatter mem (Binary src idx)));
2442   effect(TEMP pTmp, KILL cr);
2443   ins_cost(2 * SVE_COST + INSN_COST);
2444   format %{ "sve_whilelo_zr_imm $pTmp, vector_length\n\t"
2445             "store_vector_scatter $mem, $pTmp, $idx, $src\t# vector store scatter partial (I/F)" %}
2446   ins_encode %{
2447     __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), __ S,
2448                           Matcher::vector_length(this, $src));
2449     __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($pTmp$$reg),
2450                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2451   %}
2452   ins_pipe(pipe_slow);
2453 %}
2454 
2455 instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov pTmp, rFlagsReg cr) %{
2456   predicate(UseSVE > 0 &&
2457             n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
2458             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2459              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2460   match(Set mem (StoreVectorScatter mem (Binary src idx)));
2461   effect(TEMP pTmp, KILL cr);
2462   ins_cost(3 * SVE_COST + INSN_COST);
2463   format %{ "sve_whilelo_zr_imm $pTmp, vector_length\n\t"
2464             "sve_uunpklo $idx, $idx\n\t"
2465             "store_vector_scatter $mem, $pTmp, $idx, $src\t# vector store scatter partial (L/D)" %}
2466   ins_encode %{
2467     __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), __ D,
2468                           Matcher::vector_length(this, $src));
2469     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2470     __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($pTmp$$reg),























































2471                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2472   %}
2473   ins_pipe(pipe_slow);
2474 %}
2475 




















2476 
2477 // ------------------------------ Vector Load Const -------------------------------
2478 
2479 instruct loadconB(vReg dst, immI0 src) %{
2480   predicate(UseSVE > 0 &&
2481             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2482   match(Set dst (VectorLoadConst src));
2483   ins_cost(SVE_COST);
2484   format %{ "sve_index $dst, 0, 1\t# generate iota indices" %}
2485   ins_encode %{
2486     __ sve_index(as_FloatRegister($dst$$reg), __ B, 0, 1);
2487   %}
2488   ins_pipe(pipe_slow);
2489 %}
2490 
2491 // Intrisics for String.indexOf(char)
2492 
2493 dnl
2494 define(`STRING_INDEXOF_CHAR', `
2495 instruct string$1_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
2496                                   iRegI_R0 result, vReg ztmp1, vReg ztmp2,
2497                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
2498 %{
2499   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
2500   predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::$1));
2501   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
2502 
2503   format %{ "String$2 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
2504 
2505   ins_encode %{
2506     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
2507                                as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
2508                                as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), $3 /* isL */);
2509   %}
2510   ins_pipe(pipe_class_memory);
2511 %}')dnl
2512 dnl                 $1 $2      $3
2513 STRING_INDEXOF_CHAR(L, Latin1, true)
2514 STRING_INDEXOF_CHAR(U, UTF16,  false)
2515 
2516 dnl
2517 dnl VMASK_REDUCTION($1,     $2,      $3  )
2518 dnl VMASK_REDUCTION(suffix, op_name, cost)
2519 define(`VMASK_REDUCTION', `
2520 instruct vmask_$1(iRegINoSp dst, vReg src, pReg ptmp, rFlagsReg cr) %{
2521   predicate(UseSVE > 0 &&
2522             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2523   match(Set dst ($2 src));
2524   effect(TEMP ptmp, KILL cr);
2525   ins_cost($3 * SVE_COST);
2526   format %{ "vmask_$1 $dst, $src\t# vector mask $1 (sve)" %}
2527   ins_encode %{
2528     __ sve_vmask_reduction(this->ideal_Opcode(), $dst$$Register, __ B,
2529                            as_FloatRegister($src$$reg), ptrue, as_PRegister($ptmp$$reg));

2530   %}
2531   ins_pipe(pipe_slow);
2532 %}')dnl
2533 dnl
2534 // ---------------------------- Vector mask reductions ---------------------------
2535 VMASK_REDUCTION(truecount, VectorMaskTrueCount, 2)
2536 VMASK_REDUCTION(firsttrue, VectorMaskFirstTrue, 3)
2537 VMASK_REDUCTION(lasttrue,  VectorMaskLastTrue, 4)
2538 dnl
2539 dnl VMASK_REDUCTION_PARTIAL($1,     $2,      $3  )
2540 dnl VMASK_REDUCTION_PARTIAL(suffix, op_name, cost)
2541 define(`VMASK_REDUCTION_PARTIAL', `
2542 instruct vmask_$1_partial(iRegINoSp dst, vReg src, pRegGov ifelse($1, `firsttrue', `pgtmp, pReg ptmp', `ptmp'), rFlagsReg cr) %{
2543   predicate(UseSVE > 0 &&
2544             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2545   match(Set dst ($2 src));
2546   effect(TEMP ifelse($1, `firsttrue', `pgtmp, TEMP ptmp', `ptmp'), KILL cr);
2547   ins_cost($3 * SVE_COST);
2548   format %{ "vmask_$1 $dst, $src\t# vector mask $1 partial (sve)" %}
2549   ins_encode %{
2550     __ sve_whilelo_zr_imm(as_PRegister(ifelse($1, `firsttrue', `$pgtmp', `$ptmp')$$reg), __ B,
2551                           Matcher::vector_length(this, $src));
2552     __ sve_vmask_reduction(this->ideal_Opcode(), $dst$$Register, __ B, as_FloatRegister($src$$reg),
2553                            as_PRegister(ifelse($1, `firsttrue', `$pgtmp', `$ptmp')$$reg), as_PRegister($ptmp$$reg));
2554   %}
2555   ins_pipe(pipe_slow);
2556 %}')dnl
2557 dnl
2558 VMASK_REDUCTION_PARTIAL(truecount, VectorMaskTrueCount, 3)
2559 VMASK_REDUCTION_PARTIAL(firsttrue, VectorMaskFirstTrue, 4)
2560 VMASK_REDUCTION_PARTIAL(lasttrue,  VectorMaskLastTrue, 5)
2561 
2562 dnl
2563 dnl VSTOREMASK_REDUCTION($1,     $2,      $3  )
2564 dnl VSTOREMASK_REDUCTION(suffix, op_name, cost)
2565 define(`VSTOREMASK_REDUCTION', `
2566 instruct vstoremask_$1(iRegINoSp dst, vReg src, immI esize, pReg ptmp, rFlagsReg cr) %{
2567   predicate(UseSVE > 0 &&
2568             n->in(1)->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2569   match(Set dst ($2 (VectorStoreMask src esize)));
2570   effect(TEMP ptmp, KILL cr);
2571   ins_cost($3 * SVE_COST);
2572   format %{ "vstoremask_$1 $dst, $src\t# vector mask $1 (sve)" %}
2573   ins_encode %{
2574     unsigned size = $esize$$constant;
2575     assert(size == 1 || size == 2 || size == 4 || size == 8, "unsupported element size");
2576     Assembler::SIMD_RegVariant variant = __ elemBytes_to_regVariant(size);
2577     __ sve_vmask_reduction(this->ideal_Opcode(), $dst$$Register, variant, as_FloatRegister($src$$reg),
2578                            ptrue, as_PRegister($ptmp$$reg), Matcher::vector_length(this, $src));
2579   %}
2580   ins_pipe(pipe_slow);
2581 %}')dnl
2582 dnl
2583 // ----------------- Vector mask reductions combined with VectorMaskStore ---------------
2584 VSTOREMASK_REDUCTION(truecount, VectorMaskTrueCount, 2)
2585 VSTOREMASK_REDUCTION(firsttrue, VectorMaskFirstTrue, 3)
2586 VSTOREMASK_REDUCTION(lasttrue,  VectorMaskLastTrue, 4)
2587 dnl
2588 dnl VSTOREMASK_REDUCTION_PARTIAL($1,     $2,      $3  )
2589 dnl VSTOREMASK_REDUCTION_PARTIAL(suffix, op_name, cost)
2590 define(`VSTOREMASK_REDUCTION_PARTIAL', `
2591 instruct vstoremask_$1_partial(iRegINoSp dst, vReg src, immI esize, pRegGov ifelse($1, `firsttrue', `pgtmp, pReg ptmp', `ptmp'), rFlagsReg cr) %{
2592   predicate(UseSVE > 0 &&
2593             n->in(1)->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2594   match(Set dst ($2 (VectorStoreMask src esize)));
2595   effect(TEMP ifelse($1, `firsttrue', `pgtmp, TEMP ptmp', `ptmp'), KILL cr);
2596   ins_cost($3 * SVE_COST);
2597   format %{ "vstoremask_$1 $dst, $src\t# vector mask $1 partial (sve)" %}
2598   ins_encode %{
2599     unsigned size = $esize$$constant;
2600     assert(size == 1 || size == 2 || size == 4 || size == 8, "unsupported element size");
2601     Assembler::SIMD_RegVariant variant = __ elemBytes_to_regVariant(size);
2602     __ sve_whilelo_zr_imm(as_PRegister(ifelse($1, `firsttrue', `$pgtmp', `$ptmp')$$reg), variant,







2603                           Matcher::vector_length(this, $src));
2604     __ sve_vmask_reduction(this->ideal_Opcode(), $dst$$Register, variant, as_FloatRegister($src$$reg),
2605                            as_PRegister(ifelse($1, `firsttrue', `$pgtmp', `$ptmp')$$reg), as_PRegister($ptmp$$reg), MaxVectorSize / size);
2606   %}
2607   ins_pipe(pipe_slow);
2608 %}')dnl
2609 dnl
2610 VSTOREMASK_REDUCTION_PARTIAL(truecount, VectorMaskTrueCount, 3)
2611 VSTOREMASK_REDUCTION_PARTIAL(firsttrue, VectorMaskFirstTrue, 4)
2612 VSTOREMASK_REDUCTION_PARTIAL(lasttrue,  VectorMaskLastTrue, 5)














  66   constraint(ALLOC_IN_RC(ptr_reg));
  67   match(AddP reg off);
  68   op_cost(0);
  69   format %{ "[$reg, $off]" %}
  70   interface(MEMORY_INTER) %{
  71     base($reg);
  72     `index'(0xffffffff);
  73     scale(0x0);
  74     disp($off);
  75   %}
  76 %}')dnl
  77 OPERAND_VMEMORYA_INDIRECT_OFFSET(I, 4)
  78 OPERAND_VMEMORYA_INDIRECT_OFFSET(L, 4)
  79 
  80 // The indOff of vmemA is valid only when the vector element (load to/store from)
  81 // size equals to memory element (load from/store to) size.
  82 opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);
  83 
  84 source_hpp %{
  85   bool op_sve_supported(int opcode, int vlen, BasicType bt);
  86   bool masked_op_sve_supported(int opcode, int vlen, BasicType bt);
  87 %}
  88 
  89 source %{
  90 
  91   typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
  92                                                              PRegister Pg, const Address &adr);
  93 
  94   // Predicated load/store, with optional ptrue to all elements of given predicate register.
  95   static void loadStoreA_predicated(C2_MacroAssembler masm, bool is_store, FloatRegister reg,
  96                                     PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
  97                                     int opcode, Register base, int index, int size, int disp) {
  98     sve_mem_insn_predicate insn;
  99     int mesize = type2aelembytes(mem_elem_bt);
 100     if (index == -1) {
 101       assert(size == 0, "unsupported address mode: scale size = %d", size);
 102       switch(mesize) {
 103       case 1:
 104         insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
 105         break;
 106       case 2:

 123       ShouldNotReachHere();
 124     }
 125   }
 126 
 127   bool op_sve_supported(int opcode, int vlen, BasicType bt) {
 128     int length_in_bytes = vlen * type2aelembytes(bt);
 129     switch (opcode) {
 130       case Op_MulAddVS2VI:
 131       // No multiply reduction instructions
 132       case Op_MulReductionVD:
 133       case Op_MulReductionVF:
 134       case Op_MulReductionVI:
 135       case Op_MulReductionVL:
 136       // Others
 137       case Op_ExtractC:
 138       case Op_ExtractUB:
 139         return false;
 140       // Vector API specific
 141       case Op_VectorLoadShuffle:
 142       case Op_VectorRearrange:
 143         return vlen >= 4 && length_in_bytes <= MaxVectorSize;




 144       case Op_LoadVector:
 145       case Op_StoreVector:
 146         return Matcher::vector_size_supported(bt, vlen);
 147       default:
 148         break;
 149     }
 150     // By default, we only support vector operations with no less than 8 bytes and 2 elements.
 151     return 8 <= length_in_bytes && length_in_bytes <= MaxVectorSize && vlen >= 2;
 152   }
 153 
 154   bool masked_op_sve_supported(int opcode, int vlen, BasicType bt) {
 155     if (opcode == Op_VectorRearrange) {
 156       return false;
 157     }
 158     return op_sve_supported(opcode, vlen, bt);
 159   }
 160 
 161 %}
 162 
 163 definitions %{
 164   int_def SVE_COST             (200, 200);
 165 %}
 166 
 167 dnl
 168 dnl ELEMENT_SHORT_CHART($1, $2)
 169 dnl ELEMENT_SHORT_CHART(etype, node)
 170 define(`ELEMENT_SHORT_CHAR',`ifelse(`$1', `T_SHORT',
 171   `($2->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
 172             ($2->bottom_type()->is_vect()->element_basic_type() == T_CHAR))',
 173    `($2->bottom_type()->is_vect()->element_basic_type() == $1)')')dnl
 174 dnl
 175 
 176 // All SVE instructions
 177 
 178 // vector load/store
 179 
 180 // Unpredicated vector load/store

 219   match(Set ifelse(load, $3, dst (LoadVector mem), mem (StoreVector mem src)));
 220   ins_cost(4 * INSN_COST);
 221   format %{ "$1   ifelse(load, $3, `$dst,$mem', `$mem,$src')\t# vector ($6 bits)" %}
 222   ins_encode( `aarch64_enc_'ifelse(load, $3, ldr, str)v$2($7, mem) );
 223   ins_pipe(v$3`_reg_mem'ifelse(eval($4 * 8), 128, 128, 64));
 224 %}')dnl
 225 dnl        $1    $2 $3     $4  $5 $6   $7
 226 VLoadStore(ldrh, H, load,  2,  D, 16,  dst)
 227 VLoadStore(strh, H, store, 2,  D, 16,  src)
 228 VLoadStore(ldrs, S, load,  4,  D, 32,  dst)
 229 VLoadStore(strs, S, store, 4,  D, 32,  src)
 230 VLoadStore(ldrd, D, load,  8,  D, 64,  dst)
 231 VLoadStore(strd, D, store, 8,  D, 64,  src)
 232 VLoadStore(ldrq, Q, load, 16,  X, 128, dst)
 233 VLoadStore(strq, Q, store, 16, X, 128, src)
 234 
 235 // Predicated vector load/store, based on the vector length of the node.
 236 // Only load/store values in the range of the memory_size. This is needed
 237 // when the memory_size is lower than the hardware supported max vector size.
 238 // And this might happen for Vector API mask vector load/store.
 239 instruct loadV_partial(vReg dst, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
 240   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() > 16 &&
 241             n->as_LoadVector()->memory_size() < MaxVectorSize);
 242   match(Set dst (LoadVector mem));
 243   effect(TEMP pgtmp, KILL cr);
 244   ins_cost(6 * SVE_COST);
 245   format %{ "sve_whilelo_zr_imm $pgtmp, vector_length\n\t"
 246             "sve_ldr $dst, $pgtmp, $mem\t# load vector partial" %}
 247   ins_encode %{
 248     BasicType bt = Matcher::vector_element_basic_type(this);
 249     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 250                           Matcher::vector_length(this));
 251     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 252     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg,
 253                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 254                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 255   %}
 256   ins_pipe(pipe_slow);
 257 %}
 258 
 259 instruct storeV_partial(vReg src, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
 260   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() > 16 &&
 261             n->as_StoreVector()->memory_size() < MaxVectorSize);
 262   match(Set mem (StoreVector mem src));
 263   effect(TEMP pgtmp, KILL cr);
 264   ins_cost(5 * SVE_COST);
 265   format %{ "sve_whilelo_zr_imm $pgtmp, vector_length\n\t"
 266             "sve_str $src, $pgtmp, $mem\t# store vector partial" %}
 267   ins_encode %{
 268     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 269     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 270                           Matcher::vector_length(this, $src));
 271     FloatRegister src_reg = as_FloatRegister($src$$reg);
 272     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg,
 273                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 274                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 275   %}
 276   ins_pipe(pipe_slow);
 277 %}
 278 
 279 // vector load/store - predicated
 280 
 281 instruct loadV_masked(vReg dst, vmemA mem, pRegGov pg) %{
 282   predicate(UseSVE > 0 &&
 283             n->as_LoadVector()->memory_size() == MaxVectorSize);
 284   match(Set dst (LoadVectorMasked mem pg));
 285   ins_cost(4 * SVE_COST);
 286   format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated (sve)" %}
 287   ins_encode %{
 288     BasicType bt = Matcher::vector_element_basic_type(this);
 289     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
 290                           as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
 291                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 292   %}
 293   ins_pipe(pipe_slow);
 294 %}
 295 
 296 instruct loadV_masked_partial(vReg dst, vmemA mem, pRegGov pg, pRegGov pgtmp, rFlagsReg cr) %{
 297   predicate(UseSVE > 0 &&
 298             n->as_LoadVector()->memory_size() < MaxVectorSize);
 299   match(Set dst (LoadVectorMasked mem pg));
 300   effect(TEMP pgtmp, KILL cr);
 301   ins_cost(6 * SVE_COST);
 302   format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated partial (sve)" %}
 303   ins_encode %{
 304     BasicType bt = Matcher::vector_element_basic_type(this);
 305     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 306                           Matcher::vector_length(this));
 307     __ sve_and(as_PRegister($pgtmp$$reg), as_PRegister($pgtmp$$reg),
 308                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
 309     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
 310                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 311                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 312   %}
 313   ins_pipe(pipe_slow);
 314 %}
 315 
 316 instruct storeV_masked(vReg src, vmemA mem, pRegGov pg) %{
 317   predicate(UseSVE > 0 &&
 318             n->as_StoreVector()->memory_size() == MaxVectorSize);
 319   match(Set mem (StoreVectorMasked mem (Binary src pg)));
 320   ins_cost(4 * SVE_COST);
 321   format %{ "sve_str $mem, $pg, $src\t# store vector predicated (sve)" %}
 322   ins_encode %{
 323     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 324     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
 325                           as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
 326                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 327   %}
 328   ins_pipe(pipe_slow);
 329 %}
 330 
 331 instruct storeV_masked_partial(vReg src, vmemA mem, pRegGov pg, pRegGov pgtmp, rFlagsReg cr) %{
 332   predicate(UseSVE > 0 &&
 333             n->as_StoreVector()->memory_size() < MaxVectorSize);
 334   match(Set mem (StoreVectorMasked mem (Binary src pg)));
 335   effect(TEMP pgtmp, KILL cr);
 336   ins_cost(6 * SVE_COST);
 337   format %{ "sve_str $mem, $pg, $src\t# store vector predicated partial (sve)" %}
 338   ins_encode %{
 339     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 340     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 341                           Matcher::vector_length(this, $src));
 342     __ sve_and(as_PRegister($pgtmp$$reg), as_PRegister($pgtmp$$reg),
 343                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
 344     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
 345                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 346                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 347   %}
 348   ins_pipe(pipe_slow);
 349 %}
 350 
 351 dnl
 352 dnl MASKALL_IMM($1,   $2  )
 353 dnl MASKALL_IMM(type, size)
 354 define(`MASKALL_IMM', `
 355 instruct vmaskAll_imm$1(pRegGov dst, imm$1 src) %{
 356   predicate(UseSVE > 0);
 357   match(Set dst (MaskAll src));
 358   ins_cost(SVE_COST);
 359   format %{ "sve_ptrue/sve_pfalse $dst\t# mask all (sve) ($2)" %}
 360   ins_encode %{
 361     ifelse($1, `I', int, long) con = (ifelse($1, `I', int, long))$src$$constant;
 362     if (con == 0) {
 363       __ sve_pfalse(as_PRegister($dst$$reg));
 364     } else {
 365       assert(con == -1, "invalid constant value for mask");
 366       BasicType bt = Matcher::vector_element_basic_type(this);
 367       __ sve_ptrue(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt));
 368     }
 369   %}
 370   ins_pipe(pipe_slow);
 371 %}')dnl
 372 dnl
 373 dnl MASKALL($1,   $2  )
 374 dnl MASKALL(type, size)
 375 define(`MASKALL', `
 376 instruct vmaskAll$1(pRegGov dst, ifelse($1, `I', iRegIorL2I, iRegL) src, vReg tmp, rFlagsReg cr) %{
 377   predicate(UseSVE > 0);
 378   match(Set dst (MaskAll src));
 379   effect(TEMP tmp, KILL cr);
 380   ins_cost(2 * SVE_COST);
 381   format %{ "sve_dup $tmp, $src\n\t"
 382             "sve_cmpne $dst, $tmp, 0\t# mask all (sve) ($2)" %}
 383   ins_encode %{
 384     BasicType bt = Matcher::vector_element_basic_type(this);
 385     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
 386     __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
 387     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
 388   %}
 389   ins_pipe(pipe_slow);
 390 %}')dnl
 391 dnl
 392 // maskAll
 393 MASKALL_IMM(I, B/H/S)
 394 MASKALL(I, B/H/S)
 395 MASKALL_IMM(L, D)
 396 MASKALL(L, D)
 397 
 398 dnl
 399 dnl MASK_LOGICAL_OP($1,        $2,      $3  )
 400 dnl MASK_LOGICAL_OP(insn_name, op_name, insn)
 401 define(`MASK_LOGICAL_OP', `
 402 instruct vmask_$1(pRegGov pd, pRegGov pn, pRegGov pm) %{
 403   predicate(UseSVE > 0);
 404   match(Set pd ($2 pn pm));
 405   ins_cost(SVE_COST);
 406   format %{ "$3 $pd, $pn, $pm\t# predicate (sve)" %}
 407   ins_encode %{
 408     __ $3(as_PRegister($pd$$reg), ptrue,
 409                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 410   %}
 411   ins_pipe(pipe_slow);
 412 %}')dnl
 413 dnl
 414 // mask logical and/or/xor
 415 MASK_LOGICAL_OP(and, AndVMask, sve_and)
 416 MASK_LOGICAL_OP(or, OrVMask, sve_orr)
 417 MASK_LOGICAL_OP(xor, XorVMask, sve_eor)
 418 
 419 dnl
 420 dnl MASK_LOGICAL_AND_NOT($1,   $2  )
 421 dnl MASK_LOGICAL_AND_NOT(type, size)
 422 define(`MASK_LOGICAL_AND_NOT', `
 423 instruct vmask_and_not$1(pRegGov pd, pRegGov pn, pRegGov pm, imm$1_M1 m1) %{
 424   predicate(UseSVE > 0);
 425   match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
 426   ins_cost(SVE_COST);
 427   format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) ($2)" %}
 428   ins_encode %{
 429     __ sve_bic(as_PRegister($pd$$reg), ptrue,
 430                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 431   %}
 432   ins_pipe(pipe_slow);
 433 %}')dnl
 434 dnl
 435 // mask logical and_not
 436 MASK_LOGICAL_AND_NOT(I, B/H/S)
 437 MASK_LOGICAL_AND_NOT(L, D)
 438 
 439 // vector reinterpret
 440 
 441 instruct reinterpret(vReg dst) %{
 442   predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() ==
 443                           n->in(1)->bottom_type()->is_vect()->length_in_bytes());  // src == dst
 444   match(Set dst (VectorReinterpret dst));
 445   ins_cost(0);
 446   format %{ "# reinterpret $dst\t# do nothing" %}
 447   ins_encode %{
 448     // empty
 449   %}
 450   ins_pipe(pipe_class_empty);
 451 %}
 452 
 453 instruct reinterpretResize(vReg dst, vReg src, pRegGov pgtmp, rFlagsReg cr) %{
 454   predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() !=
 455                           n->in(1)->bottom_type()->is_vect()->length_in_bytes());  // src != dst
 456   match(Set dst (VectorReinterpret src));
 457   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
 458   ins_cost(3 * SVE_COST);
 459   format %{ "reinterpretResize $dst, $src\t# vector (sve)" %}
 460   ins_encode %{
 461     uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
 462     uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
 463     uint length_in_bytes_resize = length_in_bytes_src < length_in_bytes_dst ?
 464                                   length_in_bytes_src : length_in_bytes_dst;
 465     assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
 466            "invalid vector length");
 467     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ B, length_in_bytes_resize);
 468     __ sve_dup(as_FloatRegister($dst$$reg), __ B, 0);
 469     __ sve_sel(as_FloatRegister($dst$$reg), __ B, as_PRegister($pgtmp$$reg),
 470                as_FloatRegister($src$$reg), as_FloatRegister($dst$$reg));
 471   %}
 472   ins_pipe(pipe_slow);
 473 %}
 474 
 475 // vector mask reinterpret
 476 
 477 instruct vmask_reinterpret_same_esize(pRegGov dst_src) %{
 478   predicate(UseSVE > 0 &&
 479             n->as_Vector()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
 480             n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
 481   match(Set dst_src (VectorReinterpret dst_src));
 482   ins_cost(0);
 483   format %{ "# vmask_reinterpret $dst_src\t# do nothing" %}
 484   ins_encode %{
 485     // empty
 486   %}
 487   ins_pipe(pipe_class_empty);
 488 %}
 489 
 490 instruct vmask_reinterpret_diff_esize(pRegGov dst, pRegGov src, vReg tmp, rFlagsReg cr) %{
 491   predicate(UseSVE > 0 &&
 492             n->as_Vector()->length() != n->in(1)->bottom_type()->is_vect()->length() &&
 493             n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
 494   match(Set dst (VectorReinterpret src));
 495   effect(TEMP tmp, KILL cr);
 496   ins_cost(2 * SVE_COST);
 497   format %{ "# vmask_reinterpret $dst, $src\t# vector (sve)" %}
 498   ins_encode %{
 499     BasicType from_bt = Matcher::vector_element_basic_type(this, $src);
 500     Assembler::SIMD_RegVariant from_size = __ elemType_to_regVariant(from_bt);
 501     BasicType to_bt = Matcher::vector_element_basic_type(this);
 502     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
 503     __ sve_cpy(as_FloatRegister($tmp$$reg), from_size, as_PRegister($src$$reg), -1, false);
 504     __ sve_cmp(Assembler::EQ, as_PRegister($dst$$reg), to_size, ptrue, as_FloatRegister($tmp$$reg), -1);
 505   %}
 506   ins_pipe(pipe_slow);
 507 %}
 508 dnl
 509 dnl UNARY_OP_TRUE_PREDICATE($1,        $2,      $3,   $4  )
 510 dnl UNARY_OP_TRUE_PREDICATE(insn_name, op_name, size, insn)
 511 define(`UNARY_OP_TRUE_PREDICATE', `
 512 instruct $1(vReg dst, vReg src) %{
 513   predicate(UseSVE > 0 &&
 514             !n->as_Vector()->is_predicated_vector());
 515   match(Set dst ($2 src));
 516   ins_cost(SVE_COST);
 517   format %{ "$4 $dst, $src\t# vector (sve) ($3)" %}
 518   ins_encode %{
 519     __ $4(as_FloatRegister($dst$$reg), __ $3,
 520          ptrue, as_FloatRegister($src$$reg));
 521   %}
 522   ins_pipe(pipe_slow);
 523 %}')dnl
 524 dnl
 525 
 526 // vector abs
 527 UNARY_OP_TRUE_PREDICATE(vabsB, AbsVB, B, sve_abs)
 528 UNARY_OP_TRUE_PREDICATE(vabsS, AbsVS, H, sve_abs)
 529 UNARY_OP_TRUE_PREDICATE(vabsI, AbsVI, S, sve_abs)
 530 UNARY_OP_TRUE_PREDICATE(vabsL, AbsVL, D, sve_abs)
 531 UNARY_OP_TRUE_PREDICATE(vabsF, AbsVF, S, sve_fabs)
 532 UNARY_OP_TRUE_PREDICATE(vabsD, AbsVD, D, sve_fabs)
 533 
 534 dnl UNARY_OP_PREDICATE($1,        $2,      $3,   $4  )
 535 dnl UNARY_OP_PREDICATE(insn_name, op_name, size, insn)
 536 define(`UNARY_OP_PREDICATE', `
 537 instruct $1_masked(vReg dst_src, pRegGov pg) %{
 538   predicate(UseSVE > 0);
 539   match(Set dst_src ($2 dst_src pg));
 540   ins_cost(SVE_COST);
 541   format %{ "$4 $dst_src, $pg, $dst_src\t# vector (sve) ($3)" %}
 542   ins_encode %{
 543     __ $4(as_FloatRegister($dst_src$$reg), __ $3,
 544             as_PRegister($pg$$reg),
 545             as_FloatRegister($dst_src$$reg));
 546   %}
 547   ins_pipe(pipe_slow);
 548 %}')dnl
 549 // vector abs - predicated
 550 UNARY_OP_PREDICATE(vabsB, AbsVB, B, sve_abs)
 551 UNARY_OP_PREDICATE(vabsS, AbsVS, H, sve_abs)
 552 UNARY_OP_PREDICATE(vabsI, AbsVI, S, sve_abs)
 553 UNARY_OP_PREDICATE(vabsL, AbsVL, D, sve_abs)
 554 UNARY_OP_PREDICATE(vabsF, AbsVF, S, sve_fabs)
 555 UNARY_OP_PREDICATE(vabsD, AbsVD, D, sve_fabs)
 556 
 557 dnl
 558 dnl BINARY_OP_UNPREDICATE($1,        $2       $3,   $4           $5  )
 559 dnl BINARY_OP_UNPREDICATE(insn_name, op_name, size, min_vec_len, insn)
 560 define(`BINARY_OP_UNPREDICATE', `
 561 instruct $1(vReg dst, vReg src1, vReg src2) %{
 562   predicate(UseSVE > 0);
 563   match(Set dst ($2 src1 src2));
 564   ins_cost(SVE_COST);
 565   format %{ "$5 $dst, $src1, $src2\t # vector (sve) ($3)" %}
 566   ins_encode %{
 567     __ $5(as_FloatRegister($dst$$reg), __ $3,
 568          as_FloatRegister($src1$$reg),
 569          as_FloatRegister($src2$$reg));
 570   %}
 571   ins_pipe(pipe_slow);
 572 %}')dnl
 573 dnl
 574 dnl
 575 dnl BINARY_OP_PREDICATE($1,        $2,      $3,   $4  )
 576 dnl BINARY_OP_PREDICATE(insn_name, op_name, size, insn)
 577 define(`BINARY_OP_PREDICATE', `
 578 instruct $1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 579   predicate(UseSVE > 0);
 580   match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
 581   ins_cost(SVE_COST);
 582   format %{ "$4 $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) ($3)" %}
 583   ins_encode %{
 584     __ $4(as_FloatRegister($dst_src1$$reg), __ $3,
 585             as_PRegister($pg$$reg),
 586             as_FloatRegister($src2$$reg));
 587   %}
 588   ins_pipe(pipe_slow);
 589 %}')dnl
 590 dnl
 591 // vector add
 592 BINARY_OP_UNPREDICATE(vaddB, AddVB, B, 16, sve_add)
 593 BINARY_OP_UNPREDICATE(vaddS, AddVS, H, 8,  sve_add)
 594 BINARY_OP_UNPREDICATE(vaddI, AddVI, S, 4,  sve_add)
 595 BINARY_OP_UNPREDICATE(vaddL, AddVL, D, 2,  sve_add)
 596 BINARY_OP_UNPREDICATE(vaddF, AddVF, S, 4,  sve_fadd)
 597 BINARY_OP_UNPREDICATE(vaddD, AddVD, D, 2,  sve_fadd)
 598 
 599 // vector add - predicated
 600 BINARY_OP_PREDICATE(vaddB, AddVB, B, sve_add)
 601 BINARY_OP_PREDICATE(vaddS, AddVS, H, sve_add)
 602 BINARY_OP_PREDICATE(vaddI, AddVI, S, sve_add)
 603 BINARY_OP_PREDICATE(vaddL, AddVL, D, sve_add)
 604 BINARY_OP_PREDICATE(vaddF, AddVF, S, sve_fadd)
 605 BINARY_OP_PREDICATE(vaddD, AddVD, D, sve_fadd)
 606 
 607 dnl
 608 dnl BINARY_OP_UNSIZED($1,        $2,      $3  )
 609 dnl BINARY_OP_UNSIZED(insn_name, op_name, insn)
 610 define(`BINARY_OP_UNSIZED', `
 611 instruct $1(vReg dst, vReg src1, vReg src2) %{
 612   predicate(UseSVE > 0);
 613   match(Set dst ($2 src1 src2));
 614   ins_cost(SVE_COST);
 615   format %{ "$3  $dst, $src1, $src2\t# vector (sve)" %}
 616   ins_encode %{
 617     __ $3(as_FloatRegister($dst$$reg),
 618          as_FloatRegister($src1$$reg),
 619          as_FloatRegister($src2$$reg));
 620   %}
 621   ins_pipe(pipe_slow);
 622 %}')dnl
 623 dnl
 624 // vector and
 625 BINARY_OP_UNSIZED(vand, AndV, sve_and)
 626 
 627 // vector or
 628 BINARY_OP_UNSIZED(vor, OrV, sve_orr)
 629 
 630 // vector xor
 631 BINARY_OP_UNSIZED(vxor, XorV, sve_eor)
 632 
 633 dnl BINARY_LOGIC_OP_PREDICATE($1,        $2,      $3  )
 634 dnl BINARY_LOGIC_OP_PREDICATE(insn_name, op_name, insn)
 635 define(`BINARY_LOGIC_OP_PREDICATE', `
 636 instruct $1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 637   predicate(UseSVE > 0);
 638   match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
 639   ins_cost(SVE_COST);
 640   format %{ "$3 $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
 641   ins_encode %{
 642     BasicType bt = Matcher::vector_element_basic_type(this);
 643     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
 644     __ $3(as_FloatRegister($dst_src1$$reg), size,
 645           as_PRegister($pg$$reg),
 646           as_FloatRegister($src2$$reg));
 647   %}
 648   ins_pipe(pipe_slow);
 649 %}')dnl
 650 dnl
 651 // vector and - predicated
 652 BINARY_LOGIC_OP_PREDICATE(vand, AndV, sve_and)
 653 
 654 // vector or - predicated
 655 BINARY_LOGIC_OP_PREDICATE(vor, OrV, sve_orr)
 656 
 657 // vector xor - predicated
 658 BINARY_LOGIC_OP_PREDICATE(vxor, XorV, sve_eor)
 659 
 660 // vector not
 661 dnl
 662 define(`MATCH_RULE', `ifelse($1, I,
 663 `match(Set dst (XorV src (ReplicateB m1)));
 664   match(Set dst (XorV src (ReplicateS m1)));
 665   match(Set dst (XorV src (ReplicateI m1)));',
 666 `match(Set dst (XorV src (ReplicateL m1)));')')dnl
 667 dnl
 668 define(`VECTOR_NOT', `
 669 instruct vnot$1`'(vReg dst, vReg src, imm$1_M1 m1) %{
 670   predicate(UseSVE > 0);
 671   MATCH_RULE($1)
 672   ins_cost(SVE_COST);
 673   format %{ "sve_not $dst, $src\t# vector (sve) $2" %}
 674   ins_encode %{
 675     __ sve_not(as_FloatRegister($dst$$reg), __ D,
 676                ptrue, as_FloatRegister($src$$reg));
 677   %}
 678   ins_pipe(pipe_slow);
 679 %}')dnl
 680 dnl        $1,$2
 681 VECTOR_NOT(I, B/H/S)
 682 VECTOR_NOT(L, D)
 683 undefine(MATCH_RULE)
 684 dnl
 685 // vector and_not
 686 dnl
 687 define(`MATCH_RULE', `ifelse($1, I,
 688 `match(Set dst (AndV src1 (XorV src2 (ReplicateB m1))));
 689   match(Set dst (AndV src1 (XorV src2 (ReplicateS m1))));
 690   match(Set dst (AndV src1 (XorV src2 (ReplicateI m1))));',
 691 `match(Set dst (AndV src1 (XorV src2 (ReplicateL m1))));')')dnl
 692 dnl
 693 define(`VECTOR_AND_NOT', `
 694 instruct vand_not$1`'(vReg dst, vReg src1, vReg src2, imm$1_M1 m1) %{
 695   predicate(UseSVE > 0);
 696   MATCH_RULE($1)
 697   ins_cost(SVE_COST);
 698   format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) $2" %}
 699   ins_encode %{
 700     __ sve_bic(as_FloatRegister($dst$$reg),
 701                as_FloatRegister($src1$$reg),
 702                as_FloatRegister($src2$$reg));
 703   %}
 704   ins_pipe(pipe_slow);
 705 %}')dnl
 706 dnl            $1,$2
 707 VECTOR_AND_NOT(I, B/H/S)
 708 VECTOR_AND_NOT(L, D)
 709 undefine(MATCH_RULE)
 710 dnl
 711 dnl VDIVF($1,          $2  , $3         )
 712 dnl VDIVF(name_suffix, size, min_vec_len)
 713 define(`VDIVF', `
 714 instruct vdiv$1(vReg dst_src1, vReg src2) %{
 715   predicate(UseSVE > 0);
 716   match(Set dst_src1 (DivV$1 dst_src1 src2));
 717   ins_cost(SVE_COST);
 718   format %{ "sve_fdiv  $dst_src1, $dst_src1, $src2\t# vector (sve) ($2)" %}
 719   ins_encode %{
 720     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ $2,
 721          ptrue, as_FloatRegister($src2$$reg));
 722   %}
 723   ins_pipe(pipe_slow);
 724 %}')dnl
 725 dnl
 726 // vector float div
 727 VDIVF(F, S, 4)
 728 VDIVF(D, D, 2)
 729 
 730 // vector float div - predicated
 731 BINARY_OP_PREDICATE(vfdivF, DivVF, S, sve_fdiv)
 732 BINARY_OP_PREDICATE(vfdivD, DivVD, D, sve_fdiv)
 733 
 734 dnl
 735 dnl VMINMAX($1     , $2, $3   , $4  )
 736 dnl VMINMAX(op_name, op, finsn, insn)
 737 define(`VMINMAX', `
 738 instruct v$1(vReg dst_src1, vReg src2) %{
 739   predicate(UseSVE > 0);
 740   match(Set dst_src1 ($2 dst_src1 src2));
 741   ins_cost(SVE_COST);
 742   format %{ "sve_$1 $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
 743   ins_encode %{
 744     BasicType bt = Matcher::vector_element_basic_type(this);
 745     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
 746     if (is_floating_point_type(bt)) {
 747       __ $3(as_FloatRegister($dst_src1$$reg), size,
 748                   ptrue, as_FloatRegister($src2$$reg));
 749     } else {
 750       assert(is_integral_type(bt), "unsupported type");
 751       __ $4(as_FloatRegister($dst_src1$$reg), size,
 752                   ptrue, as_FloatRegister($src2$$reg));
 753     }
 754   %}
 755   ins_pipe(pipe_slow);
 756 %}')dnl
 757 dnl
 758 // vector min/max
 759 VMINMAX(min, MinV, sve_fmin, sve_smin)
 760 VMINMAX(max, MaxV, sve_fmax, sve_smax)
 761 
 762 dnl
 763 dnl VMINMAX_PREDICATE($1     , $2, $3   , $4  )
 764 dnl VMINMAX_PREDICATE(op_name, op, finsn, insn)
 765 define(`VMINMAX_PREDICATE', `
 766 instruct v$1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 767   predicate(UseSVE > 0);
 768   match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
 769   ins_cost(SVE_COST);
 770   format %{ "sve_$1 $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %}
 771   ins_encode %{
 772     BasicType bt = Matcher::vector_element_basic_type(this);
 773     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
 774     if (is_floating_point_type(bt)) {
 775       __ $3(as_FloatRegister($dst_src1$$reg), size,
 776                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
 777     } else {
 778       assert(is_integral_type(bt), "unsupported type");
 779       __ $4(as_FloatRegister($dst_src1$$reg), size,
 780                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
 781     }
 782   %}
 783   ins_pipe(pipe_slow);
 784 %}')dnl
 785 dnl
 786 // vector min/max - predicated
 787 VMINMAX_PREDICATE(min, MinV, sve_fmin, sve_smin)
 788 VMINMAX_PREDICATE(max, MaxV, sve_fmax, sve_smax)
 789 
 790 dnl
 791 dnl VFMLA($1           $2    $3         )
 792 dnl VFMLA(name_suffix, size, min_vec_len)
 793 define(`VFMLA', `
 794 // dst_src1 = dst_src1 + src2 * src3
 795 instruct vfmla$1(vReg dst_src1, vReg src2, vReg src3) %{
 796   predicate(UseFMA && UseSVE > 0);
 797   match(Set dst_src1 (FmaV$1 dst_src1 (Binary src2 src3)));
 798   ins_cost(SVE_COST);
 799   format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
 800   ins_encode %{
 801     __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ $2,
 802          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 803   %}
 804   ins_pipe(pipe_slow);
 805 %}')dnl
 806 dnl
 807 // vector fmla
 808 VFMLA(F, S, 4)
 809 VFMLA(D, D, 2)
 810 
 811 dnl
 812 dnl VFMLA_PREDICATE($1,   $2  )
 813 dnl VFMLA_PREDICATE(type, size)
 814 define(`VFMLA_PREDICATE', `
 815 // dst_src1 = dst_src1 * src2 + src3
 816 instruct vfmla$1_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
 817   predicate(UseFMA && UseSVE > 0);
 818   match(Set dst_src1 (FmaV$1 (Binary dst_src1 src2) (Binary src3 pg)));
 819   ins_cost(SVE_COST);
 820   format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) ($2)" %}
 821   ins_encode %{
 822     __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ $2, as_PRegister($pg$$reg),
 823          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 824   %}
 825   ins_pipe(pipe_slow);
 826 %}')dnl
 827 dnl
 828 // vector fmla - predicated
 829 VFMLA_PREDICATE(F, S)
 830 VFMLA_PREDICATE(D, D)
 831 
 832 dnl
 833 dnl VFMLS($1           $2    $3         )
 834 dnl VFMLS(name_suffix, size, min_vec_len)
 835 define(`VFMLS', `
 836 // dst_src1 = dst_src1 + -src2 * src3
 837 // dst_src1 = dst_src1 + src2 * -src3
 838 instruct vfmls$1(vReg dst_src1, vReg src2, vReg src3) %{
 839   predicate(UseFMA && UseSVE > 0);
 840   match(Set dst_src1 (FmaV$1 dst_src1 (Binary (NegV$1 src2) src3)));
 841   match(Set dst_src1 (FmaV$1 dst_src1 (Binary src2 (NegV$1 src3))));
 842   ins_cost(SVE_COST);
 843   format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
 844   ins_encode %{
 845     __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ $2,
 846          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 847   %}
 848   ins_pipe(pipe_slow);
 849 %}')dnl
 850 dnl
 851 // vector fmls

 942 VMLS(B, B, 16)
 943 VMLS(S, H, 8)
 944 VMLS(I, S, 4)
 945 VMLS(L, D, 2)
 946 
 947 dnl
 948 dnl BINARY_OP_TRUE_PREDICATE($1,        $2,      $3,   $4,          $5  )
 949 dnl BINARY_OP_TRUE_PREDICATE(insn_name, op_name, size, min_vec_len, insn)
 950 define(`BINARY_OP_TRUE_PREDICATE', `
 951 instruct $1(vReg dst_src1, vReg src2) %{
 952   predicate(UseSVE > 0);
 953   match(Set dst_src1 ($2 dst_src1 src2));
 954   ins_cost(SVE_COST);
 955   format %{ "$5 $dst_src1, $dst_src1, $src2\t # vector (sve) ($3)" %}
 956   ins_encode %{
 957     __ $5(as_FloatRegister($dst_src1$$reg), __ $3,
 958          ptrue, as_FloatRegister($src2$$reg));
 959   %}
 960   ins_pipe(pipe_slow);
 961 %}')dnl
 962 dnl
 963 // vector mul
 964 BINARY_OP_TRUE_PREDICATE(vmulB, MulVB, B, 16, sve_mul)
 965 BINARY_OP_TRUE_PREDICATE(vmulS, MulVS, H, 8,  sve_mul)
 966 BINARY_OP_TRUE_PREDICATE(vmulI, MulVI, S, 4,  sve_mul)
 967 BINARY_OP_TRUE_PREDICATE(vmulL, MulVL, D, 2,  sve_mul)
 968 BINARY_OP_UNPREDICATE(vmulF, MulVF, S, 4, sve_fmul)
 969 BINARY_OP_UNPREDICATE(vmulD, MulVD, D, 2, sve_fmul)
 970 
 971 // vector mul - predicated
 972 BINARY_OP_PREDICATE(vmulB, MulVB, B, sve_mul)
 973 BINARY_OP_PREDICATE(vmulS, MulVS, H, sve_mul)
 974 BINARY_OP_PREDICATE(vmulI, MulVI, S, sve_mul)
 975 BINARY_OP_PREDICATE(vmulL, MulVL, D, sve_mul)
 976 BINARY_OP_PREDICATE(vmulF, MulVF, S, sve_fmul)
 977 BINARY_OP_PREDICATE(vmulD, MulVD, D, sve_fmul)
 978 
















 979 // vector fneg
 980 UNARY_OP_TRUE_PREDICATE(vnegF, NegVF, S, sve_fneg)
 981 UNARY_OP_TRUE_PREDICATE(vnegD, NegVD, D, sve_fneg)
 982 
 983 // vector fneg - predicated
 984 UNARY_OP_PREDICATE(vnegF, NegVF, S, sve_fneg)
 985 UNARY_OP_PREDICATE(vnegD, NegVD, D, sve_fneg)
 986 
 987 // popcount vector
 988 
 989 instruct vpopcountI(vReg dst, vReg src) %{
 990   predicate(UseSVE > 0);
 991   match(Set dst (PopCountVI src));
 992   format %{ "sve_cnt $dst, $src\t# vector (sve) (S)\n\t" %}
 993   ins_encode %{
 994      __ sve_cnt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg));
 995   %}
 996   ins_pipe(pipe_slow);
 997 %}
 998 
 999 // vector mask compare
1000 
1001 instruct vmaskcmp(pRegGov dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %{
1002   predicate(UseSVE > 0);
1003   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
1004   effect(KILL cr);
1005   ins_cost(SVE_COST);
1006   format %{ "sve_cmp $dst, $src1, $src2\t# vector mask cmp (sve)" %}

1007   ins_encode %{
1008     BasicType bt = Matcher::vector_element_basic_type(this);
1009     __ sve_compare(as_PRegister($dst$$reg), bt, ptrue, as_FloatRegister($src1$$reg),
1010                    as_FloatRegister($src2$$reg), (int)$cond$$constant);


1011   %}
1012   ins_pipe(pipe_slow);
1013 %}
1014 
1015 instruct vmaskcmp_masked(pRegGov dst, vReg src1, vReg src2, immI cond, pRegGov pg, rFlagsReg cr) %{


1016   predicate(UseSVE > 0);
1017   match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond pg)));
1018   effect(KILL cr);
1019   ins_cost(SVE_COST);
1020   format %{ "sve_cmp $dst, $pg, $src1, $src2\t# vector mask cmp (sve)" %}

1021   ins_encode %{
1022     BasicType bt = Matcher::vector_element_basic_type(this);
1023     __ sve_compare(as_PRegister($dst$$reg), bt, as_PRegister($pg$$reg), as_FloatRegister($src1$$reg),
1024                    as_FloatRegister($src2$$reg), (int)$cond$$constant);



1025   %}
1026   ins_pipe(pipe_slow);
1027 %}
1028 
1029 // vector blend
1030 
1031 instruct vblend(vReg dst, vReg src1, vReg src2, pRegGov pg) %{

1032   predicate(UseSVE > 0);
1033   match(Set dst (VectorBlend (Binary src1 src2) pg));
1034   ins_cost(SVE_COST);
1035   format %{ "sve_sel $dst, $pg, $src2, $src1\t# vector blend (sve)" %}


1036   ins_encode %{
1037     Assembler::SIMD_RegVariant size =
1038                __ elemType_to_regVariant(Matcher::vector_element_basic_type(this));
1039     __ sve_sel(as_FloatRegister($dst$$reg), size, as_PRegister($pg$$reg),
1040                as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));


1041   %}
1042   ins_pipe(pipe_slow);
1043 %}
1044 
1045 // vector load mask
1046 
1047 instruct vloadmaskB(pRegGov dst, vReg src, rFlagsReg cr) %{
1048   predicate(UseSVE > 0 &&
1049             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
1050   match(Set dst (VectorLoadMask src));
1051   effect(KILL cr);
1052   ins_cost(SVE_COST);
1053   format %{ "vloadmaskB $dst, $src\t# vector load mask (sve) (B)" %}













1054   ins_encode %{
1055     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ B,
1056                ptrue, as_FloatRegister($src$$reg), 0);
1057   %}
1058   ins_pipe(pipe_slow);
1059 %}
1060 
1061 instruct vloadmask_extend(pRegGov dst, vReg src, vReg tmp, rFlagsReg cr) %{
1062   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() != T_BYTE);


1063   match(Set dst (VectorLoadMask src));
1064   effect(TEMP tmp, KILL cr);
1065   ins_cost(3 * SVE_COST);
1066   format %{ "vloadmask $dst, $src\t# vector load mask (sve) (H/S/D)" %}




















1067   ins_encode %{
1068     BasicType bt = Matcher::vector_element_basic_type(this);
1069     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1070     __ sve_vector_extend(as_FloatRegister($tmp$$reg), size, as_FloatRegister($src$$reg), __ B);
1071     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
1072   %}
1073   ins_pipe(pipe_slow);
1074 %}
1075 
1076 // vector store mask
1077 
1078 instruct vstoremaskB(vReg dst, pRegGov src, immI_1 size) %{
1079   predicate(UseSVE > 0);
1080   match(Set dst (VectorStoreMask src size));
1081   ins_cost(SVE_COST);
1082   format %{ "vstoremask $dst, $src\t# vector store mask (sve) (B)" %}
1083   ins_encode %{
1084     __ sve_cpy(as_FloatRegister($dst$$reg), __ B, as_PRegister($src$$reg), 1, false);

1085   %}
1086   ins_pipe(pipe_slow);
1087 %}
1088 
1089 instruct vstoremask_narrow(vReg dst, pRegGov src, vReg tmp, immI_gt_1 size) %{
1090   predicate(UseSVE > 0);
1091   match(Set dst (VectorStoreMask src size));
1092   effect(TEMP_DEF dst, TEMP tmp);
1093   ins_cost(3 * SVE_COST);
1094   format %{ "vstoremask $dst, $src\t# vector store mask (sve) (H/S/D)" %}


1095   ins_encode %{
1096     Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant((int)$size$$constant);
1097     __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($src$$reg), 1, false);
1098     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
1099                          as_FloatRegister($dst$$reg), size, as_FloatRegister($tmp$$reg));


1100   %}
1101   ins_pipe(pipe_slow);
1102 %}
1103 
1104 // Combine LoadVector+VectorLoadMask when the vector element type is not T_BYTE



















1105 
1106 instruct vloadmask_loadV(pRegGov dst, indirect mem, vReg tmp, rFlagsReg cr) %{
1107   predicate(UseSVE > 0 &&
1108             n->as_Vector()->length_in_bytes() == MaxVectorSize &&
1109             type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
1110   match(Set dst (VectorLoadMask (LoadVector mem)));
1111   effect(TEMP tmp, KILL cr);
1112   ins_cost(3 * SVE_COST);
1113   format %{ "sve_ld1b $tmp, $mem\n\t"
1114             "sve_cmpne $dst, $tmp, 0\t# load vector mask (sve) (H/S/D)" %}

1115   ins_encode %{
1116     // Load mask values which are boolean type, and extend them to the
1117     // expected vector element type. Convert the vector to predicate.
1118     BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
1119     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($tmp$$reg),
1120                           ptrue, T_BOOLEAN, to_vect_bt, $mem->opcode(),
1121                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1122     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ elemType_to_regVariant(to_vect_bt),
1123                ptrue, as_FloatRegister($tmp$$reg), 0);

1124   %}
1125   ins_pipe(pipe_slow);
1126 %}
1127 
1128 instruct vloadmask_loadV_partial(pRegGov dst, indirect mem, vReg vtmp, pRegGov ptmp, rFlagsReg cr) %{
1129   predicate(UseSVE > 0 &&
1130             n->as_Vector()->length_in_bytes() > 16 &&
1131             n->as_Vector()->length_in_bytes() < MaxVectorSize &&
1132             type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);


1133   match(Set dst (VectorLoadMask (LoadVector mem)));
1134   effect(TEMP vtmp, TEMP ptmp, KILL cr);
1135   ins_cost(6 * SVE_COST);
1136   format %{ "vloadmask_loadV $dst, $mem\t# load vector mask partial (sve) (H/S/D)" %}
1137   ins_encode %{
1138     // Load valid mask values which are boolean type, and extend them to the
1139     // expected vector element type. Convert the vector to predicate.
1140     BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
1141     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(to_vect_bt);
1142     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this));
1143     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($vtmp$$reg),
1144                           as_PRegister($ptmp$$reg), T_BOOLEAN, to_vect_bt, $mem->opcode(),
1145                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1146     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($vtmp$$reg), 0);
1147   %}
1148   ins_pipe(pipe_slow);
1149 %}
1150 
1151 // Combine VectorStoreMask+StoreVector when the vector element type is not T_BYTE
1152 
1153 instruct storeV_vstoremask(indirect mem, pRegGov src, vReg tmp, immI_gt_1 esize) %{
1154   predicate(UseSVE > 0 &&
1155             Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);




1156   match(Set mem (StoreVector mem (VectorStoreMask src esize)));
1157   effect(TEMP tmp);
1158   ins_cost(3 * SVE_COST);
1159   format %{ "sve_cpy $tmp, $src, 1\n\t"
1160             "sve_st1b $tmp, $mem\t# store vector mask (sve) (H/S/D)" %}
1161   ins_encode %{
1162     BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
1163     assert(type2aelembytes(from_vect_bt) == (int)$esize$$constant, "unsupported type.");
1164     Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
1165     __ sve_cpy(as_FloatRegister($tmp$$reg), size, as_PRegister($src$$reg), 1, false);

1166     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($tmp$$reg),
1167                           ptrue, T_BOOLEAN, from_vect_bt, $mem->opcode(),
1168                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1169   %}
1170   ins_pipe(pipe_slow);
1171 %}









1172 
1173 instruct storeV_vstoremask_partial(indirect mem, pRegGov src, vReg vtmp,
1174                                    immI_gt_1 esize, pRegGov ptmp, rFlagsReg cr) %{
1175   predicate(UseSVE > 0 &&
1176             n->as_StoreVector()->memory_size() > 16 &&
1177             type2aelembytes(n->as_StoreVector()->vect_type()->element_basic_type()) > 1 &&
1178             Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
1179   match(Set mem (StoreVector mem (VectorStoreMask src esize)));
1180   effect(TEMP vtmp, TEMP ptmp, KILL cr);
1181   format %{ "storeV_vstoremask $src, $mem\t# store vector mask partial (sve) (H/S/D)" %}
1182   ins_cost(6 * SVE_COST);
1183   ins_encode %{
1184     // Convert the valid src predicate to vector, and store the vector
1185     // elements as boolean values.
1186     BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
1187     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(from_vect_bt);
1188     __ sve_cpy(as_FloatRegister($vtmp$$reg), size, as_PRegister($src$$reg), 1, false);
1189     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
1190     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($vtmp$$reg),
1191                           as_PRegister($ptmp$$reg), T_BOOLEAN, from_vect_bt, $mem->opcode(),
1192                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1193   %}
1194   ins_pipe(pipe_slow);
1195 %}
1196 dnl
1197 dnl REDUCE_I($1,        $2     )
1198 dnl REDUCE_I(insn_name, op_name)
1199 define(`REDUCE_I', `
1200 instruct reduce_$1I(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
1201   ifelse($2, AddReductionVI,
1202        `predicate(UseSVE > 0 &&
1203             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);',
1204        `predicate(UseSVE > 0 &&
1205             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1206             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);')
1207   match(Set dst ($2 src1 src2));
1208   effect(TEMP_DEF dst, TEMP tmp);
1209   ins_cost(SVE_COST);
1210   format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction (sve) (may extend)" %}
1211   ins_encode %{
1212     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1213     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1214                            $src1$$Register, as_FloatRegister($src2$$reg),
1215                            ptrue, as_FloatRegister($tmp$$reg));








1216   %}
1217   ins_pipe(pipe_slow);
1218 %}')dnl
1219 dnl
1220 dnl
1221 dnl REDUCE_L($1,        $2    )
1222 dnl REDUCE_L(insn_name, op_name)
1223 define(`REDUCE_L', `
1224 instruct reduce_$1L(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
1225   ifelse($2, AddReductionVL,
1226        `predicate(UseSVE > 0 &&
1227             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);',
1228        `predicate(UseSVE > 0 &&
1229             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1230             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);')
1231   match(Set dst ($2 src1 src2));
1232   effect(TEMP_DEF dst, TEMP tmp);
1233   ins_cost(SVE_COST);
1234   format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction (sve)" %}
1235   ins_encode %{
1236     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1237                            $src1$$Register, as_FloatRegister($src2$$reg),
1238                            ptrue, as_FloatRegister($tmp$$reg));
1239   %}
1240   ins_pipe(pipe_slow);
1241 %}')dnl
1242 dnl
1243 dnl REDUCE_I_PARTIAL($1,        $2     )
1244 dnl REDUCE_I_PARTIAL(insn_name, op_name)
1245 define(`REDUCE_I_PARTIAL', `
1246 instruct reduce_$1I_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1247                              pRegGov ptmp, rFlagsReg cr) %{
1248   ifelse($2, AddReductionVI,
1249        `predicate(UseSVE > 0 &&
1250             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);',
1251        `predicate(UseSVE > 0 &&
1252             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1253             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);')
1254   match(Set dst ($2 src1 src2));
1255   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1256   ins_cost(2 * SVE_COST);
1257   format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction partial (sve) (may extend)" %}
1258   ins_encode %{
1259     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1260     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1261     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1262                           Matcher::vector_length(this, $src2));
1263     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1264                            $src1$$Register, as_FloatRegister($src2$$reg),
1265                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));






















1266   %}
1267   ins_pipe(pipe_slow);
1268 %}')dnl
1269 dnl
1270 dnl REDUCE_L_PARTIAL($1,        $2    )
1271 dnl REDUCE_L_PARTIAL(insn_name, op_name)
1272 define(`REDUCE_L_PARTIAL', `
1273 instruct reduce_$1L_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1274                              pRegGov ptmp, rFlagsReg cr) %{
1275   ifelse($2, AddReductionVL,
1276        `predicate(UseSVE > 0 &&
1277             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);',
1278        `predicate(UseSVE > 0 &&
1279             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1280             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);')
1281   match(Set dst ($2 src1 src2));
1282   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1283   ins_cost(2 * SVE_COST);
1284   format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction partial (sve)" %}
1285   ins_encode %{
1286     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1287                           Matcher::vector_length(this, $src2));
1288     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1289                            $src1$$Register, as_FloatRegister($src2$$reg),
1290                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));

1291   %}
1292   ins_pipe(pipe_slow);
1293 %}')dnl

1294 dnl
1295 dnl REDUCE_ADDF($1,        $2,      $3,      $4  )
1296 dnl REDUCE_ADDF(insn_name, op_name, reg_dst, size)
1297 define(`REDUCE_ADDF', `
1298 instruct reduce_$1($3 src1_dst, vReg src2) %{
1299   predicate(UseSVE > 0 &&
1300             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1301   match(Set src1_dst ($2 src1_dst src2));
1302   ins_cost(SVE_COST);
1303   format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) ($4)" %}
1304   ins_encode %{
1305     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1306          ptrue, as_FloatRegister($src2$$reg));
1307   %}
1308   ins_pipe(pipe_slow);
1309 %}')dnl
1310 dnl
1311 dnl
1312 dnl REDUCE_ADDF_PARTIAL($1,        $2,     $3,      $4  )
1313 dnl REDUCE_ADDF_PARTIAL(insn_name, suffix, reg_dst, size)
1314 define(`REDUCE_ADDF_PARTIAL', `
1315 instruct reduce_$1_partial($3 src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
1316   predicate(UseSVE > 0 &&
1317             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1318   match(Set src1_dst ($2 src1_dst src2));
1319   ins_cost(SVE_COST);
1320   effect(TEMP ptmp, KILL cr);
1321   format %{ "sve_reduce_$1 $src1_dst, $src1_dst, $src2\t# $1 reduction partial (sve) ($4)" %}
1322   ins_encode %{
1323     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
1324                           Matcher::vector_length(this, $src2));
1325     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1326                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1327   %}
1328   ins_pipe(pipe_slow);
1329 %}')dnl
1330 dnl
1331 dnl
1332 dnl REDUCE_I_PREDICATE($1,        $2     )
1333 dnl REDUCE_I_PREDICATE(insn_name, op_name)
1334 define(`REDUCE_I_PREDICATE', `
1335 instruct reduce_$1I_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
1336   ifelse($2, AddReductionVI,
1337        `predicate(UseSVE > 0 &&
1338             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);',
1339        `predicate(UseSVE > 0 &&
1340             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1341             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);')
1342   match(Set dst ($2 (Binary src1 src2) pg));
1343   effect(TEMP_DEF dst, TEMP tmp);
1344   ins_cost(SVE_COST);
1345   format %{ "sve_reduce_$1I $dst, $src1, $pg, $src2\t# $1I reduction predicated (sve) (may extend)" %}
1346   ins_encode %{
1347     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1348     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1349                            $src1$$Register, as_FloatRegister($src2$$reg),
1350                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));








1351   %}
1352   ins_pipe(pipe_slow);
1353 %}')dnl
1354 dnl
1355 dnl REDUCE_L_PREDICATE($1,        $2    )
1356 dnl REDUCE_L_PREDICATE(insn_name, op_name)
1357 define(`REDUCE_L_PREDICATE', `
1358 instruct reduce_$1L_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
1359   ifelse($2, AddReductionVL,
1360        `predicate(UseSVE > 0 &&
1361             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);',
1362        `predicate(UseSVE > 0 &&
1363             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1364             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);')
1365   match(Set dst ($2 (Binary src1 src2) pg));
1366   effect(TEMP_DEF dst, TEMP tmp);
1367   ins_cost(SVE_COST);
1368   format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated (sve)" %}
1369   ins_encode %{
1370     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1371                            $src1$$Register, as_FloatRegister($src2$$reg),
1372                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
1373   %}
1374   ins_pipe(pipe_slow);
1375 %}')dnl
1376 dnl
1377 dnl REDUCE_I_PREDICATE_PARTIAL($1,        $2     )
1378 dnl REDUCE_I_PREDICATE_PARTIAL(insn_name, op_name)
1379 define(`REDUCE_I_PREDICATE_PARTIAL', `
1380 instruct reduce_$1I_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1381                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
1382   ifelse($2, AddReductionVI,
1383        `predicate(UseSVE > 0 &&
1384             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);',
1385        `predicate(UseSVE > 0 &&
1386             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1387             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);')
1388   match(Set dst ($2 (Binary src1 src2) pg));
1389   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1390   ins_cost(3 * SVE_COST);
1391   format %{ "sve_reduce_$1I $dst, $src1, $pg, $src2\t# $1I reduction predicated partial (sve) (may extend)" %}
1392   ins_encode %{
1393     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1394     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1395     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1396                           Matcher::vector_length(this, $src2));
1397     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1398                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1399     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1400                            $src1$$Register, as_FloatRegister($src2$$reg),
1401                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));





















1402   %}
1403   ins_pipe(pipe_slow);
1404 %}')dnl
1405 dnl
1406 dnl REDUCE_L_PREDICATE_PARTIAL($1,        $2    )
1407 dnl REDUCE_L_PREDICATE_PARTIAL(insn_name, op_name)
1408 define(`REDUCE_L_PREDICATE_PARTIAL', `
1409 instruct reduce_$1L_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1410                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
1411   ifelse($2, AddReductionVL,
1412        `predicate(UseSVE > 0 &&
1413             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);',
1414        `predicate(UseSVE > 0 &&
1415             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1416             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);')
1417   match(Set dst ($2 (Binary src1 src2) pg));
1418   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1419   ins_cost(3 * SVE_COST);
1420   format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated partial (sve)" %}
1421   ins_encode %{
1422     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1423                           Matcher::vector_length(this, $src2));
1424     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1425                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1426     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1427                            $src1$$Register, as_FloatRegister($src2$$reg),
1428                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1429   %}
1430   ins_pipe(pipe_slow);
1431 %}')dnl
1432 dnl
1433 dnl REDUCE_ADDF_PREDICATE($1,        $2,      $3,      $4  )
1434 dnl REDUCE_ADDF_PREDICATE(insn_name, op_name, reg_dst, size)
1435 define(`REDUCE_ADDF_PREDICATE', `
1436 instruct reduce_$1_masked($3 src1_dst, vReg src2, pRegGov pg) %{
1437   predicate(UseSVE > 0 &&
1438             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1439   match(Set src1_dst ($2 (Binary src1_dst src2) pg));
1440   ins_cost(SVE_COST);
1441   format %{ "sve_reduce_$1 $src1_dst, $pg, $src2\t# $1 reduction predicated (sve)" %}
1442   ins_encode %{
1443     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1444                  as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));










1445   %}
1446   ins_pipe(pipe_slow);
1447 %}')dnl
1448 dnl
1449 dnl REDUCE_ADDF_PREDICATE_PARTIAL($1,        $2,      $3,      $4  )
1450 dnl REDUCE_ADDF_PREDICATE_PARTIAL(insn_name, op_name, reg_dst, size)
1451 define(`REDUCE_ADDF_PREDICATE_PARTIAL', `
1452 instruct reduce_$1_masked_partial($3 src1_dst, vReg src2, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
1453   predicate(UseSVE > 0 &&
1454             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1455   match(Set src1_dst ($2 (Binary src1_dst src2) pg));
1456   effect(TEMP ptmp, KILL cr);
1457   ins_cost(SVE_COST);
1458   format %{ "sve_reduce_$1 $src1_dst, $pg, $src2\t# $1 reduction predicated partial (sve)" %}
1459   ins_encode %{
1460     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,


1461                           Matcher::vector_length(this, $src2));
1462     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1463                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1464     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1465                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));







1466   %}
1467   ins_pipe(pipe_slow);
1468 %}')dnl
1469 dnl
1470 
1471 // vector add reduction
1472 REDUCE_I(add, AddReductionVI)
1473 REDUCE_L(add, AddReductionVL)
1474 REDUCE_ADDF(addF, AddReductionVF, vRegF, S)
1475 REDUCE_ADDF(addD, AddReductionVD, vRegD, D)
1476 REDUCE_I_PARTIAL(add, AddReductionVI)
1477 REDUCE_L_PARTIAL(add, AddReductionVL)
1478 REDUCE_ADDF_PARTIAL(addF, AddReductionVF, vRegF, S)
1479 REDUCE_ADDF_PARTIAL(addD, AddReductionVD, vRegD, D)
1480 
1481 // vector add reduction - predicated
1482 REDUCE_I_PREDICATE(add, AddReductionVI)
1483 REDUCE_L_PREDICATE(add, AddReductionVL)
1484 REDUCE_ADDF_PREDICATE(addF, AddReductionVF, vRegF, S)
1485 REDUCE_ADDF_PREDICATE(addD, AddReductionVD, vRegD, D)
1486 REDUCE_I_PREDICATE_PARTIAL(add, AddReductionVI)
1487 REDUCE_L_PREDICATE_PARTIAL(add, AddReductionVL)
1488 REDUCE_ADDF_PREDICATE_PARTIAL(addF, AddReductionVF, vRegF, S)
1489 REDUCE_ADDF_PREDICATE_PARTIAL(addD, AddReductionVD, vRegD, D)
1490 
1491 // vector and reduction
1492 REDUCE_I(and, AndReductionV)
1493 REDUCE_L(and, AndReductionV)
1494 REDUCE_I_PARTIAL(and, AndReductionV)
1495 REDUCE_L_PARTIAL(and, AndReductionV)
1496 
1497 // vector and reduction - predicated
1498 REDUCE_I_PREDICATE(and, AndReductionV)
1499 REDUCE_L_PREDICATE(and, AndReductionV)
1500 REDUCE_I_PREDICATE_PARTIAL(and, AndReductionV)
1501 REDUCE_L_PREDICATE_PARTIAL(and, AndReductionV)
1502 
1503 // vector or reduction
1504 REDUCE_I(or, OrReductionV)
1505 REDUCE_L(or, OrReductionV)
1506 REDUCE_I_PARTIAL(or, OrReductionV)
1507 REDUCE_L_PARTIAL(or, OrReductionV)
1508 
1509 // vector or reduction - predicated
1510 REDUCE_I_PREDICATE(or, OrReductionV)
1511 REDUCE_L_PREDICATE(or, OrReductionV)
1512 REDUCE_I_PREDICATE_PARTIAL(or, OrReductionV)
1513 REDUCE_L_PREDICATE_PARTIAL(or, OrReductionV)
1514 
1515 // vector xor reduction
1516 REDUCE_I(eor, XorReductionV)
1517 REDUCE_L(eor, XorReductionV)
1518 REDUCE_I_PARTIAL(eor, XorReductionV)
1519 REDUCE_L_PARTIAL(eor, XorReductionV)
1520 
1521 // vector xor reduction - predicated
1522 REDUCE_I_PREDICATE(eor, XorReductionV)
1523 REDUCE_L_PREDICATE(eor, XorReductionV)
1524 REDUCE_I_PREDICATE_PARTIAL(eor, XorReductionV)
1525 REDUCE_L_PREDICATE_PARTIAL(eor, XorReductionV)
1526 
1527 dnl
1528 dnl REDUCE_MAXMIN_I($1,        $2     )
1529 dnl REDUCE_MAXMIN_I(insn_name, op_name)
1530 define(`REDUCE_MAXMIN_I', `
1531 instruct reduce_$1I(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
1532   predicate(UseSVE > 0 &&
1533             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1534             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1535             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
1536   match(Set dst ($2 src1 src2));
1537   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1538   ins_cost(SVE_COST);
1539   format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction (sve)" %}
1540   ins_encode %{
1541     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1542     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1543                            $src1$$Register, as_FloatRegister($src2$$reg),
1544                            ptrue, as_FloatRegister($tmp$$reg));








1545   %}
1546   ins_pipe(pipe_slow);
1547 %}')dnl
1548 dnl
1549 dnl REDUCE_MAXMIN_L($1,        $2     )
1550 dnl REDUCE_MAXMIN_L(insn_name, op_name)
1551 define(`REDUCE_MAXMIN_L', `
1552 instruct reduce_$1L(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
1553   predicate(UseSVE > 0 &&
1554             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1555             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1556   match(Set dst ($2 src1 src2));
1557   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1558   ins_cost(SVE_COST);
1559   format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction (sve)" %}
1560   ins_encode %{
1561     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1562                            $src1$$Register, as_FloatRegister($src2$$reg),
1563                            ptrue, as_FloatRegister($tmp$$reg));
1564   %}
1565   ins_pipe(pipe_slow);
1566 %}')dnl
1567 dnl
1568 dnl REDUCE_MAXMIN_I_PARTIAL($1     , $2     )
1569 dnl REDUCE_MAXMIN_I_PARTIAL(min_max, op_name)
1570 define(`REDUCE_MAXMIN_I_PARTIAL', `
1571 instruct reduce_$1I_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1572                              pRegGov ptmp, rFlagsReg cr) %{
1573   predicate(UseSVE > 0 &&
1574             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1575             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1576             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
1577   match(Set dst ($2 src1 src2));
1578   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1579   ins_cost(2 * SVE_COST);
1580   format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction partial (sve)" %}
1581   ins_encode %{
1582     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1583     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1584     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1585                           Matcher::vector_length(this, $src2));
1586     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1587                            $src1$$Register, as_FloatRegister($src2$$reg),
1588                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));








1589   %}
1590   ins_pipe(pipe_slow);
1591 %}')dnl
1592 dnl
1593 dnl REDUCE_MAXMIN_L_PARTIAL($1     , $2     )
1594 dnl REDUCE_MAXMIN_L_PARTIAL(min_max, op_name)
1595 define(`REDUCE_MAXMIN_L_PARTIAL', `
1596 instruct reduce_$1L_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,












1597                              pRegGov ptmp, rFlagsReg cr) %{
1598   predicate(UseSVE > 0 &&
1599             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1600             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1601   match(Set dst ($2 src1 src2));
1602   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1603   ins_cost(2 * SVE_COST);
1604   format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction  partial (sve)" %}
1605   ins_encode %{
1606     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1607                           Matcher::vector_length(this, $src2));
1608     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1609                            $src1$$Register, as_FloatRegister($src2$$reg),
1610                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));

1611   %}
1612   ins_pipe(pipe_slow);
1613 %}')dnl

1614 dnl
1615 dnl REDUCE_MAXMIN_I_PREDICATE($1     , $2     )
1616 dnl REDUCE_MAXMIN_I_PREDICATE(min_max, op_name)
1617 define(`REDUCE_MAXMIN_I_PREDICATE', `
1618 instruct reduce_$1I_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp,
1619                            pRegGov pg, rFlagsReg cr) %{
1620   predicate(UseSVE > 0 &&
1621             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1622             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1623             is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
1624   match(Set dst ($2 (Binary src1 src2) pg));
1625   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1626   ins_cost(SVE_COST);
1627   format %{ "sve_reduce_$1I $dst, $src1, $pg, $src2\t# $1I reduction predicated (sve)" %}
1628   ins_encode %{
1629     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1630     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1631                            $src1$$Register, as_FloatRegister($src2$$reg),
1632                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));


1633   %}
1634   ins_pipe(pipe_slow);
1635 %}')dnl
1636 dnl
1637 dnl REDUCE_MAXMIN_L_PREDICATE($1     , $2     )
1638 dnl REDUCE_MAXMIN_L_PREDICATE(min_max, op_name)
1639 define(`REDUCE_MAXMIN_L_PREDICATE', `
1640 instruct reduce_$1L_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp,
1641                           pRegGov pg, rFlagsReg cr) %{
1642   predicate(UseSVE > 0 &&
1643             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1644             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1645   match(Set dst ($2 (Binary src1 src2) pg));
1646   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1647   ins_cost(SVE_COST);
1648   format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated (sve)" %}
1649   ins_encode %{
1650     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1651                            $src1$$Register, as_FloatRegister($src2$$reg),
1652                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));

1653   %}
1654   ins_pipe(pipe_slow);
1655 %}')dnl
1656 dnl
1657 dnl REDUCE_MAXMIN_I_PREDICATE_PARTIAL($1     , $2     )
1658 dnl REDUCE_MAXMIN_I_PREDICATE_PARTIAL(min_max, op_name)
1659 define(`REDUCE_MAXMIN_I_PREDICATE_PARTIAL', `
1660 instruct reduce_$1I_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1661                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
1662   predicate(UseSVE > 0 &&
1663             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1664             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1665             is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
1666   match(Set dst ($2 (Binary src1 src2) pg));
1667   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1668   ins_cost(3 * SVE_COST);
1669   format %{ "sve_reduce_$1I $dst, $src1, $pg, $src2\t# $1I reduction predicated partial (sve)" %}
1670   ins_encode %{
1671     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1672     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1673     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1674                           Matcher::vector_length(this, $src2));
1675     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1676                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1677     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1678                            $src1$$Register, as_FloatRegister($src2$$reg),
1679                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1680   %}
1681   ins_pipe(pipe_slow);
1682 %}')dnl
1683 dnl
1684 dnl REDUCE_MAXMIN_L_PREDICATE_PARTIAL($1     , $2     )
1685 dnl REDUCE_MAXMIN_L_PREDICATE_PARTIAL(min_max, op_name)
1686 define(`REDUCE_MAXMIN_L_PREDICATE_PARTIAL', `
1687 instruct reduce_$1L_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1688                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
1689   predicate(UseSVE > 0 &&
1690             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1691             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1692   match(Set dst ($2 (Binary src1 src2) pg));
1693   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1694   ins_cost(3 * SVE_COST);
1695   format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated partial (sve)" %}
1696   ins_encode %{
1697     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1698                           Matcher::vector_length(this, $src2));
1699     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1700                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1701     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1702                            $src1$$Register, as_FloatRegister($src2$$reg),
1703                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1704   %}
1705   ins_pipe(pipe_slow);
1706 %}')dnl
1707 dnl
1708 dnl REDUCE_FMINMAX($1,      $2,          $3,           $4,   $5         )
1709 dnl REDUCE_FMINMAX(min_max, name_suffix, element_type, size, reg_src_dst)
1710 define(`REDUCE_FMINMAX', `
1711 instruct reduce_$1$2($5 dst, $5 src1, vReg src2) %{
1712   predicate(UseSVE > 0 &&
1713             n->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1714             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1715   match(Set dst (translit($1, `m', `M')ReductionV src1 src2));
1716   ins_cost(INSN_COST);
1717   effect(TEMP_DEF dst);
1718   format %{ "sve_reduce_$1$2 $dst, $src1, $src2\t# $1$2 reduction (sve)" %}

1719   ins_encode %{
1720     __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src2$$reg));

1721     __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1722   %}
1723   ins_pipe(pipe_slow);
1724 %}')dnl
1725 dnl

1726 dnl REDUCE_FMINMAX_PARTIAL($1,      $2,          $3,           $4,   $5         )
1727 dnl REDUCE_FMINMAX_PARTIAL(min_max, name_suffix, element_type, size, reg_src_dst)
1728 define(`REDUCE_FMINMAX_PARTIAL', `
1729 instruct reduce_$1$2_partial($5 dst, $5 src1, vReg src2,
1730                              pRegGov ptmp, rFlagsReg cr) %{
1731   predicate(UseSVE > 0 &&
1732             n->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1733             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1734   match(Set dst (translit($1, `m', `M')ReductionV src1 src2));
1735   ins_cost(INSN_COST);
1736   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
1737   format %{ "sve_reduce_$1$2 $dst, $src1, $src2\t# $1$2 reduction partial (sve)" %}
1738   ins_encode %{
1739     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
1740                           Matcher::vector_length(this, $src2));
1741     __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1742     __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1743   %}
1744   ins_pipe(pipe_slow);
1745 %}')dnl
1746 dnl
1747 dnl REDUCE_FMINMAX_PREDICATE($1,      $2,          $3,           $4,   $5         )
1748 dnl REDUCE_FMINMAX_PREDICATE(min_max, name_suffix, element_type, size, reg_src_dst)
1749 define(`REDUCE_FMINMAX_PREDICATE', `
1750 instruct reduce_$1$2_masked($5 dst, $5 src1, vReg src2, pRegGov pg) %{
1751   predicate(UseSVE > 0 &&
1752             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1753             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1754   match(Set dst (translit($1, `m', `M')ReductionV (Binary src1 src2) pg));
1755   ins_cost(SVE_COST);
1756   format %{ "sve_reduce_$1$2 $dst, $src1, $pg, $src2\t# $1$2 reduction predicated (sve)" %}
1757   ins_encode %{
1758     __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1759     __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1760   %}
1761   ins_pipe(pipe_slow);
1762 %}')dnl
1763 dnl
1764 dnl REDUCE_FMINMAX_PREDICATE_PARTIAL($1,      $2,          $3,           $4,   $5         )
1765 dnl REDUCE_FMINMAX_PREDICATE_PARTIAL(min_max, name_suffix, element_type, size, reg_src_dst)
1766 define(`REDUCE_FMINMAX_PREDICATE_PARTIAL', `
1767 instruct reduce_$1$2_masked_partial($5 dst, $5 src1, vReg src2, pRegGov pg,
1768                                     pRegGov ptmp, rFlagsReg cr) %{
1769   predicate(UseSVE > 0 &&
1770             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1771             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1772   match(Set dst (translit($1, `m', `M')ReductionV (Binary src1 src2) pg));
1773   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
1774   ins_cost(3 * SVE_COST);
1775   format %{ "sve_reduce_$1$2 $dst, $src1, $pg, $src2\t# $1$2 reduction predicated partial (sve)" %}
1776   ins_encode %{
1777     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
1778                           Matcher::vector_length(this, $src2));
1779     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1780                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1781     __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4,
1782                as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1783     __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1784   %}
1785   ins_pipe(pipe_slow);
1786 %}')dnl

1787 // vector max reduction
1788 REDUCE_MAXMIN_I(max, MaxReductionV)
1789 REDUCE_MAXMIN_L(max, MaxReductionV)
1790 REDUCE_MAXMIN_I_PARTIAL(max, MaxReductionV)
1791 REDUCE_MAXMIN_L_PARTIAL(max, MaxReductionV)
1792 REDUCE_FMINMAX(max, F, T_FLOAT,  S, vRegF)
1793 REDUCE_FMINMAX_PARTIAL(max, F, T_FLOAT,  S, vRegF)
1794 REDUCE_FMINMAX(max, D, T_DOUBLE, D, vRegD)
1795 REDUCE_FMINMAX_PARTIAL(max, D, T_DOUBLE, D, vRegD)
1796 
1797 // vector max reduction - predicated
1798 REDUCE_MAXMIN_I_PREDICATE(max, MaxReductionV)
1799 REDUCE_MAXMIN_L_PREDICATE(max, MaxReductionV)
1800 REDUCE_MAXMIN_I_PREDICATE_PARTIAL(max, MaxReductionV)
1801 REDUCE_MAXMIN_L_PREDICATE_PARTIAL(max, MaxReductionV)
1802 REDUCE_FMINMAX_PREDICATE(max, F, T_FLOAT,  S, vRegF)
1803 REDUCE_FMINMAX_PREDICATE(max, D, T_DOUBLE, D, vRegD)
1804 REDUCE_FMINMAX_PREDICATE_PARTIAL(max, F, T_FLOAT,  S, vRegF)
1805 REDUCE_FMINMAX_PREDICATE_PARTIAL(max, D, T_DOUBLE, D, vRegD)
1806 
1807 // vector min reduction
1808 REDUCE_MAXMIN_I(min, MinReductionV)
1809 REDUCE_MAXMIN_L(min, MinReductionV)
1810 REDUCE_MAXMIN_I_PARTIAL(min, MinReductionV)
1811 REDUCE_MAXMIN_L_PARTIAL(min, MinReductionV)
1812 REDUCE_FMINMAX(min, F, T_FLOAT,  S, vRegF)
1813 REDUCE_FMINMAX_PARTIAL(min, F, T_FLOAT,  S, vRegF)
1814 REDUCE_FMINMAX(min, D, T_DOUBLE, D, vRegD)
1815 REDUCE_FMINMAX_PARTIAL(min, D, T_DOUBLE, D, vRegD)
1816 
1817 // vector min reduction - predicated
1818 REDUCE_MAXMIN_I_PREDICATE(min, MinReductionV)
1819 REDUCE_MAXMIN_L_PREDICATE(min, MinReductionV)
1820 REDUCE_MAXMIN_I_PREDICATE_PARTIAL(min, MinReductionV)
1821 REDUCE_MAXMIN_L_PREDICATE_PARTIAL(min, MinReductionV)
1822 REDUCE_FMINMAX_PREDICATE(min, F, T_FLOAT,  S, vRegF)
1823 REDUCE_FMINMAX_PREDICATE(min, D, T_DOUBLE, D, vRegD)
1824 REDUCE_FMINMAX_PREDICATE_PARTIAL(min, F, T_FLOAT,  S, vRegF)
1825 REDUCE_FMINMAX_PREDICATE_PARTIAL(min, D, T_DOUBLE, D, vRegD)
1826 
1827 // vector Math.rint, floor, ceil
1828 
1829 instruct vroundD(vReg dst, vReg src, immI rmode) %{
1830   predicate(UseSVE > 0 &&
1831             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
1832   match(Set dst (RoundDoubleModeV src rmode));
1833   format %{ "sve_frint $dst, $src, $rmode\t# vector (sve) (D)" %}
1834   ins_encode %{
1835     switch ($rmode$$constant) {
1836       case RoundDoubleModeNode::rmode_rint:
1837         __ sve_frintn(as_FloatRegister($dst$$reg), __ D,
1838              ptrue, as_FloatRegister($src$$reg));
1839         break;
1840       case RoundDoubleModeNode::rmode_floor:
1841         __ sve_frintm(as_FloatRegister($dst$$reg), __ D,
1842              ptrue, as_FloatRegister($src$$reg));
1843         break;
1844       case RoundDoubleModeNode::rmode_ceil:
1845         __ sve_frintp(as_FloatRegister($dst$$reg), __ D,
1846              ptrue, as_FloatRegister($src$$reg));

1984 VSHIFT_TRUE_PREDICATE(vlsrS, URShiftVS, H,  8, sve_lsr)
1985 VSHIFT_TRUE_PREDICATE(vlsrI, URShiftVI, S,  4, sve_lsr)
1986 VSHIFT_TRUE_PREDICATE(vlsrL, URShiftVL, D,  2, sve_lsr)
1987 VSHIFT_IMM_UNPREDICATED(vasrB_imm, RShiftVB,  RShiftCntV, B, 16, sve_asr)
1988 VSHIFT_IMM_UNPREDICATED(vasrS_imm, RShiftVS,  RShiftCntV, H,  8, sve_asr)
1989 VSHIFT_IMM_UNPREDICATED(vasrI_imm, RShiftVI,  RShiftCntV, S,  4, sve_asr)
1990 VSHIFT_IMM_UNPREDICATED(vasrL_imm, RShiftVL,  RShiftCntV, D,  2, sve_asr)
1991 VSHIFT_IMM_UNPREDICATED(vlsrB_imm, URShiftVB, RShiftCntV, B, 16, sve_lsr)
1992 VSHIFT_IMM_UNPREDICATED(vlsrS_imm, URShiftVS, RShiftCntV, H,  8, sve_lsr)
1993 VSHIFT_IMM_UNPREDICATED(vlsrI_imm, URShiftVI, RShiftCntV, S,  4, sve_lsr)
1994 VSHIFT_IMM_UNPREDICATED(vlsrL_imm, URShiftVL, RShiftCntV, D,  2, sve_lsr)
1995 VSHIFT_IMM_UNPREDICATED(vlslB_imm, LShiftVB,  LShiftCntV, B, 16, sve_lsl)
1996 VSHIFT_IMM_UNPREDICATED(vlslS_imm, LShiftVS,  LShiftCntV, H,  8, sve_lsl)
1997 VSHIFT_IMM_UNPREDICATED(vlslI_imm, LShiftVI,  LShiftCntV, S,  4, sve_lsl)
1998 VSHIFT_IMM_UNPREDICATED(vlslL_imm, LShiftVL,  LShiftCntV, D,  2, sve_lsl)
1999 VSHIFT_COUNT(vshiftcntB, B, 16, T_BYTE)
2000 VSHIFT_COUNT(vshiftcntS, H,  8, T_SHORT)
2001 VSHIFT_COUNT(vshiftcntI, S,  4, T_INT)
2002 VSHIFT_COUNT(vshiftcntL, D,  2, T_LONG)
2003 
2004 // vector shift - predicated
2005 BINARY_OP_PREDICATE(vasrB, RShiftVB,  B, sve_asr)
2006 BINARY_OP_PREDICATE(vasrS, RShiftVS,  H, sve_asr)
2007 BINARY_OP_PREDICATE(vasrI, RShiftVI,  S, sve_asr)
2008 BINARY_OP_PREDICATE(vasrL, RShiftVL,  D, sve_asr)
2009 BINARY_OP_PREDICATE(vlslB, LShiftVB,  B, sve_lsl)
2010 BINARY_OP_PREDICATE(vlslS, LShiftVS,  H, sve_lsl)
2011 BINARY_OP_PREDICATE(vlslI, LShiftVI,  S, sve_lsl)
2012 BINARY_OP_PREDICATE(vlslL, LShiftVL,  D, sve_lsl)
2013 BINARY_OP_PREDICATE(vlsrB, URShiftVB, B, sve_lsr)
2014 BINARY_OP_PREDICATE(vlsrS, URShiftVS, H, sve_lsr)
2015 BINARY_OP_PREDICATE(vlsrI, URShiftVI, S, sve_lsr)
2016 BINARY_OP_PREDICATE(vlsrL, URShiftVL, D, sve_lsr)
2017 dnl
2018 dnl VSHIFT_IMM_PREDICATED($1,        $2,      $3,       $4,   $5,   $6  )
2019 dnl VSHIFT_IMM_PREDICATED(insn_name, op_name, op_name2, type, size, insn)
2020 define(`VSHIFT_IMM_PREDICATED', `
2021 instruct $1_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
2022   predicate(UseSVE > 0);
2023   match(Set dst_src ($2 (Binary dst_src ($3 shift)) pg));
2024   ins_cost(SVE_COST);
2025   format %{ "$6 $dst_src, $pg, $dst_src, $shift\t# vector (sve) ($4)" %}
2026   ins_encode %{
2027     int con = (int)$shift$$constant;
2028     assert(con ifelse(index(`$1', `vlsl'), 0, `>=', `>') 0 && con < $5, "invalid shift immediate");
2029     __ $6(as_FloatRegister($dst_src$$reg), __ $4, as_PRegister($pg$$reg), con);
2030   %}
2031   ins_pipe(pipe_slow);
2032 %}')dnl
2033 dnl
2034 VSHIFT_IMM_PREDICATED(vasrB, RShiftVB,  RShiftCntV, B, 8,  sve_asr)
2035 VSHIFT_IMM_PREDICATED(vasrS, RShiftVS,  RShiftCntV, H, 16, sve_asr)
2036 VSHIFT_IMM_PREDICATED(vasrI, RShiftVI,  RShiftCntV, S, 32, sve_asr)
2037 VSHIFT_IMM_PREDICATED(vasrL, RShiftVL,  RShiftCntV, D, 64, sve_asr)
2038 VSHIFT_IMM_PREDICATED(vlsrB, URShiftVB, RShiftCntV, B, 8,  sve_lsr)
2039 VSHIFT_IMM_PREDICATED(vlsrS, URShiftVS, RShiftCntV, H, 16, sve_lsr)
2040 VSHIFT_IMM_PREDICATED(vlsrI, URShiftVI, RShiftCntV, S, 32, sve_lsr)
2041 VSHIFT_IMM_PREDICATED(vlsrL, URShiftVL, RShiftCntV, D, 64, sve_lsr)
2042 VSHIFT_IMM_PREDICATED(vlslB, LShiftVB,  LShiftCntV, B, 8,  sve_lsl)
2043 VSHIFT_IMM_PREDICATED(vlslS, LShiftVS,  LShiftCntV, H, 16, sve_lsl)
2044 VSHIFT_IMM_PREDICATED(vlslI, LShiftVI,  LShiftCntV, S, 32, sve_lsl)
2045 VSHIFT_IMM_PREDICATED(vlslL, LShiftVL,  LShiftCntV, D, 64, sve_lsl)
2046 
2047 // vector sqrt
2048 UNARY_OP_TRUE_PREDICATE(vsqrtF, SqrtVF, S, sve_fsqrt)
2049 UNARY_OP_TRUE_PREDICATE(vsqrtD, SqrtVD, D, sve_fsqrt)




2050 
2051 // vector sqrt - predicated
2052 UNARY_OP_PREDICATE(vsqrtF, SqrtVF, S, sve_fsqrt)
2053 UNARY_OP_PREDICATE(vsqrtD, SqrtVD, D, sve_fsqrt)
2054 
2055 // vector sub
2056 BINARY_OP_UNPREDICATE(vsubB, SubVB, B, 16, sve_sub)
2057 BINARY_OP_UNPREDICATE(vsubS, SubVS, H, 8, sve_sub)
2058 BINARY_OP_UNPREDICATE(vsubI, SubVI, S, 4, sve_sub)
2059 BINARY_OP_UNPREDICATE(vsubL, SubVL, D, 2, sve_sub)
2060 BINARY_OP_UNPREDICATE(vsubF, SubVF, S, 4, sve_fsub)
2061 BINARY_OP_UNPREDICATE(vsubD, SubVD, D, 2, sve_fsub)
2062 
2063 // vector sub - predicated
2064 BINARY_OP_PREDICATE(vsubB, SubVB, B, sve_sub)
2065 BINARY_OP_PREDICATE(vsubS, SubVS, H, sve_sub)
2066 BINARY_OP_PREDICATE(vsubI, SubVI, S, sve_sub)
2067 BINARY_OP_PREDICATE(vsubL, SubVL, D, sve_sub)
2068 BINARY_OP_PREDICATE(vsubF, SubVF, S, sve_fsub)
2069 BINARY_OP_PREDICATE(vsubD, SubVD, D, sve_fsub)
2070 
2071 // ------------------------------ Vector mask cast --------------------------
2072 
2073 instruct vmaskcast(pRegGov dst_src) %{
2074   predicate(UseSVE > 0 &&
2075             n->bottom_type()->is_vect()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
2076             n->bottom_type()->is_vect()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
2077   match(Set dst_src (VectorMaskCast dst_src));
2078   ins_cost(0);
2079   format %{ "vmaskcast $dst_src\t# empty (sve)" %}
2080   ins_encode %{
2081     // empty
2082   %}
2083   ins_pipe(pipe_class_empty);
2084 %}
2085 
2086 instruct vmaskcast_extend(pRegGov dst, pReg src)
2087 %{
2088   predicate(UseSVE > 0 &&
2089             (Matcher::vector_length_in_bytes(n) == 2 * Matcher::vector_length_in_bytes(n->in(1)) ||
2090              Matcher::vector_length_in_bytes(n) == 4 * Matcher::vector_length_in_bytes(n->in(1)) ||
2091              Matcher::vector_length_in_bytes(n) == 8 * Matcher::vector_length_in_bytes(n->in(1))));
2092   match(Set dst (VectorMaskCast src));
2093   ins_cost(SVE_COST * 3);
2094   format %{ "sve_vmaskcast_extend  $dst, $src\t# extend predicate $src" %}
2095   ins_encode %{
2096     __ sve_vmaskcast_extend(as_PRegister($dst$$reg), as_PRegister($src$$reg),
2097                             Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
2098   %}
2099   ins_pipe(pipe_slow);
2100 %}
2101 
2102 instruct vmaskcast_narrow(pRegGov dst, pReg src)
2103 %{
2104   predicate(UseSVE > 0 &&
2105             (Matcher::vector_length_in_bytes(n) * 2 == Matcher::vector_length_in_bytes(n->in(1)) ||
2106              Matcher::vector_length_in_bytes(n) * 4 == Matcher::vector_length_in_bytes(n->in(1)) ||
2107              Matcher::vector_length_in_bytes(n) * 8 == Matcher::vector_length_in_bytes(n->in(1))));
2108   match(Set dst (VectorMaskCast src));
2109   ins_cost(SVE_COST * 3);
2110   format %{ "sve_vmaskcast_narrow  $dst, $src\t# narrow predicate $src" %}
2111   ins_encode %{
2112     __ sve_vmaskcast_narrow(as_PRegister($dst$$reg), as_PRegister($src$$reg),
2113                             Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
2114   %}
2115   ins_pipe(pipe_slow);
2116 %}
2117 dnl
2118 
2119 // ------------------------------ Vector cast -------------------------------
2120 dnl
2121 dnl
2122 define(`VECTOR_CAST_X2X', `
2123 instruct vcvt$1to$2`'(vReg dst, vReg src)
2124 %{
2125   predicate(UseSVE > 0 &&
2126             n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
2127   match(Set dst (VectorCast$1`'2X src));
2128   ins_cost(SVE_COST);
2129   format %{ "sve_vectorcast_$5  $dst, $src\t# convert $1 to $2 vector" %}
2130   ins_encode %{
2131     __ sve_$3(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src$$reg), __ $4);
2132   %}
2133   ins_pipe(pipe_slow);
2134 %}')dnl
2135 
2136 dnl
2137 dnl Start of vector cast rules
2138 dnl
2139 instruct vcvtBtoX_extend(vReg dst, vReg src)

2140 %{
2141   predicate(UseSVE > 0);
2142   match(Set dst (VectorCastB2X src));

2143   ins_cost(2 * SVE_COST);
2144   format %{ "sve_vectorcast_b2x  $dst, $src\t# convert B to X vector (extend)" %}

2145   ins_encode %{
2146     BasicType to_bt = Matcher::vector_element_basic_type(this);
2147     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2148     __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ B);
2149     if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
2150       __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
2151     }
2152   %}
2153   ins_pipe(pipe_slow);
2154 %}
2155 
2156 instruct vcvtStoB(vReg dst, vReg src, vReg tmp)


2157 %{
2158   predicate(UseSVE > 0 &&
2159             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2160   match(Set dst (VectorCastS2X src));
2161   effect(TEMP tmp);
2162   ins_cost(2 * SVE_COST);
2163   format %{ "sve_vectorcast_s2b  $dst, $src\t# convert H to B vector" %}

2164   ins_encode %{
2165     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
2166                          as_FloatRegister($src$$reg), __ H, as_FloatRegister($tmp$$reg));

2167   %}
2168   ins_pipe(pipe_slow);
2169 %}
2170 
2171 instruct vcvtStoX_extend(vReg dst, vReg src)


2172 %{
2173   predicate(UseSVE > 0 &&
2174             type2aelembytes(Matcher::vector_element_basic_type(n)) > 2);
2175   match(Set dst (VectorCastS2X src));

2176   ins_cost(2 * SVE_COST);
2177   format %{ "sve_vectorcast_s2x  $dst, $src\t# convert H to X vector (extend)" %}

2178   ins_encode %{
2179     BasicType to_bt = Matcher::vector_element_basic_type(this);
2180     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2181     __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ H);
2182     if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
2183       __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
2184     }
2185   %}
2186   ins_pipe(pipe_slow);
2187 %}
2188 
2189 instruct vcvtItoB(vReg dst, vReg src, vReg tmp)


2190 %{
2191   predicate(UseSVE > 0 &&
2192             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2193   match(Set dst (VectorCastI2X src));
2194   effect(TEMP_DEF dst, TEMP tmp);
2195   ins_cost(3 * SVE_COST);
2196   format %{ "sve_vectorcast_i2b  $dst, $src\t# convert I to B vector" %}


2197   ins_encode %{
2198     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
2199                          as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));

2200   %}
2201   ins_pipe(pipe_slow);
2202 %}
2203 
2204 instruct vcvtItoS(vReg dst, vReg src, vReg tmp)


2205 %{
2206   predicate(UseSVE > 0 &&
2207             n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
2208   match(Set dst (VectorCastI2X src));
2209   effect(TEMP tmp);
2210   ins_cost(2 * SVE_COST);
2211   format %{ "sve_vectorcast_i2s $dst, $src\t# convert I to H vector" %}



2212   ins_encode %{
2213     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ H,
2214                          as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));


2215   %}
2216   ins_pipe(pipe_slow);
2217 %}
2218 
2219 instruct vcvtItoL(vReg dst, vReg src)


2220 %{
2221   predicate(UseSVE > 0 &&
2222             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2223   match(Set dst (VectorCastI2X src));
2224   ins_cost(SVE_COST);
2225   format %{ "sve_vectorcast_i2l  $dst, $src\t# convert I to L vector" %}


2226   ins_encode %{
2227     __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);


2228   %}
2229   ins_pipe(pipe_slow);
2230 %}

2231 dnl
2232 dnl vcvtItoF
2233 VECTOR_CAST_X2X(I, F, scvtf, S, i2f)
2234 
2235 instruct vcvtItoD(vReg dst, vReg src)
2236 %{
2237   predicate(UseSVE > 0 &&
2238             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
2239   match(Set dst (VectorCastI2X src));
2240   ins_cost(2 * SVE_COST);
2241   format %{ "sve_vectorcast_i2d  $dst, $src\t# convert I to D vector" %}



2242   ins_encode %{
2243     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
2244     __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ D);


2245   %}
2246   ins_pipe(pipe_slow);
2247 %}
2248 
2249 instruct vcvtLtoX_narrow(vReg dst, vReg src, vReg tmp)


2250 %{
2251   predicate(UseSVE > 0 && is_integral_type(Matcher::vector_element_basic_type(n)));
2252   match(Set dst (VectorCastL2X src));

2253   effect(TEMP_DEF dst, TEMP tmp);
2254   ins_cost(2 * SVE_COST);
2255   format %{ "sve_vectorcast_l2x  $dst, $src\t# convert L to B/H/S vector (narrow)" %}


2256   ins_encode %{
2257     BasicType to_bt = Matcher::vector_element_basic_type(this);
2258     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2259     __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
2260                          as_FloatRegister($src$$reg), __ D, as_FloatRegister($tmp$$reg));
2261   %}
2262   ins_pipe(pipe_slow);
2263 %}
2264 
2265 instruct vcvtLtoF(vReg dst, vReg src, vReg tmp)


2266 %{
2267   predicate(UseSVE > 0 &&
2268             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2269   match(Set dst (VectorCastL2X src));
2270   effect(TEMP_DEF dst, TEMP tmp);
2271   ins_cost(3 * SVE_COST);
2272   format %{ "sve_vectorcast_l2f  $dst, $src\t# convert L to F vector" %}
2273   ins_encode %{
2274     __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
2275     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
2276                          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
2277 
2278   %}
2279   ins_pipe(pipe_slow);
2280 %}

2281 dnl
2282 dnl vcvtLtoD
2283 VECTOR_CAST_X2X(L, D, scvtf, D, l2d)
2284 
2285 instruct vcvtFtoX_narrow(vReg dst, vReg src, vReg tmp)
2286 %{
2287   predicate(UseSVE > 0 &&
2288             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2289              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT));
2290   match(Set dst (VectorCastF2X src));
2291   effect(TEMP_DEF dst, TEMP tmp);
2292   ins_cost(3 * SVE_COST);
2293   format %{ "sve_vectorcast_f2x  $dst, $src\t# convert F to B/H vector" %}
2294   ins_encode %{
2295     BasicType to_bt = Matcher::vector_element_basic_type(this);
2296     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2297     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
2298     __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
2299                          as_FloatRegister($dst$$reg), __ S, as_FloatRegister($tmp$$reg));
2300   %}
2301   ins_pipe(pipe_slow);
2302 %}
2303 
2304 instruct vcvtFtoX_extend(vReg dst, vReg src)


2305 %{
2306   predicate(UseSVE > 0 &&
2307             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2308              n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
2309   match(Set dst (VectorCastF2X src));
2310   ins_cost(SVE_COST);
2311   format %{ "sve_vectorcast_f2x  $dst, $src\t# convert F to I/L vector" %}


2312   ins_encode %{
2313     BasicType to_bt = Matcher::vector_element_basic_type(this);
2314     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
2315     if (to_bt == T_LONG) {
2316       __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($dst$$reg), __ S);
2317     }
2318   %}
2319   ins_pipe(pipe_slow);
2320 %}
2321 
2322 instruct vcvtFtoD(vReg dst, vReg src)


2323 %{
2324   predicate(UseSVE > 0 &&
2325             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
2326   match(Set dst (VectorCastF2X src));
2327   ins_cost(2 * SVE_COST);
2328   format %{ "sve_vectorcast_f2d  $dst, $dst\t# convert F to D vector" %}




2329   ins_encode %{
2330     __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
2331     __ sve_fcvt(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);


2332   %}
2333   ins_pipe(pipe_slow);
2334 %}
2335 
2336 instruct vcvtDtoX_narrow(vReg dst, vReg src, vReg tmp)


2337 %{
2338   predicate(UseSVE > 0 &&
2339             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2340              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2341              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2342   match(Set dst (VectorCastD2X src));
2343   effect(TEMP_DEF dst, TEMP tmp);
2344   ins_cost(3 * SVE_COST);
2345   format %{ "sve_vectorcast_d2x  $dst, $src\t# convert D to X vector (narrow)" %}
2346   ins_encode %{
2347     BasicType to_bt = Matcher::vector_element_basic_type(this);
2348     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2349     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
2350     __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
2351                          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
2352   %}
2353   ins_pipe(pipe_slow);
2354 %}

2355 dnl
2356 dnl vcvtDtoL
2357 VECTOR_CAST_X2X(D, L, fcvtzs, D, d2l)
2358 
2359 instruct vcvtDtoF(vReg dst, vReg src, vReg tmp)
2360 %{
2361   predicate(UseSVE > 0 &&
2362             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2363   match(Set dst (VectorCastD2X src));
2364   effect(TEMP_DEF dst, TEMP tmp);
2365   ins_cost(3 * SVE_COST);
2366   format %{ "sve_vectorcast_d2f  $dst, S, $dst\t# convert D to F vector" %}




2367   ins_encode %{
2368     __ sve_fcvt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
2369     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
2370                          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));


2371   %}
2372   ins_pipe(pipe_slow);
2373 %}
2374 



































2375 dnl
2376 dnl
2377 // ------------------------------ Vector extract ---------------------------------
2378 define(`VECTOR_EXTRACT_SXT', `
2379 instruct extract$1`'($2 dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
2380 %{
2381   predicate(UseSVE > 0);
2382   match(Set dst (Extract$1 src idx));
2383   effect(TEMP pgtmp, KILL cr);
2384   ins_cost(2 * SVE_COST);
2385   format %{ "sve_extract $dst, $3, $pgtmp, $src, $idx\n\t"
2386             "sbfmw $dst, $dst, 0U, $5\t# extract from vector($1)" %}
2387   ins_encode %{
2388     __ sve_extract(as_$4($dst$$reg), __ $3, as_PRegister($pgtmp$$reg),
2389                    as_FloatRegister($src$$reg), (int)($idx$$constant));
2390     __ sbfmw(as_$4($dst$$reg), as_$4($dst$$reg), 0U, $5);
2391   %}
2392   ins_pipe(pipe_slow);
2393 %}')dnl
2394 dnl                $1 $2         $3 $4        $5
2395 VECTOR_EXTRACT_SXT(B, iRegINoSp, B, Register, 7U)
2396 VECTOR_EXTRACT_SXT(S, iRegINoSp, H, Register, 15U)
2397 
2398 dnl
2399 define(`VECTOR_EXTRACT', `
2400 instruct extract$1`'($2 dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
2401 %{
2402   predicate(UseSVE > 0);
2403   match(Set dst (Extract$1 src idx));
2404   effect(TEMP pgtmp, KILL cr);
2405   ins_cost(2 * SVE_COST);
2406   format %{ "sve_extract $dst, $3, $pgtmp, $src, $idx\t# extract from vector($1)" %}
2407   ins_encode %{
2408     __ sve_extract(as_$4($dst$$reg), __ $3, as_PRegister($pgtmp$$reg),
2409                    as_FloatRegister($src$$reg), (int)($idx$$constant));
2410   %}
2411   ins_pipe(pipe_slow);
2412 %}')dnl
2413 dnl            $1 $2         $3 $4
2414 VECTOR_EXTRACT(I, iRegINoSp, S, Register)
2415 VECTOR_EXTRACT(L, iRegLNoSp, D, Register)
2416 VECTOR_EXTRACT(F, vRegF,     S, FloatRegister)
2417 VECTOR_EXTRACT(D, vRegD,     D, FloatRegister)
2418 
2419 // ------------------------------- VectorTest ----------------------------------
2420 
2421 instruct vtest_alltrue(iRegINoSp dst, pRegGov src1, pRegGov src2, pReg ptmp, rFlagsReg cr)



2422 %{
2423   predicate(UseSVE > 0 &&
2424             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2425             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
2426   match(Set dst (VectorTest src1 src2));
2427   effect(TEMP ptmp, KILL cr);
2428   ins_cost(SVE_COST);
2429   format %{ "sve_eors $ptmp, $src1, $src2\t# $src2 is all true mask\n"
2430             "csetw $dst, EQ\t# VectorTest (sve) - alltrue" %}
2431   ins_encode %{
2432     __ sve_eors(as_PRegister($ptmp$$reg), ptrue,
2433                 as_PRegister($src1$$reg), as_PRegister($src2$$reg));
2434     __ csetw(as_Register($dst$$reg), Assembler::EQ);
2435   %}
2436   ins_pipe(pipe_slow);
2437 %}
2438 
2439 instruct vtest_anytrue(iRegINoSp dst, pRegGov src1, pRegGov src2, rFlagsReg cr)
2440 %{
2441   predicate(UseSVE > 0 &&
2442             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2443             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
2444   match(Set dst (VectorTest src1 src2));
2445   effect(KILL cr);
2446   ins_cost(SVE_COST);
2447   format %{ "sve_ptest $src1\n\t"
2448             "csetw $dst, NE\t# VectorTest (sve) - anytrue" %}
2449   ins_encode %{
2450     // "src2" is not used for sve.
2451     __ sve_ptest(ptrue, as_PRegister($src1$$reg));
2452     __ csetw(as_Register($dst$$reg), Assembler::NE);



2453   %}
2454   ins_pipe(pipe_slow);
2455 %}



2456 dnl
2457 dnl
2458 dnl VTEST_PARTIAL($1,      $2,   $3,   $4  )
2459 dnl VTEST_PARTIAL(op_name, pred, inst, cond)
2460 define(`VTEST_PARTIAL', `
2461 instruct vtest_$1_partial`'(iRegINoSp dst, pRegGov src1, pRegGov src2, pRegGov ptmp, rFlagsReg cr)
2462 %{
2463   predicate(UseSVE > 0 &&
2464             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2465             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::$2);
2466   match(Set dst (VectorTest src1 src2));
2467   effect(TEMP ptmp, KILL cr);
2468   ins_cost(SVE_COST);
2469   format %{ "vtest_$1_partial $dst, $src1, $src2\t# VectorTest partial (sve) - $1" %}
2470   ins_encode %{

2471     BasicType bt = Matcher::vector_element_basic_type(this, $src1);
2472     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2473     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size,
2474                           Matcher::vector_length(this, $src1));
2475     __ $3(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2476           as_PRegister($src1$$reg), as_PRegister($src2$$reg));
2477     __ csetw(as_Register($dst$$reg), Assembler::$4);
2478   %}
2479   ins_pipe(pipe_slow);
2480 %}')dnl
2481 dnl
2482 VTEST_PARTIAL(alltrue, overflow, sve_eors, EQ)
2483 VTEST_PARTIAL(anytrue, ne,       sve_ands, NE)
2484 
2485 // ------------------------------ Vector insert ---------------------------------
2486 
2487 instruct insertI_small(vReg dst, vReg src, iRegIorL2I val, immI idx, pRegGov pgtmp, rFlagsReg cr)
2488 %{
2489   predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
2490             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2491              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2492              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2493   match(Set dst (VectorInsert (Binary src val) idx));
2494   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
2495   ins_cost(4 * SVE_COST);
2496   format %{ "sve_index $dst, -16, 1\t# (B/H/S)\n\t"
2497             "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2498             "sve_orr $dst, $src, $src\n\t"
2499             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
2500   ins_encode %{
2501     BasicType bt = Matcher::vector_element_basic_type(this, $src);
2502     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2503     __ sve_index(as_FloatRegister($dst$$reg), size, -16, 1);
2504     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
2505                as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2506     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2507     __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
2508   %}
2509   ins_pipe(pipe_slow);
2510 %}
2511 
2512 instruct insertF_small(vReg dst, vReg src, vRegF val, immI idx, pRegGov pgtmp, rFlagsReg cr)
2513 %{
2514   predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
2515             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2516   match(Set dst (VectorInsert (Binary src val) idx));
2517   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
2518   ins_cost(4 * SVE_COST);
2519   format %{ "sve_index $dst, S, -16, 1\n\t"
2520             "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2521             "sve_orr $dst, $src, $src\n\t"
2522             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
2523   ins_encode %{
2524     __ sve_index(as_FloatRegister($dst$$reg), __ S, -16, 1);
2525     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
2526                as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2527     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2528     __ sve_cpy(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
2529   %}
2530   ins_pipe(pipe_slow);
2531 %}
2532 
2533 instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr)
2534 %{
2535   predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
2536             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2537              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2538              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2539   match(Set dst (VectorInsert (Binary src val) idx));
2540   effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
2541   ins_cost(5 * SVE_COST);
2542   format %{ "sve_index $tmp1, 0, 1\t# (B/H/S)\n\t"
2543             "sve_dup $dst, $idx\t# (B/H/S)\n\t"
2544             "sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
2545             "sve_orr $dst, $src, $src\n\t"
2546             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
2547   ins_encode %{
2548     BasicType bt = Matcher::vector_element_basic_type(this, $src);
2549     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2550     __ sve_index(as_FloatRegister($tmp1$$reg), size, 0, 1);
2551     __ sve_dup(as_FloatRegister($dst$$reg), size, (int)($idx$$constant));
2552     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
2553                as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
2554     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2555     __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
2556   %}
2557   ins_pipe(pipe_slow);
2558 %}
2559 dnl
2560 dnl
2561 define(`VECTOR_INSERT_D', `
2562 instruct insert$1`'(vReg dst, vReg src, $2 val, immI idx, pRegGov pgtmp, rFlagsReg cr)
2563 %{
2564   predicate(UseSVE > 0 &&
2565             n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($1));
2566   match(Set dst (VectorInsert (Binary src val) idx));
2567   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
2568   ins_cost(4 * SVE_COST);
2569   format %{ "sve_index $dst, $3, -16, 1\n\t"
2570             "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2571             "sve_orr $dst, $src, $src\n\t"
2572             "sve_cpy $dst, $pgtmp, $val\t# insert into vector ($1)" %}
2573   ins_encode %{
2574     __ sve_index(as_FloatRegister($dst$$reg), __ $3, -16, 1);
2575     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ $3, ptrue,
2576                as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2577     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2578     __ sve_cpy(as_FloatRegister($dst$$reg), __ $3, as_PRegister($pgtmp$$reg), as_$4($val$$reg));
2579   %}
2580   ins_pipe(pipe_slow);
2581 %}')dnl
2582 dnl             $1 $2     $3 $4
2583 VECTOR_INSERT_D(L, iRegL, D, Register)
2584 VECTOR_INSERT_D(D, vRegD, D, FloatRegister)
2585 
2586 instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr)
2587 %{
2588   predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
2589             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2590   match(Set dst (VectorInsert (Binary src val) idx));
2591   effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
2592   ins_cost(5 * SVE_COST);
2593   format %{ "sve_index $tmp1, S, 0, 1\n\t"
2594             "sve_dup $dst, S, $idx\n\t"
2595             "sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
2596             "sve_orr $dst, $src, $src\n\t"
2597             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
2598   ins_encode %{
2599     __ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
2600     __ sve_dup(as_FloatRegister($dst$$reg), __ S, (int)($idx$$constant));
2601     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
2602                as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
2603     __ sve_orr(as_FloatRegister($dst$$reg),
2604                as_FloatRegister($src$$reg),
2605                as_FloatRegister($src$$reg));
2606     __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
2607                as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
2608   %}
2609   ins_pipe(pipe_slow);
2610 %}
2611 
2612 // ------------------------------ Vector shuffle -------------------------------
2613 
2614 instruct loadshuffle(vReg dst, vReg src) %{
2615   predicate(UseSVE > 0);

2616   match(Set dst (VectorLoadShuffle src));
2617   ins_cost(SVE_COST);
2618   format %{ "sve_loadshuffle $dst, $src\t# vector load shuffle (B/H/S/D)" %}
2619   ins_encode %{
2620     BasicType bt = Matcher::vector_element_basic_type(this);
2621     if (bt == T_BYTE) {
2622       if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
2623         __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
2624                    as_FloatRegister($src$$reg));
2625       }
2626     } else {
2627       __ sve_vector_extend(as_FloatRegister($dst$$reg),  __ elemType_to_regVariant(bt),
2628                            as_FloatRegister($src$$reg), __ B);
2629     }
2630   %}
2631   ins_pipe(pipe_slow);
2632 %}
2633 














































2634 // ------------------------------ Vector rearrange -------------------------------
2635 
2636 instruct rearrange(vReg dst, vReg src, vReg shuffle)
2637 %{
2638   predicate(UseSVE > 0);
2639   match(Set dst (VectorRearrange src shuffle));
2640   ins_cost(SVE_COST);
2641   format %{ "sve_tbl $dst, $src, $shuffle\t# vector rearrange" %}
2642   ins_encode %{
2643     BasicType bt = Matcher::vector_element_basic_type(this, $src);
2644     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2645     __ sve_tbl(as_FloatRegister($dst$$reg), size,
2646                as_FloatRegister($src$$reg), as_FloatRegister($shuffle$$reg));
2647   %}
2648   ins_pipe(pipe_slow);
2649 %}
2650 
2651 // ------------------------------ Vector Load Gather ---------------------------------
2652 
2653 instruct gatherI(vReg dst, indirect mem, vReg idx) %{
2654   predicate(UseSVE > 0 &&
2655             n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
2656             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2657              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2658   match(Set dst (LoadVectorGather mem idx));
2659   ins_cost(SVE_COST);
2660   format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (S)" %}
2661   ins_encode %{
2662     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), ptrue,
2663                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2664   %}
2665   ins_pipe(pipe_slow);
2666 %}
2667 
2668 instruct gatherL(vReg dst, indirect mem, vReg idx) %{
2669   predicate(UseSVE > 0 &&
2670             n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
2671             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2672              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2673   match(Set dst (LoadVectorGather mem idx));
2674   ins_cost(2 * SVE_COST);
2675   format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (D)" %}

2676   ins_encode %{
2677     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2678     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), ptrue, as_Register($mem$$base),
2679                        as_FloatRegister($idx$$reg));
2680   %}
2681   ins_pipe(pipe_slow);
2682 %}
2683 
2684 // ------------------------------ Vector Load Gather Partial-------------------------------
2685 
2686 instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2687   predicate(UseSVE > 0 &&
2688             n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
2689             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2690              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2691   match(Set dst (LoadVectorGather mem idx));
2692   effect(TEMP ptmp, KILL cr);
2693   ins_cost(2 * SVE_COST + INSN_COST);
2694   format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (S)" %}

2695   ins_encode %{
2696     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this));
2697     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),

2698                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2699   %}
2700   ins_pipe(pipe_slow);
2701 %}
2702 
2703 instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2704   predicate(UseSVE > 0 &&
2705             n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
2706             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2707              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2708   match(Set dst (LoadVectorGather mem idx));
2709   effect(TEMP ptmp, KILL cr);
2710   ins_cost(3 * SVE_COST + INSN_COST);
2711   format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (D)" %}
2712   ins_encode %{
2713     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2714                           Matcher::vector_length(this));
2715     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2716     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
2717                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2718   %}
2719   ins_pipe(pipe_slow);
2720 %}
2721 
2722 // ------------------------------ Vector Load Gather Predicated -------------------------------
2723 
2724 instruct gatherI_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
2725   predicate(UseSVE > 0 &&
2726             n->as_LoadVector()->memory_size() == MaxVectorSize &&
2727             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2728              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2729   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
2730   ins_cost(SVE_COST);
2731   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (S)" %}
2732   ins_encode %{
2733     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
2734                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2735   %}
2736   ins_pipe(pipe_slow);
2737 %}
2738 
2739 instruct gatherL_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
2740   predicate(UseSVE > 0 &&
2741             n->as_LoadVector()->memory_size() == MaxVectorSize &&
2742             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2743              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2744   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
2745   ins_cost(2 * SVE_COST);
2746   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (D)" %}
2747   ins_encode %{
2748     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2749     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
2750                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2751   %}
2752   ins_pipe(pipe_slow);
2753 %}
2754 
2755 // ------------------------------ Vector Load Gather Predicated Partial -------------------------------
2756 
2757 instruct gatherI_masked_partial(vReg dst, indirect mem, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2758   predicate(UseSVE > 0 &&
2759             n->as_LoadVector()->memory_size() < MaxVectorSize &&
2760             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2761              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2762   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
2763   effect(TEMP ptmp, KILL cr);
2764   ins_cost(3 * SVE_COST);
2765   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated partial (S)" %}
2766   ins_encode %{
2767     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2768                           Matcher::vector_length(this));
2769     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2770                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2771     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
2772                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2773   %}
2774   ins_pipe(pipe_slow);
2775 %}
2776 
2777 instruct gatherL_masked_partial(vReg dst, indirect mem, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2778   predicate(UseSVE > 0 &&
2779             n->as_LoadVector()->memory_size() < MaxVectorSize &&
2780             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2781              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2782   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
2783   effect(TEMP ptmp, KILL cr);
2784   ins_cost(4 * SVE_COST);
2785   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated partial (D)" %}
2786   ins_encode %{
2787     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this));
2788     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2789                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2790     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2791     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
2792                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2793   %}
2794   ins_pipe(pipe_slow);
2795 %}
2796 
2797 // ------------------------------ Vector Store Scatter -------------------------------
2798 
2799 instruct scatterI(indirect mem, vReg src, vReg idx) %{
2800   predicate(UseSVE > 0 &&
2801             n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
2802             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2803              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2804   match(Set mem (StoreVectorScatter mem (Binary src idx)));
2805   ins_cost(SVE_COST);
2806   format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (S)" %}
2807   ins_encode %{
2808     __ sve_st1w_scatter(as_FloatRegister($src$$reg), ptrue,
2809                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2810   %}
2811   ins_pipe(pipe_slow);
2812 %}
2813 
2814 instruct scatterL(indirect mem, vReg src, vReg idx) %{
2815   predicate(UseSVE > 0 &&
2816             n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
2817             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2818              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2819   match(Set mem (StoreVectorScatter mem (Binary src idx)));
2820   ins_cost(2 * SVE_COST);
2821   format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (D)" %}

2822   ins_encode %{
2823     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));

2824     __ sve_st1d_scatter(as_FloatRegister($src$$reg), ptrue,
2825                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2826   %}
2827   ins_pipe(pipe_slow);
2828 %}
2829 
2830 // ------------------------------ Vector Store Scatter Partial -------------------------------
2831 
2832 instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2833   predicate(UseSVE > 0 &&
2834             n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
2835             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2836              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2837   match(Set mem (StoreVectorScatter mem (Binary src idx)));
2838   effect(TEMP ptmp, KILL cr);
2839   ins_cost(2 * SVE_COST + INSN_COST);
2840   format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (S)" %}

2841   ins_encode %{
2842     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2843                           Matcher::vector_length(this, $src));
2844     __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
2845                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2846   %}
2847   ins_pipe(pipe_slow);
2848 %}
2849 
2850 instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2851   predicate(UseSVE > 0 &&
2852             n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
2853             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2854              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2855   match(Set mem (StoreVectorScatter mem (Binary src idx)));
2856   effect(TEMP ptmp, KILL cr);
2857   ins_cost(3 * SVE_COST + INSN_COST);
2858   format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (D)" %}


2859   ins_encode %{
2860     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2861                           Matcher::vector_length(this, $src));
2862     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2863     __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
2864                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2865   %}
2866   ins_pipe(pipe_slow);
2867 %}
2868 
2869 // ------------------------------ Vector Store Scatter Predicated -------------------------------
2870 
2871 instruct scatterI_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
2872   predicate(UseSVE > 0 &&
2873             n->as_StoreVector()->memory_size() == MaxVectorSize &&
2874             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2875              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2876   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
2877   ins_cost(SVE_COST);
2878   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicate (S)" %}
2879   ins_encode %{
2880     __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
2881                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2882   %}
2883   ins_pipe(pipe_slow);
2884 %}
2885 
2886 instruct scatterL_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
2887   predicate(UseSVE > 0 &&
2888             n->as_StoreVector()->memory_size() == MaxVectorSize &&
2889             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2890              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2891   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
2892   ins_cost(2 * SVE_COST);
2893   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated (D)" %}
2894   ins_encode %{
2895     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2896     __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
2897                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2898   %}
2899   ins_pipe(pipe_slow);
2900 %}
2901 
2902 // ------------------------------ Vector Store Scatter Predicated Partial -------------------------------
2903 
2904 instruct scatterI_masked_partial(indirect mem, vReg src, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2905   predicate(UseSVE > 0 &&
2906             n->as_StoreVector()->memory_size() < MaxVectorSize &&
2907             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2908              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2909   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
2910   effect(TEMP ptmp, KILL cr);
2911   ins_cost(3 * SVE_COST);
2912   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated partial (S)" %}
2913   ins_encode %{
2914     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2915                           Matcher::vector_length(this, $src));
2916     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2917                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2918     __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
2919                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2920   %}
2921   ins_pipe(pipe_slow);
2922 %}
2923 
2924 instruct scatterL_masked_partial(indirect mem, vReg src, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2925   predicate(UseSVE > 0 &&
2926             n->as_StoreVector()->memory_size() < MaxVectorSize &&
2927             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2928              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2929   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
2930   effect(TEMP ptmp, KILL cr);
2931   ins_cost(4 * SVE_COST);
2932   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated partial (D)" %}
2933   ins_encode %{
2934     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2935                           Matcher::vector_length(this, $src));
2936     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2937                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2938     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2939     __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
2940                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2941   %}
2942   ins_pipe(pipe_slow);
2943 %}
2944 
2945 // ------------------------------ Vector Load Const -------------------------------
2946 
2947 instruct loadconB(vReg dst, immI0 src) %{
2948   predicate(UseSVE > 0 &&
2949             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2950   match(Set dst (VectorLoadConst src));
2951   ins_cost(SVE_COST);
2952   format %{ "sve_index $dst, 0, 1\t# generate iota indices" %}
2953   ins_encode %{
2954     __ sve_index(as_FloatRegister($dst$$reg), __ B, 0, 1);
2955   %}
2956   ins_pipe(pipe_slow);
2957 %}
2958 
2959 // Intrisics for String.indexOf(char)
2960 
2961 dnl
2962 define(`STRING_INDEXOF_CHAR', `
2963 instruct string$1_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
2964                                   iRegI_R0 result, vReg ztmp1, vReg ztmp2,
2965                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
2966 %{
2967   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
2968   predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::$1));
2969   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
2970 
2971   format %{ "String$2 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
2972 
2973   ins_encode %{
2974     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
2975                                as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
2976                                as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), $3 /* isL */);
2977   %}
2978   ins_pipe(pipe_class_memory);
2979 %}')dnl
2980 dnl                 $1 $2      $3
2981 STRING_INDEXOF_CHAR(L, Latin1, true)
2982 STRING_INDEXOF_CHAR(U, UTF16,  false)
2983 
2984 // ---------------------------- Vector mask reductions ---------------------------
2985 instruct vmask_truecount(iRegINoSp dst, pReg src) %{



2986   predicate(UseSVE > 0 &&
2987             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2988   match(Set dst (VectorMaskTrueCount src));
2989   ins_cost(SVE_COST);
2990   format %{ "vmask_truecount $dst, $src\t# vector mask truecount (sve)" %}

2991   ins_encode %{
2992     BasicType bt = Matcher::vector_element_basic_type(this, $src);
2993     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2994     __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($src$$reg));
2995   %}
2996   ins_pipe(pipe_slow);
2997 %}
2998 
2999 instruct vmask_firsttrue(iRegINoSp dst, pReg src, pReg ptmp) %{








3000   predicate(UseSVE > 0 &&
3001             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3002   match(Set dst (VectorMaskFirstTrue src));
3003   effect(TEMP ptmp);
3004   ins_cost(2 * SVE_COST);
3005   format %{ "vmask_firsttrue $dst, $src\t# vector mask firsttrue (sve)" %}
3006   ins_encode %{
3007     BasicType bt = Matcher::vector_element_basic_type(this, $src);
3008     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3009     __ sve_brkb(as_PRegister($ptmp$$reg), ptrue, as_PRegister($src$$reg), false);
3010     __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($ptmp$$reg));
3011   %}
3012   ins_pipe(pipe_slow);
3013 %}




3014 
3015 instruct vmask_lasttrue(iRegINoSp dst, pReg src, pReg ptmp) %{




3016   predicate(UseSVE > 0 &&
3017             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3018   match(Set dst (VectorMaskLastTrue src));
3019   effect(TEMP ptmp);
3020   ins_cost(3 * SVE_COST);
3021   format %{ "vmask_lasttrue $dst, $src\t# vector mask lasttrue (sve)" %}
3022   ins_encode %{
3023     BasicType bt = Matcher::vector_element_basic_type(this, $src);
3024     __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($src$$reg), as_PRegister($ptmp$$reg));



3025   %}
3026   ins_pipe(pipe_slow);
3027 %}
3028 
3029 instruct vmask_truecount_partial(iRegINoSp dst, pReg src, pRegGov pgtmp, rFlagsReg cr) %{
3030   predicate(UseSVE > 0 &&
3031             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3032   match(Set dst (VectorMaskTrueCount src));
3033   effect(TEMP pgtmp, KILL cr);
3034   ins_cost(2 * SVE_COST);
3035   format %{ "vmask_truecount_partial $dst, $src\t# vector mask truecount partial (sve)" %}
3036   ins_encode %{
3037     BasicType bt = Matcher::vector_element_basic_type(this, $src);
3038     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3039     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), size, Matcher::vector_length(this, $src));
3040     __ sve_cntp($dst$$Register, size, as_PRegister($pgtmp$$reg), as_PRegister($src$$reg));
3041   %}
3042   ins_pipe(pipe_slow);
3043 %}
3044 
3045 instruct vmask_firsttrue_partial(iRegINoSp dst, pReg src, pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
3046   predicate(UseSVE > 0 &&
3047             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3048   match(Set dst (VectorMaskFirstTrue src));
3049   effect(TEMP pgtmp, TEMP ptmp, KILL cr);
3050   ins_cost(3 * SVE_COST);
3051   format %{ "vmask_firsttrue_partial $dst, $src\t# vector mask firsttrue partial (sve)" %}
3052   ins_encode %{
3053     BasicType bt = Matcher::vector_element_basic_type(this, $src);
3054     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3055     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), size,
3056                           Matcher::vector_length(this, $src));
3057     __ sve_brkb(as_PRegister($ptmp$$reg), as_PRegister($pgtmp$$reg), as_PRegister($src$$reg), false);
3058     __ sve_cntp($dst$$Register, size, as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg));
3059   %}
3060   ins_pipe(pipe_slow);
3061 %}
3062 
3063 instruct vmask_lasttrue_partial(iRegINoSp dst, pReg src, pReg ptmp, rFlagsReg cr) %{
3064   predicate(UseSVE > 0 &&
3065             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3066   match(Set dst (VectorMaskLastTrue src));
3067   effect(TEMP ptmp, KILL cr);
3068   ins_cost(5 * SVE_COST);
3069   format %{ "vmask_lasttrue_partial $dst, $src\t# vector mask lasttrue partial (sve)" %}
3070   ins_encode %{
3071     BasicType bt = Matcher::vector_element_basic_type(this, $src);
3072     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3073     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
3074     __ sve_and(as_PRegister($ptmp$$reg), ptrue, as_PRegister($ptmp$$reg), as_PRegister($src$$reg));
3075     __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg));
3076   %}
3077   ins_pipe(pipe_slow);
3078 %}dnl
< prev index next >