< prev index next >

src/hotspot/cpu/aarch64/aarch64_sve.ad

Print this page

  71 operand vmemA_indOffL4(iRegP reg, vmemA_immLOffset4 off)
  72 %{
  73   constraint(ALLOC_IN_RC(ptr_reg));
  74   match(AddP reg off);
  75   op_cost(0);
  76   format %{ "[$reg, $off]" %}
  77   interface(MEMORY_INTER) %{
  78     base($reg);
  79     index(0xffffffff);
  80     scale(0x0);
  81     disp($off);
  82   %}
  83 %}
  84 
  85 // The indOff of vmemA is valid only when the vector element (load to/store from)
  86 // size equals to memory element (load from/store to) size.
  87 opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);
  88 
  89 source_hpp %{
  90   bool op_sve_supported(int opcode, int vlen, BasicType bt);

  91 %}
  92 
  93 source %{
  94 
  95   typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
  96                                                              PRegister Pg, const Address &adr);
  97 
  98   // Predicated load/store, with optional ptrue to all elements of given predicate register.
  99   static void loadStoreA_predicated(C2_MacroAssembler masm, bool is_store, FloatRegister reg,
 100                                     PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
 101                                     int opcode, Register base, int index, int size, int disp) {
 102     sve_mem_insn_predicate insn;
 103     int mesize = type2aelembytes(mem_elem_bt);
 104     if (index == -1) {
 105       assert(size == 0, "unsupported address mode: scale size = %d", size);
 106       switch(mesize) {
 107       case 1:
 108         insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
 109         break;
 110       case 2:

 127       ShouldNotReachHere();
 128     }
 129   }
 130 
 131   bool op_sve_supported(int opcode, int vlen, BasicType bt) {
 132     int length_in_bytes = vlen * type2aelembytes(bt);
 133     switch (opcode) {
 134       case Op_MulAddVS2VI:
 135       // No multiply reduction instructions
 136       case Op_MulReductionVD:
 137       case Op_MulReductionVF:
 138       case Op_MulReductionVI:
 139       case Op_MulReductionVL:
 140       // Others
 141       case Op_ExtractC:
 142       case Op_ExtractUB:
 143         return false;
 144       // Vector API specific
 145       case Op_VectorLoadShuffle:
 146       case Op_VectorRearrange:
 147         if (vlen < 4 || length_in_bytes > MaxVectorSize) {
 148           return false;
 149         } else {
 150           return true;
 151         }
 152       case Op_LoadVector:
 153       case Op_StoreVector:
 154         return Matcher::vector_size_supported(bt, vlen);
 155       default:
 156         break;
 157     }
 158     // By default, we only support vector operations with no less than 8 bytes and 2 elements.
 159     return 8 <= length_in_bytes && length_in_bytes <= MaxVectorSize && vlen >= 2;
 160   }








 161 %}
 162 
 163 definitions %{
 164   int_def SVE_COST             (200, 200);
 165 %}
 166 
 167 
 168 // All SVE instructions
 169 
 170 // vector load/store
 171 
 172 // Unpredicated vector load/store
 173 instruct loadV(vReg dst, vmemA mem) %{
 174   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() >= 16 &&
 175             n->as_LoadVector()->memory_size() == MaxVectorSize);
 176   match(Set dst (LoadVector mem));
 177   ins_cost(4 * SVE_COST);
 178   format %{ "sve_ldr $dst, $mem\t# vector (sve)" %}
 179   ins_encode %{
 180     FloatRegister dst_reg = as_FloatRegister($dst$$reg);

 277   format %{ "ldrq   $dst,$mem\t# vector (128 bits)" %}
 278   ins_encode( aarch64_enc_ldrvQ(dst, mem) );
 279   ins_pipe(vload_reg_mem128);
 280 %}
 281 
 282 // Store Vector (128 bits)
 283 instruct storeV16_vreg(vReg src, vmem16 mem)
 284 %{
 285   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 16);
 286   match(Set mem (StoreVector mem src));
 287   ins_cost(4 * INSN_COST);
 288   format %{ "strq   $mem,$src\t# vector (128 bits)" %}
 289   ins_encode( aarch64_enc_strvQ(src, mem) );
 290   ins_pipe(vstore_reg_mem128);
 291 %}
 292 
 293 // Predicated vector load/store, based on the vector length of the node.
 294 // Only load/store values in the range of the memory_size. This is needed
 295 // when the memory_size is lower than the hardware supported max vector size.
 296 // And this might happen for Vector API mask vector load/store.
 297 instruct loadV_partial(vReg dst, vmemA mem, pRegGov pTmp, rFlagsReg cr) %{
 298   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() > 16 &&
 299             n->as_LoadVector()->memory_size() < MaxVectorSize);
 300   match(Set dst (LoadVector mem));
 301   effect(TEMP pTmp, KILL cr);
 302   ins_cost(6 * SVE_COST);
 303   format %{ "sve_whilelo_zr_imm $pTmp, vector_length\n\t"
 304             "sve_ldr $dst, $pTmp, $mem\t# load vector predicated" %}
 305   ins_encode %{
 306     BasicType bt = Matcher::vector_element_basic_type(this);
 307     __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), __ elemType_to_regVariant(bt),
 308                           Matcher::vector_length(this));
 309     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 310     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg,
 311                           as_PRegister($pTmp$$reg), bt, bt, $mem->opcode(),
 312                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 313   %}
 314   ins_pipe(pipe_slow);
 315 %}
 316 
 317 instruct storeV_partial(vReg src, vmemA mem, pRegGov pTmp, rFlagsReg cr) %{
 318   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() > 16 &&
 319             n->as_StoreVector()->memory_size() < MaxVectorSize);
 320   match(Set mem (StoreVector mem src));
 321   effect(TEMP pTmp, KILL cr);
 322   ins_cost(5 * SVE_COST);
 323   format %{ "sve_whilelo_zr_imm $pTmp, vector_length\n\t"
 324             "sve_str $src, $pTmp, $mem\t# store vector predicated" %}
 325   ins_encode %{
 326     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 327     __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), __ elemType_to_regVariant(bt),
 328                           Matcher::vector_length(this, $src));
 329     FloatRegister src_reg = as_FloatRegister($src$$reg);
 330     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg,
 331                           as_PRegister($pTmp$$reg), bt, bt, $mem->opcode(),








































































 332                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 333   %}
 334   ins_pipe(pipe_slow);
 335 %}
 336 






































































































































 337 // vector reinterpret
 338 
 339 instruct reinterpret(vReg dst) %{
 340   predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() ==
 341                           n->in(1)->bottom_type()->is_vect()->length_in_bytes());  // src == dst
 342   match(Set dst (VectorReinterpret dst));
 343   ins_cost(0);
 344   format %{ "# reinterpret $dst\t# do nothing" %}
 345   ins_encode %{
 346     // empty
 347   %}
 348   ins_pipe(pipe_class_empty);
 349 %}
 350 
 351 instruct reinterpretResize(vReg dst, vReg src, pRegGov pTmp, rFlagsReg cr) %{
 352   predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() !=
 353                           n->in(1)->bottom_type()->is_vect()->length_in_bytes());  // src != dst
 354   match(Set dst (VectorReinterpret src));
 355   effect(TEMP_DEF dst, TEMP pTmp, KILL cr);
 356   ins_cost(3 * SVE_COST);
 357   format %{ "reinterpretResize $dst, $src\t# vector (sve)" %}
 358   ins_encode %{
 359     uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
 360     uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
 361     uint length_in_bytes_resize = length_in_bytes_src < length_in_bytes_dst ?
 362                                   length_in_bytes_src : length_in_bytes_dst;
 363     assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
 364            "invalid vector length");
 365     __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), __ B, length_in_bytes_resize);
 366     __ sve_dup(as_FloatRegister($dst$$reg), __ B, 0);
 367     __ sve_sel(as_FloatRegister($dst$$reg), __ B, as_PRegister($pTmp$$reg),
 368                as_FloatRegister($src$$reg), as_FloatRegister($dst$$reg));
 369   %}
 370   ins_pipe(pipe_slow);
 371 %}
 372 


































 373 // vector abs
 374 
 375 instruct vabsB(vReg dst, vReg src) %{
 376   predicate(UseSVE > 0 &&
 377             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
 378   match(Set dst (AbsVB src));
 379   ins_cost(SVE_COST);
 380   format %{ "sve_abs $dst, $src\t# vector (sve) (B)" %}
 381   ins_encode %{
 382     __ sve_abs(as_FloatRegister($dst$$reg), __ B,
 383          ptrue, as_FloatRegister($src$$reg));
 384   %}
 385   ins_pipe(pipe_slow);
 386 %}
 387 
 388 instruct vabsS(vReg dst, vReg src) %{
 389   predicate(UseSVE > 0 &&
 390             n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
 391   match(Set dst (AbsVS src));
 392   ins_cost(SVE_COST);
 393   format %{ "sve_abs $dst, $src\t# vector (sve) (H)" %}
 394   ins_encode %{
 395     __ sve_abs(as_FloatRegister($dst$$reg), __ H,
 396          ptrue, as_FloatRegister($src$$reg));
 397   %}
 398   ins_pipe(pipe_slow);
 399 %}
 400 
 401 instruct vabsI(vReg dst, vReg src) %{
 402   predicate(UseSVE > 0 &&
 403             n->bottom_type()->is_vect()->element_basic_type() == T_INT);
 404   match(Set dst (AbsVI src));
 405   ins_cost(SVE_COST);
 406   format %{ "sve_abs $dst, $src\t# vector (sve) (S)" %}
 407   ins_encode %{
 408     __ sve_abs(as_FloatRegister($dst$$reg), __ S,
 409          ptrue, as_FloatRegister($src$$reg));
 410   %}
 411   ins_pipe(pipe_slow);
 412 %}
 413 
 414 instruct vabsL(vReg dst, vReg src) %{
 415   predicate(UseSVE > 0 &&
 416             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
 417   match(Set dst (AbsVL src));
 418   ins_cost(SVE_COST);
 419   format %{ "sve_abs $dst, $src\t# vector (sve) (D)" %}
 420   ins_encode %{
 421     __ sve_abs(as_FloatRegister($dst$$reg), __ D,
 422          ptrue, as_FloatRegister($src$$reg));
 423   %}
 424   ins_pipe(pipe_slow);
 425 %}
 426 
 427 instruct vabsF(vReg dst, vReg src) %{
 428   predicate(UseSVE > 0 &&
 429             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
 430   match(Set dst (AbsVF src));
 431   ins_cost(SVE_COST);
 432   format %{ "sve_fabs $dst, $src\t# vector (sve) (S)" %}
 433   ins_encode %{
 434     __ sve_fabs(as_FloatRegister($dst$$reg), __ S,
 435          ptrue, as_FloatRegister($src$$reg));
 436   %}
 437   ins_pipe(pipe_slow);
 438 %}
 439 
 440 instruct vabsD(vReg dst, vReg src) %{
 441   predicate(UseSVE > 0 &&
 442             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
 443   match(Set dst (AbsVD src));
 444   ins_cost(SVE_COST);
 445   format %{ "sve_fabs $dst, $src\t# vector (sve) (D)" %}
 446   ins_encode %{
 447     __ sve_fabs(as_FloatRegister($dst$$reg), __ D,
 448          ptrue, as_FloatRegister($src$$reg));
 449   %}
 450   ins_pipe(pipe_slow);
 451 %}
 452 
















































































 453 // vector add
 454 
 455 instruct vaddB(vReg dst, vReg src1, vReg src2) %{
 456   predicate(UseSVE > 0);
 457   match(Set dst (AddVB src1 src2));
 458   ins_cost(SVE_COST);
 459   format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (B)" %}
 460   ins_encode %{
 461     __ sve_add(as_FloatRegister($dst$$reg), __ B,
 462          as_FloatRegister($src1$$reg),
 463          as_FloatRegister($src2$$reg));
 464   %}
 465   ins_pipe(pipe_slow);
 466 %}
 467 
 468 instruct vaddS(vReg dst, vReg src1, vReg src2) %{
 469   predicate(UseSVE > 0);
 470   match(Set dst (AddVS src1 src2));
 471   ins_cost(SVE_COST);
 472   format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (H)" %}

 513     __ sve_fadd(as_FloatRegister($dst$$reg), __ S,
 514          as_FloatRegister($src1$$reg),
 515          as_FloatRegister($src2$$reg));
 516   %}
 517   ins_pipe(pipe_slow);
 518 %}
 519 
 520 instruct vaddD(vReg dst, vReg src1, vReg src2) %{
 521   predicate(UseSVE > 0);
 522   match(Set dst (AddVD src1 src2));
 523   ins_cost(SVE_COST);
 524   format %{ "sve_fadd $dst, $src1, $src2\t # vector (sve) (D)" %}
 525   ins_encode %{
 526     __ sve_fadd(as_FloatRegister($dst$$reg), __ D,
 527          as_FloatRegister($src1$$reg),
 528          as_FloatRegister($src2$$reg));
 529   %}
 530   ins_pipe(pipe_slow);
 531 %}
 532 
















































































 533 // vector and
 534 
 535 instruct vand(vReg dst, vReg src1, vReg src2) %{
 536   predicate(UseSVE > 0);
 537   match(Set dst (AndV src1 src2));
 538   ins_cost(SVE_COST);
 539   format %{ "sve_and  $dst, $src1, $src2\t# vector (sve)" %}
 540   ins_encode %{
 541     __ sve_and(as_FloatRegister($dst$$reg),
 542          as_FloatRegister($src1$$reg),
 543          as_FloatRegister($src2$$reg));
 544   %}
 545   ins_pipe(pipe_slow);
 546 %}
 547 
 548 // vector or
 549 
 550 instruct vor(vReg dst, vReg src1, vReg src2) %{
 551   predicate(UseSVE > 0);
 552   match(Set dst (OrV src1 src2));

 558          as_FloatRegister($src2$$reg));
 559   %}
 560   ins_pipe(pipe_slow);
 561 %}
 562 
 563 // vector xor
 564 
 565 instruct vxor(vReg dst, vReg src1, vReg src2) %{
 566   predicate(UseSVE > 0);
 567   match(Set dst (XorV src1 src2));
 568   ins_cost(SVE_COST);
 569   format %{ "sve_eor  $dst, $src1, $src2\t# vector (sve)" %}
 570   ins_encode %{
 571     __ sve_eor(as_FloatRegister($dst$$reg),
 572          as_FloatRegister($src1$$reg),
 573          as_FloatRegister($src2$$reg));
 574   %}
 575   ins_pipe(pipe_slow);
 576 %}
 577 
 578 // vector not
 579 
 580 instruct vnotI(vReg dst, vReg src, immI_M1 m1) %{
 581   predicate(UseSVE > 0);
 582   match(Set dst (XorV src (ReplicateB m1)));
 583   match(Set dst (XorV src (ReplicateS m1)));
 584   match(Set dst (XorV src (ReplicateI m1)));
 585   ins_cost(SVE_COST);
 586   format %{ "sve_not $dst, $src\t# vector (sve) B/H/S" %}
 587   ins_encode %{
 588     __ sve_not(as_FloatRegister($dst$$reg), __ D,
 589                ptrue, as_FloatRegister($src$$reg));



 590   %}
 591   ins_pipe(pipe_slow);
 592 %}
 593 
 594 instruct vnotL(vReg dst, vReg src, immL_M1 m1) %{


 595   predicate(UseSVE > 0);
 596   match(Set dst (XorV src (ReplicateL m1)));
 597   ins_cost(SVE_COST);
 598   format %{ "sve_not $dst, $src\t# vector (sve) D" %}
 599   ins_encode %{
 600     __ sve_not(as_FloatRegister($dst$$reg), __ D,
 601                ptrue, as_FloatRegister($src$$reg));



 602   %}
 603   ins_pipe(pipe_slow);
 604 %}
 605 

 606 
 607 // vector and_not
 608 
 609 instruct vand_notI(vReg dst, vReg src1, vReg src2, immI_M1 m1) %{
 610   predicate(UseSVE > 0);
 611   match(Set dst (AndV src1 (XorV src2 (ReplicateB m1))));
 612   match(Set dst (AndV src1 (XorV src2 (ReplicateS m1))));
 613   match(Set dst (AndV src1 (XorV src2 (ReplicateI m1))));
 614   ins_cost(SVE_COST);
 615   format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) B/H/S" %}















































 616   ins_encode %{
 617     __ sve_bic(as_FloatRegister($dst$$reg),
 618                as_FloatRegister($src1$$reg),
 619                as_FloatRegister($src2$$reg));
 620   %}
 621   ins_pipe(pipe_slow);
 622 %}
 623 
 624 instruct vand_notL(vReg dst, vReg src1, vReg src2, immL_M1 m1) %{
 625   predicate(UseSVE > 0);
 626   match(Set dst (AndV src1 (XorV src2 (ReplicateL m1))));
 627   ins_cost(SVE_COST);
 628   format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) D" %}
 629   ins_encode %{
 630     __ sve_bic(as_FloatRegister($dst$$reg),
 631                as_FloatRegister($src1$$reg),
 632                as_FloatRegister($src2$$reg));
 633   %}
 634   ins_pipe(pipe_slow);
 635 %}
 636 
 637 
 638 // vector float div
 639 
 640 instruct vdivF(vReg dst_src1, vReg src2) %{
 641   predicate(UseSVE > 0);
 642   match(Set dst_src1 (DivVF dst_src1 src2));
 643   ins_cost(SVE_COST);
 644   format %{ "sve_fdiv  $dst_src1, $dst_src1, $src2\t# vector (sve) (S)" %}
 645   ins_encode %{
 646     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ S,
 647          ptrue, as_FloatRegister($src2$$reg));
 648   %}
 649   ins_pipe(pipe_slow);
 650 %}
 651 
 652 instruct vdivD(vReg dst_src1, vReg src2) %{
 653   predicate(UseSVE > 0);
 654   match(Set dst_src1 (DivVD dst_src1 src2));
 655   ins_cost(SVE_COST);
 656   format %{ "sve_fdiv  $dst_src1, $dst_src1, $src2\t# vector (sve) (D)" %}
 657   ins_encode %{
 658     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ D,
 659          ptrue, as_FloatRegister($src2$$reg));
 660   %}
 661   ins_pipe(pipe_slow);
 662 %}
 663 




























 664 // vector min/max
 665 
 666 instruct vmin(vReg dst_src1, vReg src2) %{
 667   predicate(UseSVE > 0);
 668   match(Set dst_src1 (MinV dst_src1 src2));
 669   ins_cost(SVE_COST);
 670   format %{ "sve_min $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
 671   ins_encode %{
 672     BasicType bt = Matcher::vector_element_basic_type(this);
 673     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
 674     if (is_floating_point_type(bt)) {
 675       __ sve_fmin(as_FloatRegister($dst_src1$$reg), size,
 676                   ptrue, as_FloatRegister($src2$$reg));
 677     } else {
 678       assert(is_integral_type(bt), "Unsupported type");
 679       __ sve_smin(as_FloatRegister($dst_src1$$reg), size,
 680                   ptrue, as_FloatRegister($src2$$reg));
 681     }
 682   %}
 683   ins_pipe(pipe_slow);
 684 %}
 685 
 686 instruct vmax(vReg dst_src1, vReg src2) %{
 687   predicate(UseSVE > 0);
 688   match(Set dst_src1 (MaxV dst_src1 src2));
 689   ins_cost(SVE_COST);
 690   format %{ "sve_max $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
 691   ins_encode %{
 692     BasicType bt = Matcher::vector_element_basic_type(this);
 693     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
 694     if (is_floating_point_type(bt)) {
 695       __ sve_fmax(as_FloatRegister($dst_src1$$reg), size,
 696                   ptrue, as_FloatRegister($src2$$reg));
 697     } else {
 698       assert(is_integral_type(bt), "Unsupported type");
 699       __ sve_smax(as_FloatRegister($dst_src1$$reg), size,
 700                   ptrue, as_FloatRegister($src2$$reg));
 701     }
 702   %}
 703   ins_pipe(pipe_slow);
 704 %}
 705 










































 706 // vector fmla
 707 
 708 // dst_src1 = dst_src1 + src2 * src3
 709 instruct vfmlaF(vReg dst_src1, vReg src2, vReg src3) %{
 710   predicate(UseFMA && UseSVE > 0);
 711   match(Set dst_src1 (FmaVF dst_src1 (Binary src2 src3)));
 712   ins_cost(SVE_COST);
 713   format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
 714   ins_encode %{
 715     __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ S,
 716          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 717   %}
 718   ins_pipe(pipe_slow);
 719 %}
 720 
 721 // dst_src1 = dst_src1 + src2 * src3
 722 instruct vfmlaD(vReg dst_src1, vReg src2, vReg src3) %{
 723   predicate(UseFMA && UseSVE > 0);
 724   match(Set dst_src1 (FmaVD dst_src1 (Binary src2 src3)));
 725   ins_cost(SVE_COST);
 726   format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
 727   ins_encode %{
 728     __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ D,
 729          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 730   %}
 731   ins_pipe(pipe_slow);
 732 %}
 733 




























 734 // vector fmls
 735 
 736 // dst_src1 = dst_src1 + -src2 * src3
 737 // dst_src1 = dst_src1 + src2 * -src3
 738 instruct vfmlsF(vReg dst_src1, vReg src2, vReg src3) %{
 739   predicate(UseFMA && UseSVE > 0);
 740   match(Set dst_src1 (FmaVF dst_src1 (Binary (NegVF src2) src3)));
 741   match(Set dst_src1 (FmaVF dst_src1 (Binary src2 (NegVF src3))));
 742   ins_cost(SVE_COST);
 743   format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
 744   ins_encode %{
 745     __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ S,
 746          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 747   %}
 748   ins_pipe(pipe_slow);
 749 %}
 750 
 751 // dst_src1 = dst_src1 + -src2 * src3
 752 // dst_src1 = dst_src1 + src2 * -src3
 753 instruct vfmlsD(vReg dst_src1, vReg src2, vReg src3) %{

 922     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ S,
 923       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 924   %}
 925   ins_pipe(pipe_slow);
 926 %}
 927 
 928 // dst_src1 = dst_src1 - src2 * src3
 929 instruct vmlsL(vReg dst_src1, vReg src2, vReg src3)
 930 %{
 931   predicate(UseSVE > 0);
 932   match(Set dst_src1 (SubVL dst_src1 (MulVL src2 src3)));
 933   ins_cost(SVE_COST);
 934   format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (D)" %}
 935   ins_encode %{
 936     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ D,
 937       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 938   %}
 939   ins_pipe(pipe_slow);
 940 %}
 941 
 942 
 943 // vector mul
 944 
 945 instruct vmulB(vReg dst_src1, vReg src2) %{
 946   predicate(UseSVE > 0);
 947   match(Set dst_src1 (MulVB dst_src1 src2));
 948   ins_cost(SVE_COST);
 949   format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (B)" %}
 950   ins_encode %{
 951     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ B,
 952          ptrue, as_FloatRegister($src2$$reg));
 953   %}
 954   ins_pipe(pipe_slow);
 955 %}
 956 
 957 instruct vmulS(vReg dst_src1, vReg src2) %{
 958   predicate(UseSVE > 0);
 959   match(Set dst_src1 (MulVS dst_src1 src2));
 960   ins_cost(SVE_COST);
 961   format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (H)" %}
 962   ins_encode %{

 999     __ sve_fmul(as_FloatRegister($dst$$reg), __ S,
1000          as_FloatRegister($src1$$reg),
1001          as_FloatRegister($src2$$reg));
1002   %}
1003   ins_pipe(pipe_slow);
1004 %}
1005 
1006 instruct vmulD(vReg dst, vReg src1, vReg src2) %{
1007   predicate(UseSVE > 0);
1008   match(Set dst (MulVD src1 src2));
1009   ins_cost(SVE_COST);
1010   format %{ "sve_fmul $dst, $src1, $src2\t # vector (sve) (D)" %}
1011   ins_encode %{
1012     __ sve_fmul(as_FloatRegister($dst$$reg), __ D,
1013          as_FloatRegister($src1$$reg),
1014          as_FloatRegister($src2$$reg));
1015   %}
1016   ins_pipe(pipe_slow);
1017 %}
1018 
















































































1019 // vector fneg
1020 
1021 instruct vnegF(vReg dst, vReg src) %{
1022   predicate(UseSVE > 0);

1023   match(Set dst (NegVF src));
1024   ins_cost(SVE_COST);
1025   format %{ "sve_fneg $dst, $src\t# vector (sve) (S)" %}
1026   ins_encode %{
1027     __ sve_fneg(as_FloatRegister($dst$$reg), __ S,
1028          ptrue, as_FloatRegister($src$$reg));
1029   %}
1030   ins_pipe(pipe_slow);
1031 %}
1032 
1033 instruct vnegD(vReg dst, vReg src) %{
1034   predicate(UseSVE > 0);

1035   match(Set dst (NegVD src));
1036   ins_cost(SVE_COST);
1037   format %{ "sve_fneg $dst, $src\t# vector (sve) (D)" %}
1038   ins_encode %{
1039     __ sve_fneg(as_FloatRegister($dst$$reg), __ D,
1040          ptrue, as_FloatRegister($src$$reg));
1041   %}
1042   ins_pipe(pipe_slow);
1043 %}
1044 




























1045 // popcount vector
1046 
1047 instruct vpopcountI(vReg dst, vReg src) %{
1048   predicate(UseSVE > 0);
1049   match(Set dst (PopCountVI src));
1050   format %{ "sve_cnt $dst, $src\t# vector (sve) (S)\n\t" %}
1051   ins_encode %{
1052      __ sve_cnt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg));
1053   %}
1054   ins_pipe(pipe_slow);
1055 %}
1056 
1057 // vector mask compare
1058 
1059 instruct vmaskcmp(vReg dst, vReg src1, vReg src2, immI cond, pRegGov pTmp, rFlagsReg cr) %{
1060   predicate(UseSVE > 0);
1061   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
1062   effect(TEMP pTmp, KILL cr);
1063   ins_cost(2 * SVE_COST);
1064   format %{ "sve_cmp $pTmp, $src1, $src2\n\t"
1065             "sve_cpy $dst, $pTmp, -1\t# vector mask cmp (sve)" %}
1066   ins_encode %{
1067     BasicType bt = Matcher::vector_element_basic_type(this);
1068     __ sve_compare(as_PRegister($pTmp$$reg), bt, ptrue, as_FloatRegister($src1$$reg),
1069                    as_FloatRegister($src2$$reg), (int)$cond$$constant);
1070     __ sve_cpy(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
1071                as_PRegister($pTmp$$reg), -1, false);
1072   %}
1073   ins_pipe(pipe_slow);
1074 %}
1075 
1076 // vector blend
1077 
1078 instruct vblend(vReg dst, vReg src1, vReg src2, vReg src3, pRegGov pTmp, rFlagsReg cr) %{
1079   predicate(UseSVE > 0);
1080   match(Set dst (VectorBlend (Binary src1 src2) src3));
1081   effect(TEMP pTmp, KILL cr);
1082   ins_cost(2 * SVE_COST);
1083   format %{ "sve_cmpeq $pTmp, $src3, -1\n\t"
1084             "sve_sel $dst, $pTmp, $src2, $src1\t# vector blend (sve)" %}
1085   ins_encode %{
1086     Assembler::SIMD_RegVariant size =
1087       __ elemType_to_regVariant(Matcher::vector_element_basic_type(this));
1088     __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), size,
1089                ptrue, as_FloatRegister($src3$$reg), -1);
1090     __ sve_sel(as_FloatRegister($dst$$reg), size, as_PRegister($pTmp$$reg),
1091                as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
1092   %}
1093   ins_pipe(pipe_slow);
1094 %}
1095 
1096 // vector blend with compare
1097 
1098 instruct vblend_maskcmp(vReg dst, vReg src1, vReg src2, vReg src3,
1099                         vReg src4, pRegGov pTmp, immI cond, rFlagsReg cr) %{
1100   predicate(UseSVE > 0);
1101   match(Set dst (VectorBlend (Binary src1 src2) (VectorMaskCmp (Binary src3 src4) cond)));
1102   effect(TEMP pTmp, KILL cr);
1103   ins_cost(2 * SVE_COST);
1104   format %{ "sve_cmp $pTmp, $src3, $src4\t# vector cmp (sve)\n\t"
1105             "sve_sel $dst, $pTmp, $src2, $src1\t# vector blend (sve)" %}
1106   ins_encode %{
1107     BasicType bt = Matcher::vector_element_basic_type(this);
1108     __ sve_compare(as_PRegister($pTmp$$reg), bt, ptrue, as_FloatRegister($src3$$reg),
1109                    as_FloatRegister($src4$$reg), (int)$cond$$constant);
1110     __ sve_sel(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
1111                as_PRegister($pTmp$$reg), as_FloatRegister($src2$$reg),
1112                as_FloatRegister($src1$$reg));
1113   %}
1114   ins_pipe(pipe_slow);
1115 %}
1116 
1117 // vector load mask
1118 
1119 instruct vloadmaskB(vReg dst, vReg src) %{
1120   predicate(UseSVE > 0 &&
1121             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
1122   match(Set dst (VectorLoadMask src));

1123   ins_cost(SVE_COST);
1124   format %{ "sve_neg $dst, $src\t# vector load mask (B)" %}
1125   ins_encode %{
1126     __ sve_neg(as_FloatRegister($dst$$reg), __ B, ptrue, as_FloatRegister($src$$reg));
1127   %}
1128   ins_pipe(pipe_slow);
1129 %}
1130 
1131 instruct vloadmaskS(vReg dst, vReg src) %{
1132   predicate(UseSVE > 0 &&
1133             n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
1134   match(Set dst (VectorLoadMask src));
1135   ins_cost(2 * SVE_COST);
1136   format %{ "sve_uunpklo $dst, H, $src\n\t"
1137             "sve_neg $dst, $dst\t# vector load mask (B to H)" %}
1138   ins_encode %{
1139     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
1140     __ sve_neg(as_FloatRegister($dst$$reg), __ H, ptrue, as_FloatRegister($dst$$reg));
1141   %}
1142   ins_pipe(pipe_slow);
1143 %}
1144 
1145 instruct vloadmaskI(vReg dst, vReg src) %{
1146   predicate(UseSVE > 0 &&
1147             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
1148              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
1149   match(Set dst (VectorLoadMask src));

1150   ins_cost(3 * SVE_COST);
1151   format %{ "sve_uunpklo $dst, H, $src\n\t"
1152             "sve_uunpklo $dst, S, $dst\n\t"
1153             "sve_neg $dst, $dst\t# vector load mask (B to S)" %}
1154   ins_encode %{
1155     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
1156     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($dst$$reg));
1157     __ sve_neg(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($dst$$reg));
1158   %}
1159   ins_pipe(pipe_slow);
1160 %}
1161 
1162 instruct vloadmaskL(vReg dst, vReg src) %{
1163   predicate(UseSVE > 0 &&
1164             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
1165              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
1166   match(Set dst (VectorLoadMask src));
1167   ins_cost(4 * SVE_COST);
1168   format %{ "sve_uunpklo $dst, H, $src\n\t"
1169             "sve_uunpklo $dst, S, $dst\n\t"
1170             "sve_uunpklo $dst, D, $dst\n\t"
1171             "sve_neg $dst, $dst\t# vector load mask (B to D)" %}
1172   ins_encode %{
1173     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
1174     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($dst$$reg));
1175     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($dst$$reg));
1176     __ sve_neg(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg));
1177   %}
1178   ins_pipe(pipe_slow);
1179 %}
1180 
1181 // vector store mask
1182 
1183 instruct vstoremaskB(vReg dst, vReg src, immI_1 size) %{
1184   predicate(UseSVE > 0);
1185   match(Set dst (VectorStoreMask src size));
1186   ins_cost(SVE_COST);
1187   format %{ "sve_neg $dst, $src\t# vector store mask (B)" %}
1188   ins_encode %{
1189     __ sve_neg(as_FloatRegister($dst$$reg), __ B, ptrue,
1190                as_FloatRegister($src$$reg));
1191   %}
1192   ins_pipe(pipe_slow);
1193 %}
1194 
1195 instruct vstoremaskS(vReg dst, vReg src, vReg tmp, immI_2 size) %{
1196   predicate(UseSVE > 0);
1197   match(Set dst (VectorStoreMask src size));
1198   effect(TEMP_DEF dst, TEMP tmp);
1199   ins_cost(3 * SVE_COST);
1200   format %{ "sve_dup $tmp, H, 0\n\t"
1201             "sve_uzp1 $dst, B, $src, $tmp\n\t"
1202             "sve_neg $dst, B, $dst\t# vector store mask (sve) (H to B)" %}
1203   ins_encode %{
1204     __ sve_dup(as_FloatRegister($tmp$$reg), __ H, 0);
1205     __ sve_uzp1(as_FloatRegister($dst$$reg), __ B,
1206                 as_FloatRegister($src$$reg), as_FloatRegister($tmp$$reg));
1207     __ sve_neg(as_FloatRegister($dst$$reg), __ B, ptrue,
1208                as_FloatRegister($dst$$reg));
1209 
1210   %}
1211   ins_pipe(pipe_slow);
1212 %}
1213 
1214 instruct vstoremaskI(vReg dst, vReg src, vReg tmp, immI_4 size) %{
1215   predicate(UseSVE > 0);
1216   match(Set dst (VectorStoreMask src size));
1217   effect(TEMP_DEF dst, TEMP tmp);
1218   ins_cost(4 * SVE_COST);
1219   format %{ "sve_dup $tmp, S, 0\n\t"
1220             "sve_uzp1 $dst, H, $src, $tmp\n\t"
1221             "sve_uzp1 $dst, B, $dst, $tmp\n\t"
1222             "sve_neg $dst, B, $dst\t# vector store mask (sve) (S to B)" %}
1223   ins_encode %{
1224     __ sve_dup(as_FloatRegister($tmp$$reg), __ S, 0);
1225     __ sve_uzp1(as_FloatRegister($dst$$reg), __ H,
1226                 as_FloatRegister($src$$reg), as_FloatRegister($tmp$$reg));
1227     __ sve_uzp1(as_FloatRegister($dst$$reg), __ B,
1228                 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
1229     __ sve_neg(as_FloatRegister($dst$$reg), __ B, ptrue,
1230                as_FloatRegister($dst$$reg));
1231   %}
1232   ins_pipe(pipe_slow);
1233 %}
1234 
1235 instruct vstoremaskL(vReg dst, vReg src, vReg tmp, immI_8 size) %{
1236   predicate(UseSVE > 0);
1237   match(Set dst (VectorStoreMask src size));
1238   effect(TEMP_DEF dst, TEMP tmp);
1239   ins_cost(5 * SVE_COST);
1240   format %{ "sve_dup $tmp, D, 0\n\t"
1241             "sve_uzp1 $dst, S, $src, $tmp\n\t"
1242             "sve_uzp1 $dst, H, $dst, $tmp\n\t"
1243             "sve_uzp1 $dst, B, $dst, $tmp\n\t"
1244             "sve_neg $dst, B, $dst\t# vector store mask (sve) (D to B)" %}
1245   ins_encode %{
1246     __ sve_dup(as_FloatRegister($tmp$$reg), __ D, 0);
1247     __ sve_uzp1(as_FloatRegister($dst$$reg), __ S,
1248                 as_FloatRegister($src$$reg), as_FloatRegister($tmp$$reg));
1249     __ sve_uzp1(as_FloatRegister($dst$$reg), __ H,
1250                 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
1251     __ sve_uzp1(as_FloatRegister($dst$$reg), __ B,
1252                 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
1253     __ sve_neg(as_FloatRegister($dst$$reg), __ B, ptrue,
1254                as_FloatRegister($dst$$reg));
1255   %}
1256   ins_pipe(pipe_slow);
1257 %}
1258 
1259 // load/store mask vector
1260 
1261 instruct vloadmask_loadV_byte(vReg dst, vmemA mem) %{
1262   predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() == MaxVectorSize &&
1263             type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) == 1);
1264   match(Set dst (VectorLoadMask (LoadVector mem)));
1265   ins_cost(5 * SVE_COST);
1266   format %{ "sve_ld1b $dst, $mem\n\t"
1267             "sve_neg $dst, $dst\t# load vector mask (sve)" %}

1268   ins_encode %{
1269     FloatRegister dst_reg = as_FloatRegister($dst$$reg);

1270     BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
1271     Assembler::SIMD_RegVariant to_vect_variant = __ elemType_to_regVariant(to_vect_bt);
1272     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg, ptrue,
1273                           T_BOOLEAN, to_vect_bt, $mem->opcode(),
1274                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1275     __ sve_neg(dst_reg, to_vect_variant, ptrue, dst_reg);

1276   %}
1277   ins_pipe(pipe_slow);
1278 %}
1279 
1280 instruct vloadmask_loadV_non_byte(vReg dst, indirect mem) %{
1281   predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() == MaxVectorSize &&


1282             type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
1283   match(Set dst (VectorLoadMask (LoadVector mem)));
1284   ins_cost(5 * SVE_COST);
1285   format %{ "sve_ld1b $dst, $mem\n\t"
1286             "sve_neg $dst, $dst\t# load vector mask (sve)" %}
1287   ins_encode %{
1288     FloatRegister dst_reg = as_FloatRegister($dst$$reg);

1289     BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
1290     Assembler::SIMD_RegVariant to_vect_variant = __ elemType_to_regVariant(to_vect_bt);
1291     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg, ptrue,
1292                           T_BOOLEAN, to_vect_bt, $mem->opcode(),

1293                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1294     __ sve_neg(dst_reg, to_vect_variant, ptrue, dst_reg);
1295   %}
1296   ins_pipe(pipe_slow);
1297 %}
1298 
1299 instruct storeV_vstoremask_byte(vmemA mem, vReg src, vReg tmp, immI_1 esize) %{
1300   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() *
1301                           n->as_StoreVector()->in(MemNode::ValueIn)->in(2)->get_int() == MaxVectorSize);


1302   match(Set mem (StoreVector mem (VectorStoreMask src esize)));
1303   effect(TEMP tmp);
1304   ins_cost(5 * SVE_COST);
1305   format %{ "sve_neg $tmp, $src\n\t"
1306             "sve_st1b $tmp, $mem\t# store vector mask (sve)" %}
1307   ins_encode %{
1308     BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
1309     assert(type2aelembytes(from_vect_bt) == (int)$esize$$constant, "unsupported type.");
1310     Assembler::SIMD_RegVariant from_vect_variant = __ elemBytes_to_regVariant($esize$$constant);
1311     __ sve_neg(as_FloatRegister($tmp$$reg), from_vect_variant, ptrue,
1312                as_FloatRegister($src$$reg));
1313     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($tmp$$reg),
1314                           ptrue, T_BOOLEAN, from_vect_bt, $mem->opcode(),
1315                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1316   %}
1317   ins_pipe(pipe_slow);
1318 %}
1319 
1320 instruct storeV_vstoremask_non_byte(indirect mem, vReg src, vReg tmp, immI_gt_1 esize) %{
1321   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() *
1322                           n->as_StoreVector()->in(MemNode::ValueIn)->in(2)->get_int() == MaxVectorSize);



1323   match(Set mem (StoreVector mem (VectorStoreMask src esize)));
1324   effect(TEMP tmp);
1325   ins_cost(5 * SVE_COST);
1326   format %{ "sve_neg $tmp, $src\n\t"
1327             "sve_st1b $tmp, $mem\t# store vector mask (sve)" %}
1328   ins_encode %{


1329     BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
1330     assert(type2aelembytes(from_vect_bt) == (int)$esize$$constant, "unsupported type.");
1331     Assembler::SIMD_RegVariant from_vect_variant = __ elemBytes_to_regVariant($esize$$constant);
1332     __ sve_neg(as_FloatRegister($tmp$$reg), from_vect_variant, ptrue,
1333                as_FloatRegister($src$$reg));
1334     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($tmp$$reg),
1335                           ptrue, T_BOOLEAN, from_vect_bt, $mem->opcode(),
1336                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1337   %}
1338   ins_pipe(pipe_slow);
1339 %}
1340 
1341 // vector add reduction
1342 
1343 instruct reduce_addI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp) %{
1344   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);

1345   match(Set dst (AddReductionVI src1 src2));
1346   effect(TEMP_DEF dst, TEMP vtmp);
1347   ins_cost(SVE_COST);
1348   format %{ "sve_reduce_addI $dst, $src1, $src2\t# addB/S/I reduction (sve) (may extend)" %}
1349   ins_encode %{
1350     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1351     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1352     __ sve_uaddv(as_FloatRegister($vtmp$$reg), variant, ptrue, as_FloatRegister($src2$$reg));
1353     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1354     __ addw($dst$$Register, $dst$$Register, $src1$$Register);
1355     if (bt == T_BYTE) {
1356       __ sxtb($dst$$Register, $dst$$Register);
1357     } else if (bt == T_SHORT) {
1358       __ sxth($dst$$Register, $dst$$Register);
1359     } else {
1360       assert(bt == T_INT, "unsupported type");
1361     }

































1362   %}
1363   ins_pipe(pipe_slow);
1364 %}
1365 
1366 instruct reduce_addI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1367                              pRegGov ptmp, rFlagsReg cr) %{
1368   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);

1369   match(Set dst (AddReductionVI src1 src2));
1370   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1371   ins_cost(SVE_COST);
1372   format %{ "sve_reduce_addI $dst, $src1, $src2\t# addI reduction partial (sve) (may extend)" %}
1373   ins_encode %{
1374     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1375     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1376     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1377                           Matcher::vector_length(this, $src2));
1378     __ sve_uaddv(as_FloatRegister($vtmp$$reg), variant,
1379                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1380     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1381     __ addw($dst$$Register, $dst$$Register, $src1$$Register);
1382     if (bt == T_BYTE) {
1383       __ sxtb($dst$$Register, $dst$$Register);
1384     } else if (bt == T_SHORT) {
1385       __ sxth($dst$$Register, $dst$$Register);
1386     } else {
1387       assert(bt == T_INT, "unsupported type");
1388     }
1389   %}
1390   ins_pipe(pipe_slow);
1391 %}
1392 
1393 instruct reduce_addL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp) %{
1394   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);


1395   match(Set dst (AddReductionVL src1 src2));
1396   effect(TEMP_DEF dst, TEMP vtmp);
















1397   ins_cost(SVE_COST);
1398   format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction (sve)" %}

1399   ins_encode %{
1400     __ sve_uaddv(as_FloatRegister($vtmp$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
1401     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1402     __ add($dst$$Register, $dst$$Register, $src1$$Register);

1403   %}
1404   ins_pipe(pipe_slow);
1405 %}
1406 
1407 instruct reduce_addL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1408                              pRegGov ptmp, rFlagsReg cr) %{
1409   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1410   match(Set dst (AddReductionVL src1 src2));
1411   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1412   ins_cost(SVE_COST);
1413   format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction partial (sve)" %}

1414   ins_encode %{
1415     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1416                           Matcher::vector_length(this, $src2));
1417     __ sve_uaddv(as_FloatRegister($vtmp$$reg), __ D,
1418                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1419     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1420     __ add($dst$$Register, $dst$$Register, $src1$$Register);
1421   %}
1422   ins_pipe(pipe_slow);
1423 %}
1424 

1425 
1426 instruct reduce_addF(vRegF src1_dst, vReg src2) %{
1427   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1428   match(Set src1_dst (AddReductionVF src1_dst src2));


1429   ins_cost(SVE_COST);
1430   format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (S)" %}
1431   ins_encode %{
1432     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
1433          ptrue, as_FloatRegister($src2$$reg));


1434   %}
1435   ins_pipe(pipe_slow);
1436 %}
1437 
1438 instruct reduce_addF_partial(vRegF src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
1439   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1440   match(Set src1_dst (AddReductionVF src1_dst src2));


1441   ins_cost(SVE_COST);
1442   effect(TEMP ptmp, KILL cr);
1443   format %{ "sve_reduce_addF $src1_dst, $src1_dst, $src2\t# addF reduction partial (sve) (S)" %}













1444   ins_encode %{
1445     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
1446                           Matcher::vector_length(this, $src2));
1447     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
1448                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1449   %}
1450   ins_pipe(pipe_slow);
1451 %}
1452 
1453 instruct reduce_addD(vRegD src1_dst, vReg src2) %{
1454   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1455   match(Set src1_dst (AddReductionVD src1_dst src2));

1456   ins_cost(SVE_COST);
1457   format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (D)" %}
1458   ins_encode %{
1459     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
1460          ptrue, as_FloatRegister($src2$$reg));
1461   %}
1462   ins_pipe(pipe_slow);
1463 %}
1464 
1465 instruct reduce_addD_partial(vRegD src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
1466   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1467   match(Set src1_dst (AddReductionVD src1_dst src2));












































1468   ins_cost(SVE_COST);
















1469   effect(TEMP ptmp, KILL cr);
1470   format %{ "sve_reduce_addD $src1_dst, $src1_dst, $src2\t# addD reduction partial (sve) (D)" %}

1471   ins_encode %{
1472     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1473                           Matcher::vector_length(this, $src2));


1474     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
1475                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1476   %}
1477   ins_pipe(pipe_slow);
1478 %}
1479 
1480 // vector and reduction
1481 
1482 instruct reduce_andI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp) %{
1483   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&

1484             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1485   match(Set dst (AndReductionV src1 src2));
1486   effect(TEMP_DEF dst, TEMP vtmp);
1487   ins_cost(SVE_COST);
1488   format %{ "sve_reduce_andI $dst, $src1, $src2\t# andB/S/I reduction (sve) (may extend)" %}
1489   ins_encode %{
1490     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1491     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1492     __ sve_andv(as_FloatRegister($vtmp$$reg), variant, ptrue, as_FloatRegister($src2$$reg));
1493     __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1494     __ andw($dst$$Register, $dst$$Register, $src1$$Register);
1495     if (bt == T_BYTE) {
1496       __ sxtb($dst$$Register, $dst$$Register);
1497     } else if (bt == T_SHORT) {
1498       __ sxth($dst$$Register, $dst$$Register);
1499     } else {
1500       assert(bt == T_INT, "unsupported type");
1501     }








1502   %}
1503   ins_pipe(pipe_slow);
1504 %}
1505 
1506 instruct reduce_andI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1507                              pRegGov ptmp, rFlagsReg cr) %{
1508   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&

1509             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1510   match(Set dst (AndReductionV src1 src2));
1511   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1512   ins_cost(SVE_COST);
1513   format %{ "sve_reduce_andI $dst, $src1, $src2\t# andI reduction partial (sve) (may extend)" %}
1514   ins_encode %{
1515     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1516     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1517     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1518                           Matcher::vector_length(this, $src2));
1519     __ sve_andv(as_FloatRegister($vtmp$$reg), variant,
1520                 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1521     __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1522     __ andw($dst$$Register, $dst$$Register, $src1$$Register);
1523     if (bt == T_BYTE) {
1524       __ sxtb($dst$$Register, $dst$$Register);
1525     } else if (bt == T_SHORT) {
1526       __ sxth($dst$$Register, $dst$$Register);
1527     } else {
1528       assert(bt == T_INT, "unsupported type");
1529     }
1530   %}
1531   ins_pipe(pipe_slow);
1532 %}
1533 
1534 instruct reduce_andL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp) %{
1535   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1536             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);


1537   match(Set dst (AndReductionV src1 src2));
1538   effect(TEMP_DEF dst, TEMP vtmp);




















1539   ins_cost(SVE_COST);
1540   format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction (sve)" %}
1541   ins_encode %{
1542     __ sve_andv(as_FloatRegister($vtmp$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
1543     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1544     __ andr($dst$$Register, $dst$$Register, $src1$$Register);

1545   %}
1546   ins_pipe(pipe_slow);
1547 %}
1548 
1549 instruct reduce_andL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1550                              pRegGov ptmp, rFlagsReg cr) %{
1551   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1552             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1553   match(Set dst (AndReductionV src1 src2));
1554   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1555   ins_cost(SVE_COST);
1556   format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction partial (sve)" %}








































1557   ins_encode %{
1558     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1559                           Matcher::vector_length(this, $src2));
1560     __ sve_andv(as_FloatRegister($vtmp$$reg), __ D,
1561                 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1562     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1563     __ andr($dst$$Register, $dst$$Register, $src1$$Register);

1564   %}
1565   ins_pipe(pipe_slow);
1566 %}
1567 
1568 // vector or reduction
1569 
1570 instruct reduce_orI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp) %{
1571   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&

1572             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1573   match(Set dst (OrReductionV src1 src2));
1574   effect(TEMP_DEF dst, TEMP vtmp);
1575   ins_cost(SVE_COST);
1576   format %{ "sve_reduce_orI $dst, $src1, $src2\t# orB/S/I reduction (sve) (may extend)" %}
1577   ins_encode %{
1578     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1579     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1580     __ sve_orv(as_FloatRegister($vtmp$$reg), variant, ptrue, as_FloatRegister($src2$$reg));
1581     __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1582     __ orrw($dst$$Register, $dst$$Register, $src1$$Register);
1583     if (bt == T_BYTE) {
1584       __ sxtb($dst$$Register, $dst$$Register);
1585     } else if (bt == T_SHORT) {
1586       __ sxth($dst$$Register, $dst$$Register);
1587     } else {
1588       assert(bt == T_INT, "unsupported type");
1589     }








1590   %}
1591   ins_pipe(pipe_slow);
1592 %}
1593 
1594 instruct reduce_orI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1595                              pRegGov ptmp, rFlagsReg cr) %{
1596   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&

1597             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1598   match(Set dst (OrReductionV src1 src2));
1599   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1600   ins_cost(SVE_COST);
1601   format %{ "sve_reduce_orI $dst, $src1, $src2\t# orI reduction partial (sve) (may extend)" %}
1602   ins_encode %{
1603     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1604     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1605     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1606                           Matcher::vector_length(this, $src2));
1607     __ sve_orv(as_FloatRegister($vtmp$$reg), variant,
1608                as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1609     __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1610     __ orrw($dst$$Register, $dst$$Register, $src1$$Register);
1611     if (bt == T_BYTE) {
1612       __ sxtb($dst$$Register, $dst$$Register);
1613     } else if (bt == T_SHORT) {
1614       __ sxth($dst$$Register, $dst$$Register);
1615     } else {
1616       assert(bt == T_INT, "unsupported type");
1617     }
1618   %}
1619   ins_pipe(pipe_slow);
1620 %}
1621 
1622 instruct reduce_orL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp) %{
1623   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1624             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);


1625   match(Set dst (OrReductionV src1 src2));
1626   effect(TEMP_DEF dst, TEMP vtmp);




















1627   ins_cost(SVE_COST);
1628   format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction (sve)" %}
1629   ins_encode %{
1630     __ sve_orv(as_FloatRegister($vtmp$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
1631     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1632     __ orr($dst$$Register, $dst$$Register, $src1$$Register);

1633   %}
1634   ins_pipe(pipe_slow);
1635 %}
1636 
1637 instruct reduce_orL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,































































































1638                              pRegGov ptmp, rFlagsReg cr) %{
1639   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&

1640             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1641   match(Set dst (OrReductionV src1 src2));
1642   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1643   ins_cost(SVE_COST);
1644   format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction partial (sve)" %}





















1645   ins_encode %{
1646     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1647                           Matcher::vector_length(this, $src2));
1648     __ sve_orv(as_FloatRegister($vtmp$$reg), __ D,
1649                as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1650     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1651     __ orr($dst$$Register, $dst$$Register, $src1$$Register);
1652   %}
1653   ins_pipe(pipe_slow);
1654 %}
1655 
1656 // vector xor reduction
1657 
1658 instruct reduce_eorI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp) %{
1659   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1660             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1661   match(Set dst (XorReductionV src1 src2));
1662   effect(TEMP_DEF dst, TEMP vtmp);

1663   ins_cost(SVE_COST);
1664   format %{ "sve_reduce_eorI $dst, $src1, $src2\t# xorB/H/I reduction (sve) (may extend)" %}
1665   ins_encode %{
1666     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1667     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1668     __ sve_eorv(as_FloatRegister($vtmp$$reg), variant, ptrue, as_FloatRegister($src2$$reg));
1669     __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1670     __ eorw($dst$$Register, $dst$$Register, $src1$$Register);
1671     if (bt == T_BYTE) {
1672       __ sxtb($dst$$Register, $dst$$Register);
1673     } else if (bt == T_SHORT) {
1674       __ sxth($dst$$Register, $dst$$Register);
1675     } else {
1676       assert(bt == T_INT, "unsupported type");
1677     }
1678   %}
1679   ins_pipe(pipe_slow);
1680 %}
1681 
1682 instruct reduce_eorI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1683                              pRegGov ptmp, rFlagsReg cr) %{
1684   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1685             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1686   match(Set dst (XorReductionV src1 src2));
1687   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1688   ins_cost(SVE_COST);
1689   format %{ "sve_reduce_eorI $dst, $src1, $src2\t# xorI reduction partial (sve) (may extend)" %}
1690   ins_encode %{
1691     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1692     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1693     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1694                           Matcher::vector_length(this, $src2));
1695     __ sve_eorv(as_FloatRegister($vtmp$$reg), variant,
1696                 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1697     __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1698     __ eorw($dst$$Register, $dst$$Register, $src1$$Register);
1699     if (bt == T_BYTE) {
1700       __ sxtb($dst$$Register, $dst$$Register);
1701     } else if (bt == T_SHORT) {
1702       __ sxth($dst$$Register, $dst$$Register);
1703     } else {
1704       assert(bt == T_INT, "unsupported type");
1705     }
1706   %}
1707   ins_pipe(pipe_slow);
1708 %}
1709 
1710 instruct reduce_eorL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp) %{
1711   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1712             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1713   match(Set dst (XorReductionV src1 src2));
1714   effect(TEMP_DEF dst, TEMP vtmp);
1715   ins_cost(SVE_COST);
1716   format %{ "sve_reduce_eorL $dst, $src1, $src2\t# xorL reduction (sve)" %}


1717   ins_encode %{
1718     __ sve_eorv(as_FloatRegister($vtmp$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
1719     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1720     __ eor($dst$$Register, $dst$$Register, $src1$$Register);






1721   %}
1722   ins_pipe(pipe_slow);
1723 %}
1724 
1725 instruct reduce_eorL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1726                              pRegGov ptmp, rFlagsReg cr) %{
1727   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1728             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1729   match(Set dst (XorReductionV src1 src2));

1730   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1731   ins_cost(SVE_COST);
1732   format %{ "sve_reduce_eorL $dst, $src1, $src2\t# xorL reduction partial (sve)" %}
1733   ins_encode %{
1734     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1735                           Matcher::vector_length(this, $src2));
1736     __ sve_eorv(as_FloatRegister($vtmp$$reg), __ D,
1737                 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1738     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1739     __ eor($dst$$Register, $dst$$Register, $src1$$Register);

1740   %}
1741   ins_pipe(pipe_slow);
1742 %}
1743 
1744 
1745 // vector max reduction
1746 
1747 instruct reduce_maxI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp) %{
1748   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1749             (n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
1750              n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
1751              n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT));
1752   match(Set dst (MaxReductionV src1 src2));
1753   effect(TEMP_DEF dst, TEMP vtmp);
1754   ins_cost(SVE_COST);
1755   format %{ "sve_reduce_maxI $dst, $src1, $src2\t# reduce maxB/S/I (sve)" %}
1756   ins_encode %{
1757     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1758     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1759     __ sve_smaxv(as_FloatRegister($vtmp$$reg), variant, ptrue, as_FloatRegister($src2$$reg));
1760     __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1761     __ cmpw($dst$$Register, $src1$$Register);
1762     __ cselw(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($src1$$reg), Assembler::GT);
1763   %}
1764   ins_pipe(pipe_slow);
1765 %}
1766 
1767 instruct reduce_maxI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1768                              pRegGov ptmp, rFlagsReg cr) %{
1769   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1770             (n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
1771              n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
1772              n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT));
1773   match(Set dst (MaxReductionV src1 src2));
1774   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1775   ins_cost(SVE_COST);
1776   format %{ "sve_reduce_maxI $dst, $src1, $src2\t# reduce maxI partial (sve)" %}
1777   ins_encode %{
1778     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1779     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1780     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1781                           Matcher::vector_length(this, $src2));
1782     __ sve_smaxv(as_FloatRegister($vtmp$$reg), variant,
1783                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1784     __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1785     __ cmpw($dst$$Register, $src1$$Register);
1786     __ cselw(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($src1$$reg), Assembler::GT);
1787   %}
1788   ins_pipe(pipe_slow);
1789 %}
1790 
1791 instruct reduce_maxL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp) %{
1792   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1793             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);



1794   match(Set dst (MaxReductionV src1 src2));
1795   effect(TEMP_DEF dst, TEMP vtmp);
1796   ins_cost(SVE_COST);
1797   format %{ "sve_reduce_maxL $dst, $src1, $src2\t# reduce maxL partial (sve)" %}
1798   ins_encode %{
1799     __ sve_smaxv(as_FloatRegister($vtmp$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
1800     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1801     __ cmp($dst$$Register, $src1$$Register);
1802     __ csel(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($src1$$reg), Assembler::GT);



1803   %}
1804   ins_pipe(pipe_slow);
1805 %}
1806 
1807 instruct reduce_maxL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1808                              pRegGov ptmp, rFlagsReg cr) %{
1809   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&

1810             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1811   match(Set dst (MaxReductionV src1 src2));
1812   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1813   ins_cost(SVE_COST);
1814   format %{ "sve_reduce_maxL $dst, $src1, $src2\t# reduce maxL partial (sve)" %}
1815   ins_encode %{
1816     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1817                           Matcher::vector_length(this, $src2));
1818     __ sve_smaxv(as_FloatRegister($vtmp$$reg), __ D,
1819                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1820     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1821     __ cmp($dst$$Register, $src1$$Register);
1822     __ csel(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($src1$$reg), Assembler::GT);
1823   %}
1824   ins_pipe(pipe_slow);
1825 %}
1826 
1827 instruct reduce_maxF(vRegF dst, vRegF src1, vReg src2) %{
1828   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&

1829             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1830   match(Set dst (MaxReductionV src1 src2));
1831   ins_cost(INSN_COST);
1832   effect(TEMP_DEF dst);
1833   format %{ "sve_fmaxv $dst, $src2 # vector (sve) (S)\n\t"
1834             "fmaxs $dst, $dst, $src1\t# max reduction F" %}
1835   ins_encode %{
1836     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S,
1837          ptrue, as_FloatRegister($src2$$reg));
1838     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1839   %}
1840   ins_pipe(pipe_slow);
1841 %}
1842 
1843 instruct reduce_maxF_partial(vRegF dst, vRegF src1, vReg src2,
1844                              pRegGov ptmp, rFlagsReg cr) %{
1845   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&

1846             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1847   match(Set dst (MaxReductionV src1 src2));
1848   ins_cost(INSN_COST);
1849   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
1850   format %{ "sve_reduce_maxF $dst, $src1, $src2\t# reduce max S partial (sve)" %}
1851   ins_encode %{
1852     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
1853                           Matcher::vector_length(this, $src2));
1854     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S,
1855          as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1856     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1857   %}
1858   ins_pipe(pipe_slow);
1859 %}
1860 
1861 instruct reduce_maxD(vRegD dst, vRegD src1, vReg src2) %{
1862   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&

1863             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1864   match(Set dst (MaxReductionV src1 src2));
1865   ins_cost(INSN_COST);
1866   effect(TEMP_DEF dst);
1867   format %{ "sve_fmaxv $dst, $src2 # vector (sve) (D)\n\t"
1868             "fmaxs $dst, $dst, $src1\t# max reduction D" %}
1869   ins_encode %{
1870     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D,
1871          ptrue, as_FloatRegister($src2$$reg));
1872     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1873   %}
1874   ins_pipe(pipe_slow);
1875 %}
1876 
1877 instruct reduce_maxD_partial(vRegD dst, vRegD src1, vReg src2,
1878                              pRegGov ptmp, rFlagsReg cr) %{
1879   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&

1880             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1881   match(Set dst (MaxReductionV src1 src2));
1882   ins_cost(INSN_COST);
1883   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
1884   format %{ "sve_reduce_maxD $dst, $src1, $src2\t# reduce max D partial (sve)" %}
1885   ins_encode %{
1886     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1887                           Matcher::vector_length(this, $src2));
1888     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D,
1889          as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1890     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1891   %}
1892   ins_pipe(pipe_slow);
1893 %}
1894 
1895 // vector min reduction
1896 
1897 instruct reduce_minI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp) %{
1898   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1899             (n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
1900              n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
1901              n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT));
1902   match(Set dst (MinReductionV src1 src2));
1903   effect(TEMP_DEF dst, TEMP vtmp);

1904   ins_cost(SVE_COST);
1905   format %{ "sve_reduce_minI $dst, $src1, $src2\t# reduce minB/S/I (sve)" %}
1906   ins_encode %{
1907     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1908     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1909     __ sve_sminv(as_FloatRegister($vtmp$$reg), variant, ptrue, as_FloatRegister($src2$$reg));
1910     __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1911     __ cmpw($dst$$Register, $src1$$Register);
1912     __ cselw(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($src1$$reg), Assembler::LT);
1913   %}
1914   ins_pipe(pipe_slow);
1915 %}
1916 
1917 instruct reduce_minI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1918                              pRegGov ptmp, rFlagsReg cr) %{
1919   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1920             (n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
1921              n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
1922              n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT));
1923   match(Set dst (MinReductionV src1 src2));
1924   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1925   ins_cost(SVE_COST);
1926   format %{ "sve_reduce_minI $dst, $src1, $src2\t# reduce minI partial (sve)" %}


















1927   ins_encode %{
1928     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1929     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1930     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1931                           Matcher::vector_length(this, $src2));
1932     __ sve_sminv(as_FloatRegister($vtmp$$reg), variant,
1933                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1934     __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1935     __ cmpw($dst$$Register, $src1$$Register);
1936     __ cselw(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($src1$$reg), Assembler::LT);















































































































1937   %}
1938   ins_pipe(pipe_slow);
1939 %}
1940 
1941 instruct reduce_minL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp) %{
1942   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&

1943             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1944   match(Set dst (MinReductionV src1 src2));
1945   effect(TEMP_DEF dst, TEMP vtmp);
1946   ins_cost(SVE_COST);
1947   format %{ "sve_reduce_minL $dst, $src1, $src2\t# reduce minL partial (sve)" %}
1948   ins_encode %{
1949     __ sve_sminv(as_FloatRegister($vtmp$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
1950     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1951     __ cmp($dst$$Register, $src1$$Register);
1952     __ csel(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($src1$$reg), Assembler::LT);





















1953   %}
1954   ins_pipe(pipe_slow);
1955 %}
1956 
1957 instruct reduce_minL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1958                              pRegGov ptmp, rFlagsReg cr) %{
1959   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&

1960             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1961   match(Set dst (MinReductionV src1 src2));
1962   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1963   ins_cost(SVE_COST);
1964   format %{ "sve_reduce_minL $dst, $src1, $src2\t# reduce minL partial (sve)" %}
1965   ins_encode %{
1966     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1967                           Matcher::vector_length(this, $src2));
1968     __ sve_sminv(as_FloatRegister($vtmp$$reg), __ D,
1969                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1970     __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1971     __ cmp($dst$$Register, $src1$$Register);
1972     __ csel(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($src1$$reg), Assembler::LT);
1973   %}
1974   ins_pipe(pipe_slow);
1975 %}
1976 
1977 instruct reduce_minF(vRegF dst, vRegF src1, vReg src2) %{
1978   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&

1979             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1980   match(Set dst (MinReductionV src1 src2));
1981   ins_cost(INSN_COST);
1982   effect(TEMP_DEF dst);
1983   format %{ "sve_fminv $dst, $src2 # vector (sve) (S)\n\t"
1984             "fmins $dst, $dst, $src1\t# min reduction F" %}
1985   ins_encode %{
1986     __ sve_fminv(as_FloatRegister($dst$$reg), __ S,
1987          ptrue, as_FloatRegister($src2$$reg));
1988     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1989   %}
1990   ins_pipe(pipe_slow);
1991 %}
1992 
1993 instruct reduce_minF_partial(vRegF dst, vRegF src1, vReg src2,
1994                              pRegGov ptmp, rFlagsReg cr) %{
1995   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&

1996             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1997   match(Set dst (MinReductionV src1 src2));
1998   ins_cost(INSN_COST);
1999   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
2000   format %{ "sve_reduce_minF $dst, $src1, $src2\t# reduce min S partial (sve)" %}
2001   ins_encode %{
2002     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2003                           Matcher::vector_length(this, $src2));
2004     __ sve_fminv(as_FloatRegister($dst$$reg), __ S,
2005          as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2006     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2007   %}
2008   ins_pipe(pipe_slow);
2009 %}
2010 
2011 instruct reduce_minD(vRegD dst, vRegD src1, vReg src2) %{
2012   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&

2013             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2014   match(Set dst (MinReductionV src1 src2));
2015   ins_cost(INSN_COST);
2016   effect(TEMP_DEF dst);
2017   format %{ "sve_fminv $dst, $src2 # vector (sve) (D)\n\t"
2018             "fmins $dst, $dst, $src1\t# min reduction D" %}
2019   ins_encode %{
2020     __ sve_fminv(as_FloatRegister($dst$$reg), __ D,
2021          ptrue, as_FloatRegister($src2$$reg));
































































































































2022     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2023   %}
2024   ins_pipe(pipe_slow);
2025 %}
2026 
2027 instruct reduce_minD_partial(vRegD dst, vRegD src1, vReg src2,
2028                              pRegGov ptmp, rFlagsReg cr) %{
2029   predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
2030             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2031   match(Set dst (MinReductionV src1 src2));
2032   ins_cost(INSN_COST);





















2033   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
2034   format %{ "sve_reduce_minD $dst, $src1, $src2\t# reduce min D partial (sve)" %}

2035   ins_encode %{
2036     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2037                           Matcher::vector_length(this, $src2));


2038     __ sve_fminv(as_FloatRegister($dst$$reg), __ D,
2039          as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2040     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2041   %}
2042   ins_pipe(pipe_slow);
2043 %}
2044 
2045 // vector Math.rint, floor, ceil
2046 
2047 instruct vroundD(vReg dst, vReg src, immI rmode) %{
2048   predicate(UseSVE > 0 &&
2049             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
2050   match(Set dst (RoundDoubleModeV src rmode));
2051   format %{ "sve_frint $dst, $src, $rmode\t# vector (sve) (D)" %}
2052   ins_encode %{
2053     switch ($rmode$$constant) {
2054       case RoundDoubleModeNode::rmode_rint:
2055         __ sve_frintn(as_FloatRegister($dst$$reg), __ D,
2056              ptrue, as_FloatRegister($src$$reg));
2057         break;
2058       case RoundDoubleModeNode::rmode_floor:
2059         __ sve_frintm(as_FloatRegister($dst$$reg), __ D,

2449 %}
2450 
2451 instruct vlsrI_imm(vReg dst, vReg src, immI shift) %{
2452   predicate(UseSVE > 0);
2453   match(Set dst (URShiftVI src (RShiftCntV shift)));
2454   ins_cost(SVE_COST);
2455   format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (S)" %}
2456   ins_encode %{
2457     int con = (int)$shift$$constant;
2458     if (con == 0) {
2459       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
2460            as_FloatRegister($src$$reg));
2461       return;
2462     }
2463     __ sve_lsr(as_FloatRegister($dst$$reg), __ S,
2464          as_FloatRegister($src$$reg), con);
2465   %}
2466   ins_pipe(pipe_slow);
2467 %}
2468 
2469 instruct vlsrL_imm(vReg dst, vReg src, immI shift) %{






































































































































































































































































































































2470   predicate(UseSVE > 0);
2471   match(Set dst (URShiftVL src (RShiftCntV shift)));
2472   ins_cost(SVE_COST);
2473   format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (D)" %}
2474   ins_encode %{
2475     int con = (int)$shift$$constant;
2476     if (con == 0) {
2477       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
2478            as_FloatRegister($src$$reg));
2479       return;
2480     }
2481     __ sve_lsr(as_FloatRegister($dst$$reg), __ D,
2482          as_FloatRegister($src$$reg), con);
2483   %}
2484   ins_pipe(pipe_slow);
2485 %}
2486 
2487 instruct vlslB_imm(vReg dst, vReg src, immI shift) %{
2488   predicate(UseSVE > 0);
2489   match(Set dst (LShiftVB src (LShiftCntV shift)));
2490   ins_cost(SVE_COST);
2491   format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (B)" %}
2492   ins_encode %{
2493     int con = (int)$shift$$constant;
2494     if (con >= 8) {
2495       __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
2496            as_FloatRegister($src$$reg));
2497       return;
2498     }
2499     __ sve_lsl(as_FloatRegister($dst$$reg), __ B,
2500          as_FloatRegister($src$$reg), con);
2501   %}
2502   ins_pipe(pipe_slow);
2503 %}
2504 
2505 instruct vlslS_imm(vReg dst, vReg src, immI shift) %{
2506   predicate(UseSVE > 0);
2507   match(Set dst (LShiftVS src (LShiftCntV shift)));
2508   ins_cost(SVE_COST);
2509   format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (H)" %}
2510   ins_encode %{
2511     int con = (int)$shift$$constant;
2512     if (con >= 16) {
2513       __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
2514            as_FloatRegister($src$$reg));
2515       return;
2516     }
2517     __ sve_lsl(as_FloatRegister($dst$$reg), __ H,
2518          as_FloatRegister($src$$reg), con);
2519   %}
2520   ins_pipe(pipe_slow);
2521 %}
2522 
2523 instruct vlslI_imm(vReg dst, vReg src, immI shift) %{
2524   predicate(UseSVE > 0);
2525   match(Set dst (LShiftVI src (LShiftCntV shift)));
2526   ins_cost(SVE_COST);
2527   format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (S)" %}
2528   ins_encode %{
2529     int con = (int)$shift$$constant;
2530     __ sve_lsl(as_FloatRegister($dst$$reg), __ S,
2531          as_FloatRegister($src$$reg), con);
2532   %}
2533   ins_pipe(pipe_slow);
2534 %}
2535 
2536 instruct vlslL_imm(vReg dst, vReg src, immI shift) %{
2537   predicate(UseSVE > 0);
2538   match(Set dst (LShiftVL src (LShiftCntV shift)));
2539   ins_cost(SVE_COST);
2540   format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (D)" %}
2541   ins_encode %{
2542     int con = (int)$shift$$constant;
2543     __ sve_lsl(as_FloatRegister($dst$$reg), __ D,
2544          as_FloatRegister($src$$reg), con);
2545   %}
2546   ins_pipe(pipe_slow);
2547 %}
2548 
2549 instruct vshiftcntB(vReg dst, iRegIorL2I cnt) %{
2550   predicate(UseSVE > 0 &&
2551             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE));
2552   match(Set dst (LShiftCntV cnt));
2553   match(Set dst (RShiftCntV cnt));
2554   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (B)" %}
2555   ins_encode %{
2556     __ sve_dup(as_FloatRegister($dst$$reg), __ B, as_Register($cnt$$reg));


2557   %}
2558   ins_pipe(pipe_slow);
2559 %}
2560 
2561 instruct vshiftcntS(vReg dst, iRegIorL2I cnt) %{
2562   predicate(UseSVE > 0 &&
2563             (n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2564             (n->bottom_type()->is_vect()->element_basic_type() == T_CHAR)));
2565   match(Set dst (LShiftCntV cnt));
2566   match(Set dst (RShiftCntV cnt));
2567   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (H)" %}
2568   ins_encode %{
2569     __ sve_dup(as_FloatRegister($dst$$reg), __ H, as_Register($cnt$$reg));


2570   %}
2571   ins_pipe(pipe_slow);
2572 %}
2573 
2574 instruct vshiftcntI(vReg dst, iRegIorL2I cnt) %{
2575   predicate(UseSVE > 0 &&
2576             (n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2577   match(Set dst (LShiftCntV cnt));
2578   match(Set dst (RShiftCntV cnt));
2579   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (S)" %}
2580   ins_encode %{
2581     __ sve_dup(as_FloatRegister($dst$$reg), __ S, as_Register($cnt$$reg));


2582   %}
2583   ins_pipe(pipe_slow);
2584 %}
2585 
2586 instruct vshiftcntL(vReg dst, iRegIorL2I cnt) %{
2587   predicate(UseSVE > 0 &&
2588             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
2589   match(Set dst (LShiftCntV cnt));
2590   match(Set dst (RShiftCntV cnt));
2591   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (D)" %}
2592   ins_encode %{
2593     __ sve_dup(as_FloatRegister($dst$$reg), __ D, as_Register($cnt$$reg));


2594   %}
2595   ins_pipe(pipe_slow);
2596 %}
2597 
2598 // vector sqrt
2599 
2600 instruct vsqrtF(vReg dst, vReg src) %{
2601   predicate(UseSVE > 0);

2602   match(Set dst (SqrtVF src));
2603   ins_cost(SVE_COST);
2604   format %{ "sve_fsqrt $dst, $src\t# vector (sve) (S)" %}
2605   ins_encode %{
2606     __ sve_fsqrt(as_FloatRegister($dst$$reg), __ S,
2607          ptrue, as_FloatRegister($src$$reg));
2608   %}
2609   ins_pipe(pipe_slow);
2610 %}
2611 
2612 instruct vsqrtD(vReg dst, vReg src) %{
2613   predicate(UseSVE > 0);

2614   match(Set dst (SqrtVD src));
2615   ins_cost(SVE_COST);
2616   format %{ "sve_fsqrt $dst, $src\t# vector (sve) (D)" %}
2617   ins_encode %{
2618     __ sve_fsqrt(as_FloatRegister($dst$$reg), __ D,
2619          ptrue, as_FloatRegister($src$$reg));
2620   %}
2621   ins_pipe(pipe_slow);
2622 %}
2623 




























2624 // vector sub
2625 
2626 instruct vsubB(vReg dst, vReg src1, vReg src2) %{
2627   predicate(UseSVE > 0);
2628   match(Set dst (SubVB src1 src2));
2629   ins_cost(SVE_COST);
2630   format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (B)" %}
2631   ins_encode %{
2632     __ sve_sub(as_FloatRegister($dst$$reg), __ B,
2633          as_FloatRegister($src1$$reg),
2634          as_FloatRegister($src2$$reg));
2635   %}
2636   ins_pipe(pipe_slow);
2637 %}
2638 
2639 instruct vsubS(vReg dst, vReg src1, vReg src2) %{
2640   predicate(UseSVE > 0);
2641   match(Set dst (SubVS src1 src2));
2642   ins_cost(SVE_COST);
2643   format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (H)" %}

2684     __ sve_fsub(as_FloatRegister($dst$$reg), __ S,
2685          as_FloatRegister($src1$$reg),
2686          as_FloatRegister($src2$$reg));
2687   %}
2688   ins_pipe(pipe_slow);
2689 %}
2690 
2691 instruct vsubD(vReg dst, vReg src1, vReg src2) %{
2692   predicate(UseSVE > 0);
2693   match(Set dst (SubVD src1 src2));
2694   ins_cost(SVE_COST);
2695   format %{ "sve_fsub $dst, $src1, $src2\t # vector (sve) (D)" %}
2696   ins_encode %{
2697     __ sve_fsub(as_FloatRegister($dst$$reg), __ D,
2698          as_FloatRegister($src1$$reg),
2699          as_FloatRegister($src2$$reg));
2700   %}
2701   ins_pipe(pipe_slow);
2702 %}
2703 
2704 // vector mask cast
2705 
2706 instruct vmaskcast(vReg dst) %{
2707   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
2708             n->bottom_type()->is_vect()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
2709   match(Set dst (VectorMaskCast dst));
2710   ins_cost(0);
2711   format %{ "vmaskcast $dst\t# empty (sve)" %}
2712   ins_encode %{
2713     // empty


2714   %}
2715   ins_pipe(pipe_class_empty);
2716 %}
2717 
2718 // ------------------------------ Vector cast -------------------------------











2719 
2720 instruct vcvtBtoS(vReg dst, vReg src)
2721 %{
2722   predicate(UseSVE > 0 &&
2723             n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
2724   match(Set dst (VectorCastB2X src));
2725   ins_cost(SVE_COST);
2726   format %{ "sve_sunpklo  $dst, H, $src\t# convert B to S vector" %}
2727   ins_encode %{
2728     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));


2729   %}
2730   ins_pipe(pipe_slow);
2731 %}
2732 
2733 instruct vcvtBtoI(vReg dst, vReg src)
2734 %{
2735   predicate(UseSVE > 0 &&
2736             n->bottom_type()->is_vect()->element_basic_type() == T_INT);
2737   match(Set dst (VectorCastB2X src));
2738   ins_cost(2 * SVE_COST);
2739   format %{ "sve_sunpklo  $dst, H, $src\n\t"
2740             "sve_sunpklo  $dst, S, $dst\t# convert B to I vector" %}
2741   ins_encode %{
2742     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
2743     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($dst$$reg));

2744   %}
2745   ins_pipe(pipe_slow);
2746 %}
2747 
2748 instruct vcvtBtoL(vReg dst, vReg src)
2749 %{
2750   predicate(UseSVE > 0 &&
2751             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2752   match(Set dst (VectorCastB2X src));
2753   ins_cost(3 * SVE_COST);
2754   format %{ "sve_sunpklo  $dst, H, $src\n\t"
2755             "sve_sunpklo  $dst, S, $dst\n\t"
2756             "sve_sunpklo  $dst, D, $dst\t# convert B to L vector" %}
2757   ins_encode %{
2758     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
2759     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($dst$$reg));
2760     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($dst$$reg));
2761   %}
2762   ins_pipe(pipe_slow);
2763 %}
2764 
2765 instruct vcvtBtoF(vReg dst, vReg src)
2766 %{
2767   predicate(UseSVE > 0 &&
2768             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2769   match(Set dst (VectorCastB2X src));
2770   ins_cost(3 * SVE_COST);
2771   format %{ "sve_sunpklo  $dst, H, $src\n\t"
2772             "sve_sunpklo  $dst, S, $dst\n\t"
2773             "sve_scvtf  $dst, S, $dst, S\t# convert B to F vector" %}
2774   ins_encode %{
2775     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
2776     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($dst$$reg));
2777     __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($dst$$reg), __ S);
2778   %}
2779   ins_pipe(pipe_slow);
2780 %}
2781 
2782 instruct vcvtBtoD(vReg dst, vReg src)
2783 %{

2784   predicate(UseSVE > 0 &&
2785             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
2786   match(Set dst (VectorCastB2X src));
2787   ins_cost(4 * SVE_COST);
2788   format %{ "sve_sunpklo  $dst, H, $src\n\t"
2789             "sve_sunpklo  $dst, S, $dst\n\t"
2790             "sve_sunpklo  $dst, D, $dst\n\t"
2791             "sve_scvtf  $dst, D, $dst, D\t# convert B to D vector" %}
2792   ins_encode %{
2793     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
2794     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($dst$$reg));
2795     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($dst$$reg));
2796     __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ D);
2797   %}
2798   ins_pipe(pipe_slow);
2799 %}
2800 
2801 instruct vcvtStoB(vReg dst, vReg src, vReg tmp)
2802 %{
2803   predicate(UseSVE > 0 &&
2804             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2805   match(Set dst (VectorCastS2X src));
2806   effect(TEMP tmp);
2807   ins_cost(2 * SVE_COST);
2808   format %{ "sve_dup  $tmp, B, 0\n\t"
2809             "sve_uzp1  $dst, B, $src, tmp\t# convert S to B vector" %}
2810   ins_encode %{
2811     __ sve_dup(as_FloatRegister($tmp$$reg), __ B, 0);
2812     __ sve_uzp1(as_FloatRegister($dst$$reg), __ B, as_FloatRegister($src$$reg), as_FloatRegister($tmp$$reg));
2813   %}
2814   ins_pipe(pipe_slow);
2815 %}
2816 
2817 instruct vcvtStoI(vReg dst, vReg src)
2818 %{
2819   predicate(UseSVE > 0 &&
2820             n->bottom_type()->is_vect()->element_basic_type() == T_INT);
2821   match(Set dst (VectorCastS2X src));
2822   ins_cost(SVE_COST);
2823   format %{ "sve_sunpklo  $dst, S, $src\t# convert S to I vector" %}


2824   ins_encode %{
2825     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($src$$reg));

2826   %}
2827   ins_pipe(pipe_slow);
2828 %}
2829 
2830 instruct vcvtStoL(vReg dst, vReg src)


2831 %{
2832   predicate(UseSVE > 0 &&
2833             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2834   match(Set dst (VectorCastS2X src));
2835   ins_cost(2 * SVE_COST);
2836   format %{ "sve_sunpklo  $dst, S, $src\n\t"
2837             "sve_sunpklo  $dst, D, $dst\t# convert S to L vector" %}
2838   ins_encode %{
2839     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($src$$reg));
2840     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($dst$$reg));




2841   %}
2842   ins_pipe(pipe_slow);
2843 %}
2844 
2845 instruct vcvtStoF(vReg dst, vReg src)
2846 %{
2847   predicate(UseSVE > 0 &&
2848             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2849   match(Set dst (VectorCastS2X src));

2850   ins_cost(2 * SVE_COST);
2851   format %{ "sve_sunpklo  $dst, S, $src\n\t"
2852             "sve_scvtf  $dst, S, $dst, S\t# convert S to F vector" %}
2853   ins_encode %{
2854     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($src$$reg));
2855     __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($dst$$reg), __ S);
2856   %}
2857   ins_pipe(pipe_slow);
2858 %}
2859 
2860 instruct vcvtStoD(vReg dst, vReg src)
2861 %{
2862   predicate(UseSVE > 0 &&
2863             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
2864   match(Set dst (VectorCastS2X src));
2865   ins_cost(3 * SVE_COST);
2866   format %{ "sve_sunpklo  $dst, S, $src\n\t"
2867             "sve_sunpklo  $dst, D, $dst\n\t"
2868             "sve_scvtf  $dst, D, $dst, D\t# convert S to D vector" %}
2869   ins_encode %{
2870     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($src$$reg));
2871     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($dst$$reg));
2872     __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ D);



2873   %}
2874   ins_pipe(pipe_slow);
2875 %}
2876 
2877 instruct vcvtItoB(vReg dst, vReg src, vReg tmp)
2878 %{
2879   predicate(UseSVE > 0 &&
2880             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2881   match(Set dst (VectorCastI2X src));
2882   effect(TEMP_DEF dst, TEMP tmp);
2883   ins_cost(3 * SVE_COST);
2884   format %{ "sve_dup  $tmp, H, 0\n\t"
2885             "sve_uzp1  $dst, H, $src, tmp\n\t"
2886             "sve_uzp1  $dst, B, $dst, tmp\n\t# convert I to B vector" %}
2887   ins_encode %{
2888     __ sve_dup(as_FloatRegister($tmp$$reg), __ H, 0);
2889     __ sve_uzp1(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg), as_FloatRegister($tmp$$reg));
2890     __ sve_uzp1(as_FloatRegister($dst$$reg), __ B, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
2891   %}
2892   ins_pipe(pipe_slow);
2893 %}
2894 
2895 instruct vcvtItoS(vReg dst, vReg src, vReg tmp)
2896 %{
2897   predicate(UseSVE > 0 &&
2898             n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
2899   match(Set dst (VectorCastI2X src));
2900   effect(TEMP tmp);
2901   ins_cost(2 * SVE_COST);
2902   format %{ "sve_dup  $tmp, H, 0\n\t"
2903             "sve_uzp1  $dst, H, $src, tmp\t# convert I to S vector" %}
2904   ins_encode %{
2905     __ sve_dup(as_FloatRegister($tmp$$reg), __ H, 0);
2906     __ sve_uzp1(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg), as_FloatRegister($tmp$$reg));
2907   %}
2908   ins_pipe(pipe_slow);
2909 %}
2910 
2911 instruct vcvtItoL(vReg dst, vReg src)
2912 %{
2913   predicate(UseSVE > 0 &&
2914             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2915   match(Set dst (VectorCastI2X src));
2916   ins_cost(SVE_COST);
2917   format %{ "sve_sunpklo  $dst, D, $src\t# convert I to L vector" %}
2918   ins_encode %{
2919     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
2920   %}
2921   ins_pipe(pipe_slow);
2922 %}
2923 
2924 instruct vcvtItoF(vReg dst, vReg src)
2925 %{
2926   predicate(UseSVE > 0 &&
2927             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2928   match(Set dst (VectorCastI2X src));
2929   ins_cost(SVE_COST);
2930   format %{ "sve_scvtf  $dst, S, $src, S\t# convert I to F vector" %}
2931   ins_encode %{
2932     __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
2933   %}
2934   ins_pipe(pipe_slow);
2935 %}
2936 
2937 instruct vcvtItoD(vReg dst, vReg src)
2938 %{
2939   predicate(UseSVE > 0 &&
2940             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
2941   match(Set dst (VectorCastI2X src));
2942   ins_cost(2 * SVE_COST);
2943   format %{ "sve_sunpklo  $dst, D, $src\n\t"
2944             "sve_scvtf  $dst, D, $dst, D\t# convert I to D vector" %}
2945   ins_encode %{
2946     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
2947     __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ D);
2948   %}
2949   ins_pipe(pipe_slow);
2950 %}
2951 
2952 instruct vcvtLtoB(vReg dst, vReg src, vReg tmp)
2953 %{
2954   predicate(UseSVE > 0 &&
2955             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2956   match(Set dst (VectorCastL2X src));
2957   effect(TEMP_DEF dst, TEMP tmp);
2958   ins_cost(4 * SVE_COST);
2959   format %{ "sve_dup  $tmp, S, 0\n\t"
2960             "sve_uzp1  $dst, S, $src, tmp\n\t"
2961             "sve_uzp1  $dst, H, $dst, tmp\n\t"
2962             "sve_uzp1  $dst, B, $dst, tmp\n\t# convert L to B vector" %}
2963   ins_encode %{
2964     __ sve_dup(as_FloatRegister($tmp$$reg), __ S, 0);
2965     __ sve_uzp1(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($src$$reg), as_FloatRegister($tmp$$reg));
2966     __ sve_uzp1(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
2967     __ sve_uzp1(as_FloatRegister($dst$$reg), __ B, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
2968   %}
2969   ins_pipe(pipe_slow);
2970 %}
2971 
2972 instruct vcvtLtoS(vReg dst, vReg src, vReg tmp)
2973 %{
2974   predicate(UseSVE > 0 &&
2975             n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
2976   match(Set dst (VectorCastL2X src));
2977   effect(TEMP_DEF dst, TEMP tmp);
2978   ins_cost(3 * SVE_COST);
2979   format %{ "sve_dup  $tmp, S, 0\n\t"
2980             "sve_uzp1  $dst, S, $src, tmp\n\t"
2981             "sve_uzp1  $dst, H, $dst, tmp\n\t# convert L to S vector" %}
2982   ins_encode %{
2983     __ sve_dup(as_FloatRegister($tmp$$reg), __ S, 0);
2984     __ sve_uzp1(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($src$$reg), as_FloatRegister($tmp$$reg));
2985     __ sve_uzp1(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
2986   %}
2987   ins_pipe(pipe_slow);
2988 %}
2989 
2990 instruct vcvtLtoI(vReg dst, vReg src, vReg tmp)
2991 %{
2992   predicate(UseSVE > 0 &&
2993             n->bottom_type()->is_vect()->element_basic_type() == T_INT);
2994   match(Set dst (VectorCastL2X src));
2995   effect(TEMP tmp);
2996   ins_cost(2 * SVE_COST);
2997   format %{ "sve_dup  $tmp, S, 0\n\t"
2998             "sve_uzp1  $dst, S, $src, tmp\t# convert L to I vector" %}
2999   ins_encode %{
3000     __ sve_dup(as_FloatRegister($tmp$$reg), __ S, 0);
3001     __ sve_uzp1(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($src$$reg), as_FloatRegister($tmp$$reg));


3002   %}
3003   ins_pipe(pipe_slow);
3004 %}
3005 
3006 instruct vcvtLtoF(vReg dst, vReg src, vReg tmp)
3007 %{
3008   predicate(UseSVE > 0 &&
3009             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
3010   match(Set dst (VectorCastL2X src));
3011   effect(TEMP_DEF dst, TEMP tmp);
3012   ins_cost(3 * SVE_COST);
3013   format %{ "sve_scvtf  $dst, S, $src, D\n\t"
3014             "sve_dup  $tmp, S, 0\n\t"
3015             "sve_uzp1  $dst, S, $dst, $tmp\t# convert L to F vector" %}
3016   ins_encode %{
3017     __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
3018     __ sve_dup(as_FloatRegister($tmp$$reg), __ S, 0);
3019     __ sve_uzp1(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));

3020   %}
3021   ins_pipe(pipe_slow);
3022 %}
3023 
3024 instruct vcvtLtoD(vReg dst, vReg src)
3025 %{
3026   predicate(UseSVE > 0 &&
3027             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
3028   match(Set dst (VectorCastL2X src));
3029   ins_cost(SVE_COST);
3030   format %{ "sve_scvtf  $dst, D, $src, D\t# convert L to D vector" %}
3031   ins_encode %{
3032     __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
3033   %}
3034   ins_pipe(pipe_slow);
3035 %}
3036 
3037 instruct vcvtFtoB(vReg dst, vReg src, vReg tmp)
3038 %{
3039   predicate(UseSVE > 0 &&
3040             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
3041   match(Set dst (VectorCastF2X src));
3042   effect(TEMP_DEF dst, TEMP tmp);
3043   ins_cost(4 * SVE_COST);
3044   format %{ "sve_fcvtzs  $dst, S, $src, S\n\t"
3045             "sve_dup  $tmp, H, 0\n\t"
3046             "sve_uzp1  $dst, H, $dst, tmp\n\t"
3047             "sve_uzp1  $dst, B, $dst, tmp\n\t# convert F to B vector" %}
3048   ins_encode %{
3049     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
3050     __ sve_dup(as_FloatRegister($tmp$$reg), __ H, 0);
3051     __ sve_uzp1(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
3052     __ sve_uzp1(as_FloatRegister($dst$$reg), __ B, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
3053   %}
3054   ins_pipe(pipe_slow);
3055 %}
3056 
3057 instruct vcvtFtoS(vReg dst, vReg src, vReg tmp)
3058 %{
3059   predicate(UseSVE > 0 &&
3060             n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);

3061   match(Set dst (VectorCastF2X src));
3062   effect(TEMP_DEF dst, TEMP tmp);
3063   ins_cost(3 * SVE_COST);
3064   format %{ "sve_fcvtzs  $dst, S, $src, S\n\t"
3065             "sve_dup  $tmp, H, 0\n\t"
3066             "sve_uzp1  $dst, H, $dst, tmp\t# convert F to S vector" %}
3067   ins_encode %{
3068     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
3069     __ sve_dup(as_FloatRegister($tmp$$reg), __ H, 0);
3070     __ sve_uzp1(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
3071   %}
3072   ins_pipe(pipe_slow);
3073 %}
3074 
3075 instruct vcvtFtoI(vReg dst, vReg src)
3076 %{
3077   predicate(UseSVE > 0 &&
3078             n->bottom_type()->is_vect()->element_basic_type() == T_INT);
3079   match(Set dst (VectorCastF2X src));
3080   ins_cost(SVE_COST);
3081   format %{ "sve_fcvtzs  $dst, S, $src, S\t# convert F to I vector" %}
3082   ins_encode %{
3083     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
3084   %}
3085   ins_pipe(pipe_slow);
3086 %}
3087 
3088 instruct vcvtFtoL(vReg dst, vReg src)
3089 %{
3090   predicate(UseSVE > 0 &&
3091             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3092   match(Set dst (VectorCastF2X src));
3093   ins_cost(2 * SVE_COST);
3094   format %{ "sve_fcvtzs  $dst, S, $src, S\n\t"
3095             "sve_sunpklo  $dst, D, $dst\t# convert F to L vector" %}
3096   ins_encode %{


3097     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
3098     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($dst$$reg));
3099   %}
3100   ins_pipe(pipe_slow);
3101 %}
3102 
3103 instruct vcvtFtoD(vReg dst, vReg src)
3104 %{
3105   predicate(UseSVE > 0 &&
3106             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
3107   match(Set dst (VectorCastF2X src));
3108   ins_cost(2 * SVE_COST);
3109   format %{ "sve_sunpklo  $dst, D, $src\n\t"
3110             "sve_fcvt  $dst, D, $dst, S\t# convert F to D vector" %}
3111   ins_encode %{
3112     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
3113     __ sve_fcvt(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);
3114   %}
3115   ins_pipe(pipe_slow);
3116 %}
3117 
3118 instruct vcvtDtoB(vReg dst, vReg src, vReg tmp)
3119 %{
3120   predicate(UseSVE > 0 &&
3121             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
3122   match(Set dst (VectorCastD2X src));
3123   effect(TEMP_DEF dst, TEMP tmp);
3124   ins_cost(5 * SVE_COST);
3125   format %{ "sve_fcvtzs  $dst, D, $src, D\n\t"
3126             "sve_dup  $tmp, S, 0\n\t"
3127             "sve_uzp1  $dst, S, $dst, tmp\n\t"
3128             "sve_uzp1  $dst, H, $dst, tmp\n\t"
3129             "sve_uzp1  $dst, B, $dst, tmp\n\t# convert D to B vector" %}
3130   ins_encode %{
3131     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
3132     __ sve_dup(as_FloatRegister($tmp$$reg), __ S, 0);
3133     __ sve_uzp1(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
3134     __ sve_uzp1(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
3135     __ sve_uzp1(as_FloatRegister($dst$$reg), __ B, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
3136   %}
3137   ins_pipe(pipe_slow);
3138 %}
3139 
3140 instruct vcvtDtoS(vReg dst, vReg src, vReg tmp)
3141 %{
3142   predicate(UseSVE > 0 &&
3143             n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
3144   match(Set dst (VectorCastD2X src));
3145   effect(TEMP_DEF dst, TEMP tmp);
3146   ins_cost(4 * SVE_COST);
3147   format %{ "sve_fcvtzs  $dst, D, $src, D\n\t"
3148             "sve_dup  $tmp, S, 0\n\t"
3149             "sve_uzp1  $dst, S, $dst, tmp\n\t"
3150             "sve_uzp1  $dst, H, $dst, tmp\n\t# convert D to S vector" %}
3151   ins_encode %{
3152     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
3153     __ sve_dup(as_FloatRegister($tmp$$reg), __ S, 0);
3154     __ sve_uzp1(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
3155     __ sve_uzp1(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
3156   %}
3157   ins_pipe(pipe_slow);
3158 %}
3159 
3160 instruct vcvtDtoI(vReg dst, vReg src, vReg tmp)
3161 %{
3162   predicate(UseSVE > 0 &&
3163             n->bottom_type()->is_vect()->element_basic_type() == T_INT);


3164   match(Set dst (VectorCastD2X src));
3165   effect(TEMP_DEF dst, TEMP tmp);
3166   ins_cost(3 * SVE_COST);
3167   format %{ "sve_fcvtzs  $dst, D, $src, D\n\t"
3168             "sve_dup  $tmp, S, 0\n\t"
3169             "sve_uzp1  $dst, S, $dst, tmp\t# convert D to I vector" %}
3170   ins_encode %{


3171     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
3172     __ sve_dup(as_FloatRegister($tmp$$reg), __ S, 0);
3173     __ sve_uzp1(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
3174   %}
3175   ins_pipe(pipe_slow);
3176 %}
3177 
3178 instruct vcvtDtoL(vReg dst, vReg src)
3179 %{
3180   predicate(UseSVE > 0 &&
3181             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3182   match(Set dst (VectorCastD2X src));
3183   ins_cost(SVE_COST);
3184   format %{ "sve_fcvtzs  $dst, D, $src, D\t# convert D to L vector" %}
3185   ins_encode %{
3186     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
3187   %}
3188   ins_pipe(pipe_slow);
3189 %}
3190 
3191 instruct vcvtDtoF(vReg dst, vReg src, vReg tmp)
3192 %{
3193   predicate(UseSVE > 0 &&
3194             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
3195   match(Set dst (VectorCastD2X src));
3196   effect(TEMP_DEF dst, TEMP tmp);
3197   ins_cost(3 * SVE_COST);
3198   format %{ "sve_fcvt  $dst, S, $src, D\n\t"
3199             "sve_dup  $tmp, S, 0\n\t"
3200             "sve_uzp1  $dst, S, $dst, $tmp\t# convert D to F vector" %}
3201   ins_encode %{
3202     __ sve_fcvt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
3203     __ sve_dup(as_FloatRegister($tmp$$reg), __ S, 0);
3204     __ sve_uzp1(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
3205   %}
3206   ins_pipe(pipe_slow);
3207 %}

3208 // ------------------------------ Vector extract ---------------------------------
3209 
3210 instruct extractB(iRegINoSp dst, vReg src, immI idx, pRegGov pTmp, rFlagsReg cr)
3211 %{
3212   predicate(UseSVE > 0);
3213   match(Set dst (ExtractB src idx));
3214   effect(TEMP pTmp, KILL cr);
3215   ins_cost(2 * SVE_COST);
3216   format %{ "sve_extract $dst, B, $pTmp, $src, $idx\n\t"
3217             "sbfmw $dst, $dst, 0U, 7U\t# extract from vector(B)" %}
3218   ins_encode %{
3219     __ sve_extract(as_Register($dst$$reg), __ B, as_PRegister($pTmp$$reg),
3220                    as_FloatRegister($src$$reg), (int)($idx$$constant));
3221     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 7U);
3222   %}
3223   ins_pipe(pipe_slow);
3224 %}
3225 
3226 instruct extractS(iRegINoSp dst, vReg src, immI idx, pRegGov pTmp, rFlagsReg cr)
3227 %{
3228   predicate(UseSVE > 0);
3229   match(Set dst (ExtractS src idx));
3230   effect(TEMP pTmp, KILL cr);
3231   ins_cost(2 * SVE_COST);
3232   format %{ "sve_extract $dst, H, $pTmp, $src, $idx\n\t"
3233             "sbfmw $dst, $dst, 0U, 15U\t# extract from vector(S)" %}
3234   ins_encode %{
3235     __ sve_extract(as_Register($dst$$reg), __ H, as_PRegister($pTmp$$reg),
3236                    as_FloatRegister($src$$reg), (int)($idx$$constant));
3237     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
3238   %}
3239   ins_pipe(pipe_slow);
3240 %}
3241 
3242 
3243 instruct extractI(iRegINoSp dst, vReg src, immI idx, pRegGov pTmp, rFlagsReg cr)
3244 %{
3245   predicate(UseSVE > 0);
3246   match(Set dst (ExtractI src idx));
3247   effect(TEMP pTmp, KILL cr);
3248   ins_cost(2 * SVE_COST);
3249   format %{ "sve_extract $dst, S, $pTmp, $src, $idx\t# extract from vector(I)" %}
3250   ins_encode %{
3251     __ sve_extract(as_Register($dst$$reg), __ S, as_PRegister($pTmp$$reg),
3252                    as_FloatRegister($src$$reg), (int)($idx$$constant));
3253   %}
3254   ins_pipe(pipe_slow);
3255 %}
3256 
3257 instruct extractL(iRegLNoSp dst, vReg src, immI idx, pRegGov pTmp, rFlagsReg cr)
3258 %{
3259   predicate(UseSVE > 0);
3260   match(Set dst (ExtractL src idx));
3261   effect(TEMP pTmp, KILL cr);
3262   ins_cost(2 * SVE_COST);
3263   format %{ "sve_extract $dst, D, $pTmp, $src, $idx\t# extract from vector(L)" %}
3264   ins_encode %{
3265     __ sve_extract(as_Register($dst$$reg), __ D, as_PRegister($pTmp$$reg),
3266                    as_FloatRegister($src$$reg), (int)($idx$$constant));
3267   %}
3268   ins_pipe(pipe_slow);
3269 %}
3270 
3271 instruct extractF(vRegF dst, vReg src, immI idx, pRegGov pTmp, rFlagsReg cr)
3272 %{
3273   predicate(UseSVE > 0);
3274   match(Set dst (ExtractF src idx));
3275   effect(TEMP pTmp, KILL cr);
3276   ins_cost(2 * SVE_COST);
3277   format %{ "sve_extract $dst, S, $pTmp, $src, $idx\t# extract from vector(F)" %}
3278   ins_encode %{
3279     __ sve_extract(as_FloatRegister($dst$$reg), __ S, as_PRegister($pTmp$$reg),
3280                    as_FloatRegister($src$$reg), (int)($idx$$constant));
3281   %}
3282   ins_pipe(pipe_slow);
3283 %}
3284 
3285 instruct extractD(vRegD dst, vReg src, immI idx, pRegGov pTmp, rFlagsReg cr)
3286 %{
3287   predicate(UseSVE > 0);
3288   match(Set dst (ExtractD src idx));
3289   effect(TEMP pTmp, KILL cr);
3290   ins_cost(2 * SVE_COST);
3291   format %{ "sve_extract $dst, D, $pTmp, $src, $idx\t# extract from vector(D)" %}
3292   ins_encode %{
3293     __ sve_extract(as_FloatRegister($dst$$reg), __ D, as_PRegister($pTmp$$reg),
3294                    as_FloatRegister($src$$reg), (int)($idx$$constant));
3295   %}
3296   ins_pipe(pipe_slow);
3297 %}
3298 
3299 // ------------------------------- VectorTest ----------------------------------
3300 
3301 instruct vtest_alltrue(iRegINoSp dst, vReg src1, vReg src2, pReg pTmp, rFlagsReg cr)
3302 %{
3303   predicate(UseSVE > 0 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&

3304             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
3305   match(Set dst (VectorTest src1 src2));
3306   effect(TEMP pTmp, KILL cr);
3307   ins_cost(SVE_COST);
3308   format %{ "sve_cmpeq $pTmp, $src1, 0\n\t"
3309             "csetw $dst, EQ\t# VectorTest (sve) - alltrue" %}
3310   ins_encode %{
3311     // "src2" is not used for sve.
3312     BasicType bt = Matcher::vector_element_basic_type(this, $src1);
3313     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3314     __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), size,
3315                ptrue, as_FloatRegister($src1$$reg), 0);
3316     __ csetw(as_Register($dst$$reg), Assembler::EQ);
3317   %}
3318   ins_pipe(pipe_slow);
3319 %}
3320 
3321 instruct vtest_anytrue(iRegINoSp dst, vReg src1, vReg src2, pReg pTmp, rFlagsReg cr)
3322 %{
3323   predicate(UseSVE > 0 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&

3324             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
3325   match(Set dst (VectorTest src1 src2));
3326   effect(TEMP pTmp, KILL cr);
3327   ins_cost(SVE_COST);
3328   format %{ "sve_cmpeq $pTmp, $src1, -1\n\t"
3329             "csetw $dst, NE\t# VectorTest (sve) - anytrue" %}
3330   ins_encode %{
3331     // "src2" is not used for sve.
3332     BasicType bt = Matcher::vector_element_basic_type(this, $src1);
3333     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3334     __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), size,
3335                ptrue, as_FloatRegister($src1$$reg), -1);
3336     __ csetw(as_Register($dst$$reg), Assembler::NE);
3337   %}
3338   ins_pipe(pipe_slow);
3339 %}
3340 
3341 instruct vtest_alltrue_partial(iRegINoSp dst, vReg src1, vReg src2, pRegGov pTmp, rFlagsReg cr)
3342 %{
3343   predicate(UseSVE > 0 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&

3344             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
3345   match(Set dst (VectorTest src1 src2));
3346   effect(TEMP pTmp, KILL cr);
3347   ins_cost(SVE_COST);
3348   format %{ "vtest_alltrue_partial $dst, $src1, $src2\t# VectorTest partial (sve) - alltrue" %}
3349   ins_encode %{
3350     // "src2" is not used for sve.
3351     BasicType bt = Matcher::vector_element_basic_type(this, $src1);
3352     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3353     __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), size,
3354                           Matcher::vector_length(this, $src1));
3355     __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), size,
3356                as_PRegister($pTmp$$reg), as_FloatRegister($src1$$reg), 0);
3357     __ csetw(as_Register($dst$$reg), Assembler::EQ);
3358   %}
3359   ins_pipe(pipe_slow);
3360 %}
3361 
3362 instruct vtest_anytrue_partial(iRegINoSp dst, vReg src1, vReg src2, pRegGov pTmp, rFlagsReg cr)
3363 %{
3364   predicate(UseSVE > 0 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&

3365             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
3366   match(Set dst (VectorTest src1 src2));
3367   effect(TEMP pTmp, KILL cr);
3368   ins_cost(SVE_COST);
3369   format %{ "vtest_anytrue_partial $dst, $src1, $src2\t# VectorTest partial (sve) - anytrue" %}
3370   ins_encode %{
3371     // "src2" is not used for sve.
3372     BasicType bt = Matcher::vector_element_basic_type(this, $src1);
3373     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3374     __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), size,
3375                           Matcher::vector_length(this, $src1));
3376     __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), size,
3377                as_PRegister($pTmp$$reg), as_FloatRegister($src1$$reg), -1);
3378     __ csetw(as_Register($dst$$reg), Assembler::NE);
3379   %}
3380   ins_pipe(pipe_slow);
3381 %}
3382 
3383 // ------------------------------ Vector insert ---------------------------------
3384 
3385 instruct insertI_small(vReg dst, vReg src, iRegIorL2I val, immI idx, pRegGov pTmp, rFlagsReg cr)
3386 %{
3387   predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
3388             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
3389              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
3390              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
3391   match(Set dst (VectorInsert (Binary src val) idx));
3392   effect(TEMP_DEF dst, TEMP pTmp, KILL cr);
3393   ins_cost(4 * SVE_COST);
3394   format %{ "sve_index $dst, -16, 1\t# (B/S/I)\n\t"
3395             "sve_cmpeq $pTmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
3396             "sve_orr $dst, $src, $src\n\t"
3397             "sve_cpy $dst, $pTmp, $val\t# insert into vector (B/S/I)" %}
3398   ins_encode %{
3399     BasicType bt = Matcher::vector_element_basic_type(this, $src);
3400     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3401     __ sve_index(as_FloatRegister($dst$$reg), size, -16, 1);
3402     __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), size, ptrue,
3403                as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
3404     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
3405     __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pTmp$$reg), as_Register($val$$reg));
3406   %}
3407   ins_pipe(pipe_slow);
3408 %}
3409 
3410 instruct insertF_small(vReg dst, vReg src, vRegF val, immI idx, pRegGov pTmp, rFlagsReg cr)
3411 %{
3412   predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
3413             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
3414   match(Set dst (VectorInsert (Binary src val) idx));
3415   effect(TEMP_DEF dst, TEMP pTmp, KILL cr);
3416   ins_cost(4 * SVE_COST);
3417   format %{ "sve_index $dst, S, -16, 1\n\t"
3418             "sve_cmpeq $pTmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
3419             "sve_orr $dst, $src, $src\n\t"
3420             "sve_cpy $dst, $pTmp, $val\t# insert into vector (F)" %}
3421   ins_encode %{
3422     __ sve_index(as_FloatRegister($dst$$reg), __ S, -16, 1);
3423     __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), __ S, ptrue,
3424                as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
3425     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
3426     __ sve_cpy(as_FloatRegister($dst$$reg), __ S, as_PRegister($pTmp$$reg), as_FloatRegister($val$$reg));
3427   %}
3428   ins_pipe(pipe_slow);
3429 %}
3430 
3431 instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp1, pRegGov pTmp, rFlagsReg cr)
3432 %{
3433   predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
3434             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
3435              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
3436              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
3437   match(Set dst (VectorInsert (Binary src val) idx));
3438   effect(TEMP_DEF dst, TEMP tmp1, TEMP pTmp, KILL cr);
3439   ins_cost(5 * SVE_COST);
3440   format %{ "sve_index $tmp1, 0, 1\t# (B/S/I)\n\t"
3441             "sve_dup $dst, $idx\t# (B/S/I)\n\t"
3442             "sve_cmpeq $pTmp, $tmp1, $dst\n\t"
3443             "sve_orr $dst, $src, $src\n\t"
3444             "sve_cpy $dst, $pTmp, $val\t# insert into vector (B/S/I)" %}
3445   ins_encode %{
3446     BasicType bt = Matcher::vector_element_basic_type(this, $src);
3447     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3448     __ sve_index(as_FloatRegister($tmp1$$reg), size, 0, 1);
3449     __ sve_dup(as_FloatRegister($dst$$reg), size, (int)($idx$$constant));
3450     __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), size, ptrue,
3451                as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
3452     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
3453     __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pTmp$$reg), as_Register($val$$reg));
3454   %}
3455   ins_pipe(pipe_slow);
3456 %}
3457 
3458 instruct insertL(vReg dst, vReg src, iRegL val, immI idx, pRegGov pTmp, rFlagsReg cr)
3459 %{
3460   predicate(UseSVE > 0 &&
3461             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3462   match(Set dst (VectorInsert (Binary src val) idx));
3463   effect(TEMP_DEF dst, TEMP pTmp, KILL cr);
3464   ins_cost(4 * SVE_COST);
3465   format %{ "sve_index $dst, D, -16, 1\n\t"
3466             "sve_cmpeq $pTmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
3467             "sve_orr $dst, $src, $src\n\t"
3468             "sve_cpy $dst, $pTmp, $val\t# insert into vector (L)" %}
3469   ins_encode %{
3470     __ sve_index(as_FloatRegister($dst$$reg), __ D, -16, 1);
3471     __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), __ D, ptrue,
3472                as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
3473     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
3474     __ sve_cpy(as_FloatRegister($dst$$reg), __ D, as_PRegister($pTmp$$reg), as_Register($val$$reg));
3475   %}
3476   ins_pipe(pipe_slow);
3477 %}
3478 
3479 instruct insertD(vReg dst, vReg src, vRegD val, immI idx, pRegGov pTmp, rFlagsReg cr)
3480 %{
3481   predicate(UseSVE > 0 &&
3482             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
3483   match(Set dst (VectorInsert (Binary src val) idx));
3484   effect(TEMP_DEF dst, TEMP pTmp, KILL cr);
3485   ins_cost(4 * SVE_COST);
3486   format %{ "sve_index $dst, D, -16, 1\n\t"
3487             "sve_cmpeq $pTmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
3488             "sve_orr $dst, $src, $src\n\t"
3489             "sve_cpy $dst, $pTmp, $val\t# insert into vector (D)" %}
3490   ins_encode %{
3491     __ sve_index(as_FloatRegister($dst$$reg), __ D, -16, 1);
3492     __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), __ D, ptrue,
3493                as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
3494     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
3495     __ sve_cpy(as_FloatRegister($dst$$reg), __ D, as_PRegister($pTmp$$reg), as_FloatRegister($val$$reg));
3496   %}
3497   ins_pipe(pipe_slow);
3498 %}
3499 
3500 instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pTmp, rFlagsReg cr)
3501 %{
3502   predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
3503             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
3504   match(Set dst (VectorInsert (Binary src val) idx));
3505   effect(TEMP_DEF dst, TEMP tmp1, TEMP pTmp, KILL cr);
3506   ins_cost(5 * SVE_COST);
3507   format %{ "sve_index $tmp1, S, 0, 1\n\t"
3508             "sve_dup $dst, S, $idx\n\t"
3509             "sve_cmpeq $pTmp, $tmp1, $dst\n\t"
3510             "sve_orr $dst, $src, $src\n\t"
3511             "sve_cpy $dst, $pTmp, $val\t# insert into vector (F)" %}
3512   ins_encode %{
3513     __ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
3514     __ sve_dup(as_FloatRegister($dst$$reg), __ S, (int)($idx$$constant));
3515     __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), __ S, ptrue,
3516                as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
3517     __ sve_orr(as_FloatRegister($dst$$reg),
3518                as_FloatRegister($src$$reg),
3519                as_FloatRegister($src$$reg));
3520     __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
3521                as_PRegister($pTmp$$reg), as_FloatRegister($val$$reg));
3522   %}
3523   ins_pipe(pipe_slow);
3524 %}
3525 
3526 // ------------------------------ Vector shuffle -------------------------------
3527 
3528 instruct loadshuffleB(vReg dst, vReg src)
3529 %{
3530   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
3531   match(Set dst (VectorLoadShuffle src));
3532   ins_cost(SVE_COST);
3533   format %{ "sve_orr $dst, $src, $src\t# vector load shuffle (B)" %}
3534   ins_encode %{
3535     if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
3536       __ sve_orr(as_FloatRegister($dst$$reg),
3537                  as_FloatRegister($src$$reg),
3538                  as_FloatRegister($src$$reg));





3539     }
3540   %}
3541   ins_pipe(pipe_slow);
3542 %}
3543 
3544 instruct loadshuffleS(vReg dst, vReg src)
3545 %{
3546   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
3547   match(Set dst (VectorLoadShuffle src));
3548   ins_cost(SVE_COST);
3549   format %{ "sve_uunpklo $dst, $src\t# vector load shuffle (B to H)" %}
3550   ins_encode %{
3551     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
3552   %}
3553   ins_pipe(pipe_slow);
3554 %}
3555 
3556 instruct loadshuffleI(vReg dst, vReg src)
3557 %{
3558   predicate(UseSVE > 0 &&
3559            (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
3560             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
3561   match(Set dst (VectorLoadShuffle src));
3562   ins_cost(2 * SVE_COST);
3563   format %{ "sve_uunpklo $dst, H, $src\n\t"
3564             "sve_uunpklo $dst, S, $dst\t# vector load shuffle (B to S)" %}
3565   ins_encode %{
3566     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
3567     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($dst$$reg));
3568   %}
3569   ins_pipe(pipe_slow);
3570 %}
3571 
3572 instruct loadshuffleL(vReg dst, vReg src)
3573 %{
3574   predicate(UseSVE > 0 &&
3575            (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
3576             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
3577   match(Set dst (VectorLoadShuffle src));
3578   ins_cost(3 * SVE_COST);
3579   format %{ "sve_uunpklo $dst, H, $src\n\t"
3580             "sve_uunpklo $dst, S, $dst\n\t"
3581             "sve_uunpklo $dst, D, $dst\t# vector load shuffle (B to D)" %}
3582   ins_encode %{
3583     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
3584     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($dst$$reg));
3585     __ sve_uunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($dst$$reg));
3586   %}
3587   ins_pipe(pipe_slow);
3588 %}
3589 
3590 // ------------------------------ Vector rearrange -------------------------------
3591 
3592 instruct rearrange(vReg dst, vReg src, vReg shuffle)
3593 %{
3594   predicate(UseSVE > 0);
3595   match(Set dst (VectorRearrange src shuffle));
3596   ins_cost(SVE_COST);
3597   format %{ "sve_tbl $dst, $src, $shuffle\t# vector rearrange" %}
3598   ins_encode %{
3599     BasicType bt = Matcher::vector_element_basic_type(this, $src);
3600     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3601     __ sve_tbl(as_FloatRegister($dst$$reg), size,
3602                as_FloatRegister($src$$reg), as_FloatRegister($shuffle$$reg));
3603   %}
3604   ins_pipe(pipe_slow);
3605 %}
3606 
3607 // ------------------------------ Vector Load Gather ---------------------------------
3608 
3609 instruct gatherI(vReg dst, indirect mem, vReg idx) %{
3610   predicate(UseSVE > 0 &&
3611             n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
3612             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
3613              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
3614   match(Set dst (LoadVectorGather mem idx));
3615   ins_cost(SVE_COST);
3616   format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (I/F)" %}
3617   ins_encode %{
3618     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), ptrue,
3619                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
3620   %}
3621   ins_pipe(pipe_slow);
3622 %}
3623 
3624 instruct gatherL(vReg dst, indirect mem, vReg idx) %{
3625   predicate(UseSVE > 0 &&
3626             n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
3627             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
3628              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
3629   match(Set dst (LoadVectorGather mem idx));
3630   ins_cost(2 * SVE_COST);
3631   format %{ "sve_uunpklo $idx, $idx\n\t"
3632             "load_vector_gather $dst, $mem, $idx\t# vector load gather (L/D)" %}
3633   ins_encode %{
3634     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
3635     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), ptrue, as_Register($mem$$base), as_FloatRegister($idx$$reg));

3636   %}
3637   ins_pipe(pipe_slow);
3638 %}
3639 
3640 // ------------------------------ Vector Load Gather Partial-------------------------------
3641 
3642 instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov pTmp, rFlagsReg cr) %{
3643   predicate(UseSVE > 0 &&
3644             n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
3645             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
3646              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
3647   match(Set dst (LoadVectorGather mem idx));
3648   effect(TEMP pTmp, KILL cr);
3649   ins_cost(2 * SVE_COST + INSN_COST);
3650   format %{ "sve_whilelo_zr_imm $pTmp, vector_length\n\t"
3651             "load_vector_gather $dst, $pTmp, $mem, $idx\t# vector load gather partial (I/F)" %}
3652   ins_encode %{
3653     __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), __ S,
3654                           Matcher::vector_length(this));
3655     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($pTmp$$reg),
3656                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
3657   %}
3658   ins_pipe(pipe_slow);
3659 %}
3660 
3661 instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov pTmp, rFlagsReg cr) %{
3662   predicate(UseSVE > 0 &&
3663             n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
3664             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
3665              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
3666   match(Set dst (LoadVectorGather mem idx));
3667   effect(TEMP pTmp, KILL cr);
3668   ins_cost(3 * SVE_COST + INSN_COST);
3669   format %{ "sve_whilelo_zr_imm $pTmp, vector_length\n\t"
3670             "sve_uunpklo $idx, $idx\n\t"
3671             "load_vector_gather $dst, $pTmp, $mem, $idx\t# vector load gather partial (L/D)" %}




















































3672   ins_encode %{
3673     __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), __ D,
3674                           Matcher::vector_length(this));





















3675     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
3676     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($pTmp$$reg),
3677                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
3678   %}
3679   ins_pipe(pipe_slow);
3680 %}
3681 
3682 // ------------------------------ Vector Store Scatter -------------------------------
3683 
3684 instruct scatterI(indirect mem, vReg src, vReg idx) %{
3685   predicate(UseSVE > 0 &&
3686             n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
3687             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
3688              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
3689   match(Set mem (StoreVectorScatter mem (Binary src idx)));
3690   ins_cost(SVE_COST);
3691   format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (I/F)" %}
3692   ins_encode %{
3693     __ sve_st1w_scatter(as_FloatRegister($src$$reg), ptrue,
3694                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
3695   %}
3696   ins_pipe(pipe_slow);
3697 %}
3698 
3699 instruct scatterL(indirect mem, vReg src, vReg idx) %{
3700   predicate(UseSVE > 0 &&
3701             n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
3702             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
3703              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
3704   match(Set mem (StoreVectorScatter mem (Binary src idx)));
3705   ins_cost(2 * SVE_COST);
3706   format %{ "sve_uunpklo $idx, $idx\n\t"
3707             "store_vector_scatter $mem, $idx, $src\t# vector store scatter (L/D)" %}
3708   ins_encode %{
3709     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D,
3710                    as_FloatRegister($idx$$reg));
3711     __ sve_st1d_scatter(as_FloatRegister($src$$reg), ptrue,
3712                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
3713   %}
3714   ins_pipe(pipe_slow);
3715 %}
3716 
3717 // ------------------------------ Vector Store Scatter Partial-------------------------------
3718 
3719 instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov pTmp, rFlagsReg cr) %{
3720   predicate(UseSVE > 0 &&
3721             n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
3722             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
3723              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
3724   match(Set mem (StoreVectorScatter mem (Binary src idx)));
3725   effect(TEMP pTmp, KILL cr);
3726   ins_cost(2 * SVE_COST + INSN_COST);
3727   format %{ "sve_whilelo_zr_imm $pTmp, vector_length\n\t"
3728             "store_vector_scatter $mem, $pTmp, $idx, $src\t# vector store scatter partial (I/F)" %}
3729   ins_encode %{
3730     __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), __ S,
3731                           Matcher::vector_length(this, $src));
3732     __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($pTmp$$reg),
3733                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
3734   %}
3735   ins_pipe(pipe_slow);
3736 %}
3737 
3738 instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov pTmp, rFlagsReg cr) %{
3739   predicate(UseSVE > 0 &&
3740             n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
3741             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
3742              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
3743   match(Set mem (StoreVectorScatter mem (Binary src idx)));
3744   effect(TEMP pTmp, KILL cr);
3745   ins_cost(3 * SVE_COST + INSN_COST);
3746   format %{ "sve_whilelo_zr_imm $pTmp, vector_length\n\t"
3747             "sve_uunpklo $idx, $idx\n\t"
3748             "store_vector_scatter $mem, $pTmp, $idx, $src\t# vector store scatter partial (L/D)" %}
3749   ins_encode %{
3750     __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), __ D,
3751                           Matcher::vector_length(this, $src));
3752     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
3753     __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($pTmp$$reg),























































3754                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
3755   %}
3756   ins_pipe(pipe_slow);
3757 %}
3758 




















3759 
3760 // ------------------------------ Vector Load Const -------------------------------
3761 
3762 instruct loadconB(vReg dst, immI0 src) %{
3763   predicate(UseSVE > 0 &&
3764             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
3765   match(Set dst (VectorLoadConst src));
3766   ins_cost(SVE_COST);
3767   format %{ "sve_index $dst, 0, 1\t# generate iota indices" %}
3768   ins_encode %{
3769     __ sve_index(as_FloatRegister($dst$$reg), __ B, 0, 1);
3770   %}
3771   ins_pipe(pipe_slow);
3772 %}
3773 
3774 // Intrisics for String.indexOf(char)
3775 
3776 
3777 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
3778                                   iRegI_R0 result, vReg ztmp1, vReg ztmp2,

3794 
3795 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
3796                                   iRegI_R0 result, vReg ztmp1, vReg ztmp2,
3797                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
3798 %{
3799   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
3800   predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
3801   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
3802 
3803   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
3804 
3805   ins_encode %{
3806     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
3807                                as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
3808                                as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), false /* isL */);
3809   %}
3810   ins_pipe(pipe_class_memory);
3811 %}
3812 
3813 // ---------------------------- Vector mask reductions ---------------------------
3814 
3815 instruct vmask_truecount(iRegINoSp dst, vReg src, pReg ptmp, rFlagsReg cr) %{
3816   predicate(UseSVE > 0 &&
3817             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3818   match(Set dst (VectorMaskTrueCount src));
3819   effect(TEMP ptmp, KILL cr);
3820   ins_cost(2 * SVE_COST);
3821   format %{ "vmask_truecount $dst, $src\t# vector mask truecount (sve)" %}
3822   ins_encode %{
3823     __ sve_vmask_reduction(this->ideal_Opcode(), $dst$$Register, __ B,
3824                            as_FloatRegister($src$$reg), ptrue, as_PRegister($ptmp$$reg));

3825   %}
3826   ins_pipe(pipe_slow);
3827 %}
3828 
3829 instruct vmask_firsttrue(iRegINoSp dst, vReg src, pReg ptmp, rFlagsReg cr) %{
3830   predicate(UseSVE > 0 &&
3831             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3832   match(Set dst (VectorMaskFirstTrue src));
3833   effect(TEMP ptmp, KILL cr);
3834   ins_cost(3 * SVE_COST);
3835   format %{ "vmask_firsttrue $dst, $src\t# vector mask firsttrue (sve)" %}
3836   ins_encode %{
3837     __ sve_vmask_reduction(this->ideal_Opcode(), $dst$$Register, __ B,
3838                            as_FloatRegister($src$$reg), ptrue, as_PRegister($ptmp$$reg));


3839   %}
3840   ins_pipe(pipe_slow);
3841 %}
3842 
3843 instruct vmask_lasttrue(iRegINoSp dst, vReg src, pReg ptmp, rFlagsReg cr) %{
3844   predicate(UseSVE > 0 &&
3845             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3846   match(Set dst (VectorMaskLastTrue src));
3847   effect(TEMP ptmp, KILL cr);
3848   ins_cost(4 * SVE_COST);
3849   format %{ "vmask_lasttrue $dst, $src\t# vector mask lasttrue (sve)" %}
3850   ins_encode %{
3851     __ sve_vmask_reduction(this->ideal_Opcode(), $dst$$Register, __ B,
3852                            as_FloatRegister($src$$reg), ptrue, as_PRegister($ptmp$$reg));
3853   %}
3854   ins_pipe(pipe_slow);
3855 %}
3856 
3857 instruct vmask_truecount_partial(iRegINoSp dst, vReg src, pRegGov ptmp, rFlagsReg cr) %{
3858   predicate(UseSVE > 0 &&
3859             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3860   match(Set dst (VectorMaskTrueCount src));
3861   effect(TEMP ptmp, KILL cr);
3862   ins_cost(3 * SVE_COST);
3863   format %{ "vmask_truecount $dst, $src\t# vector mask truecount partial (sve)" %}
3864   ins_encode %{
3865     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ B,
3866                           Matcher::vector_length(this, $src));
3867     __ sve_vmask_reduction(this->ideal_Opcode(), $dst$$Register, __ B, as_FloatRegister($src$$reg),
3868                            as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg));
3869   %}
3870   ins_pipe(pipe_slow);
3871 %}
3872 
3873 instruct vmask_firsttrue_partial(iRegINoSp dst, vReg src, pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
3874   predicate(UseSVE > 0 &&
3875             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3876   match(Set dst (VectorMaskFirstTrue src));
3877   effect(TEMP pgtmp, TEMP ptmp, KILL cr);
3878   ins_cost(4 * SVE_COST);
3879   format %{ "vmask_firsttrue $dst, $src\t# vector mask firsttrue partial (sve)" %}
3880   ins_encode %{
3881     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ B,


3882                           Matcher::vector_length(this, $src));
3883     __ sve_vmask_reduction(this->ideal_Opcode(), $dst$$Register, __ B, as_FloatRegister($src$$reg),
3884                            as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg));
3885   %}
3886   ins_pipe(pipe_slow);
3887 %}
3888 
3889 instruct vmask_lasttrue_partial(iRegINoSp dst, vReg src, pRegGov ptmp, rFlagsReg cr) %{
3890   predicate(UseSVE > 0 &&
3891             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3892   match(Set dst (VectorMaskLastTrue src));
3893   effect(TEMP ptmp, KILL cr);
3894   ins_cost(5 * SVE_COST);
3895   format %{ "vmask_lasttrue $dst, $src\t# vector mask lasttrue partial (sve)" %}
3896   ins_encode %{
3897     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ B,
3898                           Matcher::vector_length(this, $src));
3899     __ sve_vmask_reduction(this->ideal_Opcode(), $dst$$Register, __ B, as_FloatRegister($src$$reg),
3900                            as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg));
3901   %}
3902   ins_pipe(pipe_slow);
3903 %}
3904 
3905 // ----------------- Vector mask reductions combined with VectorMaskStore ---------------
3906 
3907 instruct vstoremask_truecount(iRegINoSp dst, vReg src, immI esize, pReg ptmp, rFlagsReg cr) %{
3908   predicate(UseSVE > 0 &&
3909             n->in(1)->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3910   match(Set dst (VectorMaskTrueCount (VectorStoreMask src esize)));
3911   effect(TEMP ptmp, KILL cr);
3912   ins_cost(2 * SVE_COST);
3913   format %{ "vstoremask_truecount $dst, $src\t# vector mask truecount (sve)" %}
3914   ins_encode %{
3915     unsigned size = $esize$$constant;
3916     assert(size == 1 || size == 2 || size == 4 || size == 8, "unsupported element size");
3917     Assembler::SIMD_RegVariant variant = __ elemBytes_to_regVariant(size);
3918     __ sve_vmask_reduction(this->ideal_Opcode(), $dst$$Register, variant, as_FloatRegister($src$$reg),
3919                            ptrue, as_PRegister($ptmp$$reg), Matcher::vector_length(this, $src));
3920   %}
3921   ins_pipe(pipe_slow);
3922 %}
3923 
3924 instruct vstoremask_firsttrue(iRegINoSp dst, vReg src, immI esize, pReg ptmp, rFlagsReg cr) %{
3925   predicate(UseSVE > 0 &&
3926             n->in(1)->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3927   match(Set dst (VectorMaskFirstTrue (VectorStoreMask src esize)));
3928   effect(TEMP ptmp, KILL cr);
3929   ins_cost(3 * SVE_COST);
3930   format %{ "vstoremask_firsttrue $dst, $src\t# vector mask firsttrue (sve)" %}
3931   ins_encode %{
3932     unsigned size = $esize$$constant;
3933     assert(size == 1 || size == 2 || size == 4 || size == 8, "unsupported element size");
3934     Assembler::SIMD_RegVariant variant = __ elemBytes_to_regVariant(size);
3935     __ sve_vmask_reduction(this->ideal_Opcode(), $dst$$Register, variant, as_FloatRegister($src$$reg),
3936                            ptrue, as_PRegister($ptmp$$reg), Matcher::vector_length(this, $src));
3937   %}
3938   ins_pipe(pipe_slow);
3939 %}
3940 
3941 instruct vstoremask_lasttrue(iRegINoSp dst, vReg src, immI esize, pReg ptmp, rFlagsReg cr) %{
3942   predicate(UseSVE > 0 &&
3943             n->in(1)->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3944   match(Set dst (VectorMaskLastTrue (VectorStoreMask src esize)));
3945   effect(TEMP ptmp, KILL cr);
3946   ins_cost(4 * SVE_COST);
3947   format %{ "vstoremask_lasttrue $dst, $src\t# vector mask lasttrue (sve)" %}
3948   ins_encode %{
3949     unsigned size = $esize$$constant;
3950     assert(size == 1 || size == 2 || size == 4 || size == 8, "unsupported element size");
3951     Assembler::SIMD_RegVariant variant = __ elemBytes_to_regVariant(size);
3952     __ sve_vmask_reduction(this->ideal_Opcode(), $dst$$Register, variant, as_FloatRegister($src$$reg),
3953                            ptrue, as_PRegister($ptmp$$reg), Matcher::vector_length(this, $src));
3954   %}
3955   ins_pipe(pipe_slow);
3956 %}
3957 
3958 instruct vstoremask_truecount_partial(iRegINoSp dst, vReg src, immI esize, pRegGov ptmp, rFlagsReg cr) %{
3959   predicate(UseSVE > 0 &&
3960             n->in(1)->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3961   match(Set dst (VectorMaskTrueCount (VectorStoreMask src esize)));
3962   effect(TEMP ptmp, KILL cr);
3963   ins_cost(3 * SVE_COST);
3964   format %{ "vstoremask_truecount $dst, $src\t# vector mask truecount partial (sve)" %}
3965   ins_encode %{
3966     unsigned size = $esize$$constant;
3967     assert(size == 1 || size == 2 || size == 4 || size == 8, "unsupported element size");
3968     Assembler::SIMD_RegVariant variant = __ elemBytes_to_regVariant(size);
3969     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
3970                           Matcher::vector_length(this, $src));
3971     __ sve_vmask_reduction(this->ideal_Opcode(), $dst$$Register, variant, as_FloatRegister($src$$reg),
3972                            as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg), MaxVectorSize / size);
3973   %}
3974   ins_pipe(pipe_slow);
3975 %}
3976 
3977 instruct vstoremask_firsttrue_partial(iRegINoSp dst, vReg src, immI esize, pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
3978   predicate(UseSVE > 0 &&
3979             n->in(1)->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3980   match(Set dst (VectorMaskFirstTrue (VectorStoreMask src esize)));
3981   effect(TEMP pgtmp, TEMP ptmp, KILL cr);
3982   ins_cost(4 * SVE_COST);
3983   format %{ "vstoremask_firsttrue $dst, $src\t# vector mask firsttrue partial (sve)" %}
3984   ins_encode %{
3985     unsigned size = $esize$$constant;
3986     assert(size == 1 || size == 2 || size == 4 || size == 8, "unsupported element size");
3987     Assembler::SIMD_RegVariant variant = __ elemBytes_to_regVariant(size);
3988     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), variant,
3989                           Matcher::vector_length(this, $src));
3990     __ sve_vmask_reduction(this->ideal_Opcode(), $dst$$Register, variant, as_FloatRegister($src$$reg),
3991                            as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), MaxVectorSize / size);
3992   %}
3993   ins_pipe(pipe_slow);
3994 %}
3995 
3996 instruct vstoremask_lasttrue_partial(iRegINoSp dst, vReg src, immI esize, pRegGov ptmp, rFlagsReg cr) %{
3997   predicate(UseSVE > 0 &&
3998             n->in(1)->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3999   match(Set dst (VectorMaskLastTrue (VectorStoreMask src esize)));
4000   effect(TEMP ptmp, KILL cr);
4001   ins_cost(5 * SVE_COST);
4002   format %{ "vstoremask_lasttrue $dst, $src\t# vector mask lasttrue partial (sve)" %}
4003   ins_encode %{
4004     unsigned size = $esize$$constant;
4005     assert(size == 1 || size == 2 || size == 4 || size == 8, "unsupported element size");
4006     Assembler::SIMD_RegVariant variant = __ elemBytes_to_regVariant(size);
4007     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
4008                           Matcher::vector_length(this, $src));
4009     __ sve_vmask_reduction(this->ideal_Opcode(), $dst$$Register, variant, as_FloatRegister($src$$reg),
4010                            as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg), MaxVectorSize / size);
4011   %}
4012   ins_pipe(pipe_slow);
4013 %}

  71 operand vmemA_indOffL4(iRegP reg, vmemA_immLOffset4 off)
  72 %{
  73   constraint(ALLOC_IN_RC(ptr_reg));
  74   match(AddP reg off);
  75   op_cost(0);
  76   format %{ "[$reg, $off]" %}
  77   interface(MEMORY_INTER) %{
  78     base($reg);
  79     index(0xffffffff);
  80     scale(0x0);
  81     disp($off);
  82   %}
  83 %}
  84 
  85 // The indOff of vmemA is valid only when the vector element (load to/store from)
  86 // size equals to memory element (load from/store to) size.
  87 opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);
  88 
  89 source_hpp %{
  90   bool op_sve_supported(int opcode, int vlen, BasicType bt);
  91   bool masked_op_sve_supported(int opcode, int vlen, BasicType bt);
  92 %}
  93 
  94 source %{
  95 
  96   typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
  97                                                              PRegister Pg, const Address &adr);
  98 
  99   // Predicated load/store, with optional ptrue to all elements of given predicate register.
 100   static void loadStoreA_predicated(C2_MacroAssembler masm, bool is_store, FloatRegister reg,
 101                                     PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
 102                                     int opcode, Register base, int index, int size, int disp) {
 103     sve_mem_insn_predicate insn;
 104     int mesize = type2aelembytes(mem_elem_bt);
 105     if (index == -1) {
 106       assert(size == 0, "unsupported address mode: scale size = %d", size);
 107       switch(mesize) {
 108       case 1:
 109         insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
 110         break;
 111       case 2:

 128       ShouldNotReachHere();
 129     }
 130   }
 131 
 132   bool op_sve_supported(int opcode, int vlen, BasicType bt) {
 133     int length_in_bytes = vlen * type2aelembytes(bt);
 134     switch (opcode) {
 135       case Op_MulAddVS2VI:
 136       // No multiply reduction instructions
 137       case Op_MulReductionVD:
 138       case Op_MulReductionVF:
 139       case Op_MulReductionVI:
 140       case Op_MulReductionVL:
 141       // Others
 142       case Op_ExtractC:
 143       case Op_ExtractUB:
 144         return false;
 145       // Vector API specific
 146       case Op_VectorLoadShuffle:
 147       case Op_VectorRearrange:
 148         return vlen >= 4 && length_in_bytes <= MaxVectorSize;




 149       case Op_LoadVector:
 150       case Op_StoreVector:
 151         return Matcher::vector_size_supported(bt, vlen);
 152       default:
 153         break;
 154     }
 155     // By default, we only support vector operations with no less than 8 bytes and 2 elements.
 156     return 8 <= length_in_bytes && length_in_bytes <= MaxVectorSize && vlen >= 2;
 157   }
 158 
 159   bool masked_op_sve_supported(int opcode, int vlen, BasicType bt) {
 160     if (opcode == Op_VectorRearrange) {
 161       return false;
 162     }
 163     return op_sve_supported(opcode, vlen, bt);
 164   }
 165 
 166 %}
 167 
 168 definitions %{
 169   int_def SVE_COST             (200, 200);
 170 %}
 171 
 172 
 173 // All SVE instructions
 174 
 175 // vector load/store
 176 
 177 // Unpredicated vector load/store
 178 instruct loadV(vReg dst, vmemA mem) %{
 179   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() >= 16 &&
 180             n->as_LoadVector()->memory_size() == MaxVectorSize);
 181   match(Set dst (LoadVector mem));
 182   ins_cost(4 * SVE_COST);
 183   format %{ "sve_ldr $dst, $mem\t# vector (sve)" %}
 184   ins_encode %{
 185     FloatRegister dst_reg = as_FloatRegister($dst$$reg);

 282   format %{ "ldrq   $dst,$mem\t# vector (128 bits)" %}
 283   ins_encode( aarch64_enc_ldrvQ(dst, mem) );
 284   ins_pipe(vload_reg_mem128);
 285 %}
 286 
 287 // Store Vector (128 bits)
 288 instruct storeV16_vreg(vReg src, vmem16 mem)
 289 %{
 290   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 16);
 291   match(Set mem (StoreVector mem src));
 292   ins_cost(4 * INSN_COST);
 293   format %{ "strq   $mem,$src\t# vector (128 bits)" %}
 294   ins_encode( aarch64_enc_strvQ(src, mem) );
 295   ins_pipe(vstore_reg_mem128);
 296 %}
 297 
 298 // Predicated vector load/store, based on the vector length of the node.
 299 // Only load/store values in the range of the memory_size. This is needed
 300 // when the memory_size is lower than the hardware supported max vector size.
 301 // And this might happen for Vector API mask vector load/store.
 302 instruct loadV_partial(vReg dst, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
 303   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() > 16 &&
 304             n->as_LoadVector()->memory_size() < MaxVectorSize);
 305   match(Set dst (LoadVector mem));
 306   effect(TEMP pgtmp, KILL cr);
 307   ins_cost(6 * SVE_COST);
 308   format %{ "sve_whilelo_zr_imm $pgtmp, vector_length\n\t"
 309             "sve_ldr $dst, $pgtmp, $mem\t# load vector partial" %}
 310   ins_encode %{
 311     BasicType bt = Matcher::vector_element_basic_type(this);
 312     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 313                           Matcher::vector_length(this));
 314     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 315     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg,
 316                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 317                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 318   %}
 319   ins_pipe(pipe_slow);
 320 %}
 321 
 322 instruct storeV_partial(vReg src, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
 323   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() > 16 &&
 324             n->as_StoreVector()->memory_size() < MaxVectorSize);
 325   match(Set mem (StoreVector mem src));
 326   effect(TEMP pgtmp, KILL cr);
 327   ins_cost(5 * SVE_COST);
 328   format %{ "sve_whilelo_zr_imm $pgtmp, vector_length\n\t"
 329             "sve_str $src, $pgtmp, $mem\t# store vector partial" %}
 330   ins_encode %{
 331     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 332     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 333                           Matcher::vector_length(this, $src));
 334     FloatRegister src_reg = as_FloatRegister($src$$reg);
 335     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg,
 336                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 337                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 338   %}
 339   ins_pipe(pipe_slow);
 340 %}
 341 
 342 // vector load/store - predicated
 343 
 344 instruct loadV_masked(vReg dst, vmemA mem, pRegGov pg) %{
 345   predicate(UseSVE > 0 &&
 346             n->as_LoadVector()->memory_size() == MaxVectorSize);
 347   match(Set dst (LoadVectorMasked mem pg));
 348   ins_cost(4 * SVE_COST);
 349   format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated (sve)" %}
 350   ins_encode %{
 351     BasicType bt = Matcher::vector_element_basic_type(this);
 352     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
 353                           as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
 354                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 355   %}
 356   ins_pipe(pipe_slow);
 357 %}
 358 
 359 instruct loadV_masked_partial(vReg dst, vmemA mem, pRegGov pg, pRegGov pgtmp, rFlagsReg cr) %{
 360   predicate(UseSVE > 0 &&
 361             n->as_LoadVector()->memory_size() < MaxVectorSize);
 362   match(Set dst (LoadVectorMasked mem pg));
 363   effect(TEMP pgtmp, KILL cr);
 364   ins_cost(6 * SVE_COST);
 365   format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated partial (sve)" %}
 366   ins_encode %{
 367     BasicType bt = Matcher::vector_element_basic_type(this);
 368     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 369                           Matcher::vector_length(this));
 370     __ sve_and(as_PRegister($pgtmp$$reg), as_PRegister($pgtmp$$reg),
 371                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
 372     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
 373                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 374                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 375   %}
 376   ins_pipe(pipe_slow);
 377 %}
 378 
 379 instruct storeV_masked(vReg src, vmemA mem, pRegGov pg) %{
 380   predicate(UseSVE > 0 &&
 381             n->as_StoreVector()->memory_size() == MaxVectorSize);
 382   match(Set mem (StoreVectorMasked mem (Binary src pg)));
 383   ins_cost(4 * SVE_COST);
 384   format %{ "sve_str $mem, $pg, $src\t# store vector predicated (sve)" %}
 385   ins_encode %{
 386     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 387     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
 388                           as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
 389                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 390   %}
 391   ins_pipe(pipe_slow);
 392 %}
 393 
 394 instruct storeV_masked_partial(vReg src, vmemA mem, pRegGov pg, pRegGov pgtmp, rFlagsReg cr) %{
 395   predicate(UseSVE > 0 &&
 396             n->as_StoreVector()->memory_size() < MaxVectorSize);
 397   match(Set mem (StoreVectorMasked mem (Binary src pg)));
 398   effect(TEMP pgtmp, KILL cr);
 399   ins_cost(6 * SVE_COST);
 400   format %{ "sve_str $mem, $pg, $src\t# store vector predicated partial (sve)" %}
 401   ins_encode %{
 402     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 403     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 404                           Matcher::vector_length(this, $src));
 405     __ sve_and(as_PRegister($pgtmp$$reg), as_PRegister($pgtmp$$reg),
 406                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
 407     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
 408                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 409                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 410   %}
 411   ins_pipe(pipe_slow);
 412 %}
 413 
 414 // maskAll
 415 
 416 instruct vmaskAll_immI(pRegGov dst, immI src) %{
 417   predicate(UseSVE > 0);
 418   match(Set dst (MaskAll src));
 419   ins_cost(SVE_COST);
 420   format %{ "sve_ptrue/sve_pfalse $dst\t# mask all (sve) (B/H/S)" %}
 421   ins_encode %{
 422     int con = (int)$src$$constant;
 423     if (con == 0) {
 424       __ sve_pfalse(as_PRegister($dst$$reg));
 425     } else {
 426       assert(con == -1, "invalid constant value for mask");
 427       BasicType bt = Matcher::vector_element_basic_type(this);
 428       __ sve_ptrue(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt));
 429     }
 430   %}
 431   ins_pipe(pipe_slow);
 432 %}
 433 
 434 instruct vmaskAllI(pRegGov dst, iRegIorL2I src, vReg tmp, rFlagsReg cr) %{
 435   predicate(UseSVE > 0);
 436   match(Set dst (MaskAll src));
 437   effect(TEMP tmp, KILL cr);
 438   ins_cost(2 * SVE_COST);
 439   format %{ "sve_dup $tmp, $src\n\t"
 440             "sve_cmpne $dst, $tmp, 0\t# mask all (sve) (B/H/S)" %}
 441   ins_encode %{
 442     BasicType bt = Matcher::vector_element_basic_type(this);
 443     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
 444     __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
 445     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
 446   %}
 447   ins_pipe(pipe_slow);
 448 %}
 449 
 450 instruct vmaskAll_immL(pRegGov dst, immL src) %{
 451   predicate(UseSVE > 0);
 452   match(Set dst (MaskAll src));
 453   ins_cost(SVE_COST);
 454   format %{ "sve_ptrue/sve_pfalse $dst\t# mask all (sve) (D)" %}
 455   ins_encode %{
 456     long con = (long)$src$$constant;
 457     if (con == 0) {
 458       __ sve_pfalse(as_PRegister($dst$$reg));
 459     } else {
 460       assert(con == -1, "invalid constant value for mask");
 461       BasicType bt = Matcher::vector_element_basic_type(this);
 462       __ sve_ptrue(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt));
 463     }
 464   %}
 465   ins_pipe(pipe_slow);
 466 %}
 467 
 468 instruct vmaskAllL(pRegGov dst, iRegL src, vReg tmp, rFlagsReg cr) %{
 469   predicate(UseSVE > 0);
 470   match(Set dst (MaskAll src));
 471   effect(TEMP tmp, KILL cr);
 472   ins_cost(2 * SVE_COST);
 473   format %{ "sve_dup $tmp, $src\n\t"
 474             "sve_cmpne $dst, $tmp, 0\t# mask all (sve) (D)" %}
 475   ins_encode %{
 476     BasicType bt = Matcher::vector_element_basic_type(this);
 477     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
 478     __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
 479     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
 480   %}
 481   ins_pipe(pipe_slow);
 482 %}
 483 
 484 // mask logical and/or/xor
 485 
 486 instruct vmask_and(pRegGov pd, pRegGov pn, pRegGov pm) %{
 487   predicate(UseSVE > 0);
 488   match(Set pd (AndVMask pn pm));
 489   ins_cost(SVE_COST);
 490   format %{ "sve_and $pd, $pn, $pm\t# predicate (sve)" %}
 491   ins_encode %{
 492     __ sve_and(as_PRegister($pd$$reg), ptrue,
 493                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 494   %}
 495   ins_pipe(pipe_slow);
 496 %}
 497 
 498 instruct vmask_or(pRegGov pd, pRegGov pn, pRegGov pm) %{
 499   predicate(UseSVE > 0);
 500   match(Set pd (OrVMask pn pm));
 501   ins_cost(SVE_COST);
 502   format %{ "sve_orr $pd, $pn, $pm\t# predicate (sve)" %}
 503   ins_encode %{
 504     __ sve_orr(as_PRegister($pd$$reg), ptrue,
 505                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 506   %}
 507   ins_pipe(pipe_slow);
 508 %}
 509 
 510 instruct vmask_xor(pRegGov pd, pRegGov pn, pRegGov pm) %{
 511   predicate(UseSVE > 0);
 512   match(Set pd (XorVMask pn pm));
 513   ins_cost(SVE_COST);
 514   format %{ "sve_eor $pd, $pn, $pm\t# predicate (sve)" %}
 515   ins_encode %{
 516     __ sve_eor(as_PRegister($pd$$reg), ptrue,
 517                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 518   %}
 519   ins_pipe(pipe_slow);
 520 %}
 521 
 522 // mask logical and_not
 523 
 524 instruct vmask_and_notI(pRegGov pd, pRegGov pn, pRegGov pm, immI_M1 m1) %{
 525   predicate(UseSVE > 0);
 526   match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
 527   ins_cost(SVE_COST);
 528   format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) (B/H/S)" %}
 529   ins_encode %{
 530     __ sve_bic(as_PRegister($pd$$reg), ptrue,
 531                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 532   %}
 533   ins_pipe(pipe_slow);
 534 %}
 535 
 536 instruct vmask_and_notL(pRegGov pd, pRegGov pn, pRegGov pm, immL_M1 m1) %{
 537   predicate(UseSVE > 0);
 538   match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
 539   ins_cost(SVE_COST);
 540   format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) (D)" %}
 541   ins_encode %{
 542     __ sve_bic(as_PRegister($pd$$reg), ptrue,
 543                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 544   %}
 545   ins_pipe(pipe_slow);
 546 %}
 547 
 548 // vector reinterpret
 549 
 550 instruct reinterpret(vReg dst) %{
 551   predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() ==
 552                           n->in(1)->bottom_type()->is_vect()->length_in_bytes());  // src == dst
 553   match(Set dst (VectorReinterpret dst));
 554   ins_cost(0);
 555   format %{ "# reinterpret $dst\t# do nothing" %}
 556   ins_encode %{
 557     // empty
 558   %}
 559   ins_pipe(pipe_class_empty);
 560 %}
 561 
 562 instruct reinterpretResize(vReg dst, vReg src, pRegGov pgtmp, rFlagsReg cr) %{
 563   predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() !=
 564                           n->in(1)->bottom_type()->is_vect()->length_in_bytes());  // src != dst
 565   match(Set dst (VectorReinterpret src));
 566   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
 567   ins_cost(3 * SVE_COST);
 568   format %{ "reinterpretResize $dst, $src\t# vector (sve)" %}
 569   ins_encode %{
 570     uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
 571     uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
 572     uint length_in_bytes_resize = length_in_bytes_src < length_in_bytes_dst ?
 573                                   length_in_bytes_src : length_in_bytes_dst;
 574     assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
 575            "invalid vector length");
 576     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ B, length_in_bytes_resize);
 577     __ sve_dup(as_FloatRegister($dst$$reg), __ B, 0);
 578     __ sve_sel(as_FloatRegister($dst$$reg), __ B, as_PRegister($pgtmp$$reg),
 579                as_FloatRegister($src$$reg), as_FloatRegister($dst$$reg));
 580   %}
 581   ins_pipe(pipe_slow);
 582 %}
 583 
 584 // vector mask reinterpret
 585 
 586 instruct vmask_reinterpret_same_esize(pRegGov dst_src) %{
 587   predicate(UseSVE > 0 &&
 588             n->as_Vector()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
 589             n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
 590   match(Set dst_src (VectorReinterpret dst_src));
 591   ins_cost(0);
 592   format %{ "# vmask_reinterpret $dst_src\t# do nothing" %}
 593   ins_encode %{
 594     // empty
 595   %}
 596   ins_pipe(pipe_class_empty);
 597 %}
 598 
 599 instruct vmask_reinterpret_diff_esize(pRegGov dst, pRegGov src, vReg tmp, rFlagsReg cr) %{
 600   predicate(UseSVE > 0 &&
 601             n->as_Vector()->length() != n->in(1)->bottom_type()->is_vect()->length() &&
 602             n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
 603   match(Set dst (VectorReinterpret src));
 604   effect(TEMP tmp, KILL cr);
 605   ins_cost(2 * SVE_COST);
 606   format %{ "# vmask_reinterpret $dst, $src\t# vector (sve)" %}
 607   ins_encode %{
 608     BasicType from_bt = Matcher::vector_element_basic_type(this, $src);
 609     Assembler::SIMD_RegVariant from_size = __ elemType_to_regVariant(from_bt);
 610     BasicType to_bt = Matcher::vector_element_basic_type(this);
 611     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
 612     __ sve_cpy(as_FloatRegister($tmp$$reg), from_size, as_PRegister($src$$reg), -1, false);
 613     __ sve_cmp(Assembler::EQ, as_PRegister($dst$$reg), to_size, ptrue, as_FloatRegister($tmp$$reg), -1);
 614   %}
 615   ins_pipe(pipe_slow);
 616 %}
 617 
 618 // vector abs
 619 
 620 instruct vabsB(vReg dst, vReg src) %{
 621   predicate(UseSVE > 0 &&
 622             !n->as_Vector()->is_predicated_vector());
 623   match(Set dst (AbsVB src));
 624   ins_cost(SVE_COST);
 625   format %{ "sve_abs $dst, $src\t# vector (sve) (B)" %}
 626   ins_encode %{
 627     __ sve_abs(as_FloatRegister($dst$$reg), __ B,
 628          ptrue, as_FloatRegister($src$$reg));
 629   %}
 630   ins_pipe(pipe_slow);
 631 %}
 632 
 633 instruct vabsS(vReg dst, vReg src) %{
 634   predicate(UseSVE > 0 &&
 635             !n->as_Vector()->is_predicated_vector());
 636   match(Set dst (AbsVS src));
 637   ins_cost(SVE_COST);
 638   format %{ "sve_abs $dst, $src\t# vector (sve) (H)" %}
 639   ins_encode %{
 640     __ sve_abs(as_FloatRegister($dst$$reg), __ H,
 641          ptrue, as_FloatRegister($src$$reg));
 642   %}
 643   ins_pipe(pipe_slow);
 644 %}
 645 
 646 instruct vabsI(vReg dst, vReg src) %{
 647   predicate(UseSVE > 0 &&
 648             !n->as_Vector()->is_predicated_vector());
 649   match(Set dst (AbsVI src));
 650   ins_cost(SVE_COST);
 651   format %{ "sve_abs $dst, $src\t# vector (sve) (S)" %}
 652   ins_encode %{
 653     __ sve_abs(as_FloatRegister($dst$$reg), __ S,
 654          ptrue, as_FloatRegister($src$$reg));
 655   %}
 656   ins_pipe(pipe_slow);
 657 %}
 658 
 659 instruct vabsL(vReg dst, vReg src) %{
 660   predicate(UseSVE > 0 &&
 661             !n->as_Vector()->is_predicated_vector());
 662   match(Set dst (AbsVL src));
 663   ins_cost(SVE_COST);
 664   format %{ "sve_abs $dst, $src\t# vector (sve) (D)" %}
 665   ins_encode %{
 666     __ sve_abs(as_FloatRegister($dst$$reg), __ D,
 667          ptrue, as_FloatRegister($src$$reg));
 668   %}
 669   ins_pipe(pipe_slow);
 670 %}
 671 
 672 instruct vabsF(vReg dst, vReg src) %{
 673   predicate(UseSVE > 0 &&
 674             !n->as_Vector()->is_predicated_vector());
 675   match(Set dst (AbsVF src));
 676   ins_cost(SVE_COST);
 677   format %{ "sve_fabs $dst, $src\t# vector (sve) (S)" %}
 678   ins_encode %{
 679     __ sve_fabs(as_FloatRegister($dst$$reg), __ S,
 680          ptrue, as_FloatRegister($src$$reg));
 681   %}
 682   ins_pipe(pipe_slow);
 683 %}
 684 
 685 instruct vabsD(vReg dst, vReg src) %{
 686   predicate(UseSVE > 0 &&
 687             !n->as_Vector()->is_predicated_vector());
 688   match(Set dst (AbsVD src));
 689   ins_cost(SVE_COST);
 690   format %{ "sve_fabs $dst, $src\t# vector (sve) (D)" %}
 691   ins_encode %{
 692     __ sve_fabs(as_FloatRegister($dst$$reg), __ D,
 693          ptrue, as_FloatRegister($src$$reg));
 694   %}
 695   ins_pipe(pipe_slow);
 696 %}
 697 
 698 // vector abs - predicated
 699 
 700 instruct vabsB_masked(vReg dst_src, pRegGov pg) %{
 701   predicate(UseSVE > 0);
 702   match(Set dst_src (AbsVB dst_src pg));
 703   ins_cost(SVE_COST);
 704   format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (B)" %}
 705   ins_encode %{
 706     __ sve_abs(as_FloatRegister($dst_src$$reg), __ B,
 707             as_PRegister($pg$$reg),
 708             as_FloatRegister($dst_src$$reg));
 709   %}
 710   ins_pipe(pipe_slow);
 711 %}
 712 
 713 instruct vabsS_masked(vReg dst_src, pRegGov pg) %{
 714   predicate(UseSVE > 0);
 715   match(Set dst_src (AbsVS dst_src pg));
 716   ins_cost(SVE_COST);
 717   format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (H)" %}
 718   ins_encode %{
 719     __ sve_abs(as_FloatRegister($dst_src$$reg), __ H,
 720             as_PRegister($pg$$reg),
 721             as_FloatRegister($dst_src$$reg));
 722   %}
 723   ins_pipe(pipe_slow);
 724 %}
 725 
 726 instruct vabsI_masked(vReg dst_src, pRegGov pg) %{
 727   predicate(UseSVE > 0);
 728   match(Set dst_src (AbsVI dst_src pg));
 729   ins_cost(SVE_COST);
 730   format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
 731   ins_encode %{
 732     __ sve_abs(as_FloatRegister($dst_src$$reg), __ S,
 733             as_PRegister($pg$$reg),
 734             as_FloatRegister($dst_src$$reg));
 735   %}
 736   ins_pipe(pipe_slow);
 737 %}
 738 
 739 instruct vabsL_masked(vReg dst_src, pRegGov pg) %{
 740   predicate(UseSVE > 0);
 741   match(Set dst_src (AbsVL dst_src pg));
 742   ins_cost(SVE_COST);
 743   format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
 744   ins_encode %{
 745     __ sve_abs(as_FloatRegister($dst_src$$reg), __ D,
 746             as_PRegister($pg$$reg),
 747             as_FloatRegister($dst_src$$reg));
 748   %}
 749   ins_pipe(pipe_slow);
 750 %}
 751 
 752 instruct vabsF_masked(vReg dst_src, pRegGov pg) %{
 753   predicate(UseSVE > 0);
 754   match(Set dst_src (AbsVF dst_src pg));
 755   ins_cost(SVE_COST);
 756   format %{ "sve_fabs $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
 757   ins_encode %{
 758     __ sve_fabs(as_FloatRegister($dst_src$$reg), __ S,
 759             as_PRegister($pg$$reg),
 760             as_FloatRegister($dst_src$$reg));
 761   %}
 762   ins_pipe(pipe_slow);
 763 %}
 764 
 765 instruct vabsD_masked(vReg dst_src, pRegGov pg) %{
 766   predicate(UseSVE > 0);
 767   match(Set dst_src (AbsVD dst_src pg));
 768   ins_cost(SVE_COST);
 769   format %{ "sve_fabs $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
 770   ins_encode %{
 771     __ sve_fabs(as_FloatRegister($dst_src$$reg), __ D,
 772             as_PRegister($pg$$reg),
 773             as_FloatRegister($dst_src$$reg));
 774   %}
 775   ins_pipe(pipe_slow);
 776 %}
 777 
 778 // vector add
 779 
 780 instruct vaddB(vReg dst, vReg src1, vReg src2) %{
 781   predicate(UseSVE > 0);
 782   match(Set dst (AddVB src1 src2));
 783   ins_cost(SVE_COST);
 784   format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (B)" %}
 785   ins_encode %{
 786     __ sve_add(as_FloatRegister($dst$$reg), __ B,
 787          as_FloatRegister($src1$$reg),
 788          as_FloatRegister($src2$$reg));
 789   %}
 790   ins_pipe(pipe_slow);
 791 %}
 792 
 793 instruct vaddS(vReg dst, vReg src1, vReg src2) %{
 794   predicate(UseSVE > 0);
 795   match(Set dst (AddVS src1 src2));
 796   ins_cost(SVE_COST);
 797   format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (H)" %}

 838     __ sve_fadd(as_FloatRegister($dst$$reg), __ S,
 839          as_FloatRegister($src1$$reg),
 840          as_FloatRegister($src2$$reg));
 841   %}
 842   ins_pipe(pipe_slow);
 843 %}
 844 
 845 instruct vaddD(vReg dst, vReg src1, vReg src2) %{
 846   predicate(UseSVE > 0);
 847   match(Set dst (AddVD src1 src2));
 848   ins_cost(SVE_COST);
 849   format %{ "sve_fadd $dst, $src1, $src2\t # vector (sve) (D)" %}
 850   ins_encode %{
 851     __ sve_fadd(as_FloatRegister($dst$$reg), __ D,
 852          as_FloatRegister($src1$$reg),
 853          as_FloatRegister($src2$$reg));
 854   %}
 855   ins_pipe(pipe_slow);
 856 %}
 857 
 858 // vector add - predicated
 859 
 860 instruct vaddB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 861   predicate(UseSVE > 0);
 862   match(Set dst_src1 (AddVB (Binary dst_src1 src2) pg));
 863   ins_cost(SVE_COST);
 864   format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
 865   ins_encode %{
 866     __ sve_add(as_FloatRegister($dst_src1$$reg), __ B,
 867             as_PRegister($pg$$reg),
 868             as_FloatRegister($src2$$reg));
 869   %}
 870   ins_pipe(pipe_slow);
 871 %}
 872 
 873 instruct vaddS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 874   predicate(UseSVE > 0);
 875   match(Set dst_src1 (AddVS (Binary dst_src1 src2) pg));
 876   ins_cost(SVE_COST);
 877   format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
 878   ins_encode %{
 879     __ sve_add(as_FloatRegister($dst_src1$$reg), __ H,
 880             as_PRegister($pg$$reg),
 881             as_FloatRegister($src2$$reg));
 882   %}
 883   ins_pipe(pipe_slow);
 884 %}
 885 
 886 instruct vaddI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 887   predicate(UseSVE > 0);
 888   match(Set dst_src1 (AddVI (Binary dst_src1 src2) pg));
 889   ins_cost(SVE_COST);
 890   format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
 891   ins_encode %{
 892     __ sve_add(as_FloatRegister($dst_src1$$reg), __ S,
 893             as_PRegister($pg$$reg),
 894             as_FloatRegister($src2$$reg));
 895   %}
 896   ins_pipe(pipe_slow);
 897 %}
 898 
 899 instruct vaddL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 900   predicate(UseSVE > 0);
 901   match(Set dst_src1 (AddVL (Binary dst_src1 src2) pg));
 902   ins_cost(SVE_COST);
 903   format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
 904   ins_encode %{
 905     __ sve_add(as_FloatRegister($dst_src1$$reg), __ D,
 906             as_PRegister($pg$$reg),
 907             as_FloatRegister($src2$$reg));
 908   %}
 909   ins_pipe(pipe_slow);
 910 %}
 911 
 912 instruct vaddF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 913   predicate(UseSVE > 0);
 914   match(Set dst_src1 (AddVF (Binary dst_src1 src2) pg));
 915   ins_cost(SVE_COST);
 916   format %{ "sve_fadd $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
 917   ins_encode %{
 918     __ sve_fadd(as_FloatRegister($dst_src1$$reg), __ S,
 919             as_PRegister($pg$$reg),
 920             as_FloatRegister($src2$$reg));
 921   %}
 922   ins_pipe(pipe_slow);
 923 %}
 924 
 925 instruct vaddD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 926   predicate(UseSVE > 0);
 927   match(Set dst_src1 (AddVD (Binary dst_src1 src2) pg));
 928   ins_cost(SVE_COST);
 929   format %{ "sve_fadd $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
 930   ins_encode %{
 931     __ sve_fadd(as_FloatRegister($dst_src1$$reg), __ D,
 932             as_PRegister($pg$$reg),
 933             as_FloatRegister($src2$$reg));
 934   %}
 935   ins_pipe(pipe_slow);
 936 %}
 937 
 938 // vector and
 939 
 940 instruct vand(vReg dst, vReg src1, vReg src2) %{
 941   predicate(UseSVE > 0);
 942   match(Set dst (AndV src1 src2));
 943   ins_cost(SVE_COST);
 944   format %{ "sve_and  $dst, $src1, $src2\t# vector (sve)" %}
 945   ins_encode %{
 946     __ sve_and(as_FloatRegister($dst$$reg),
 947          as_FloatRegister($src1$$reg),
 948          as_FloatRegister($src2$$reg));
 949   %}
 950   ins_pipe(pipe_slow);
 951 %}
 952 
 953 // vector or
 954 
 955 instruct vor(vReg dst, vReg src1, vReg src2) %{
 956   predicate(UseSVE > 0);
 957   match(Set dst (OrV src1 src2));

 963          as_FloatRegister($src2$$reg));
 964   %}
 965   ins_pipe(pipe_slow);
 966 %}
 967 
 968 // vector xor
 969 
 970 instruct vxor(vReg dst, vReg src1, vReg src2) %{
 971   predicate(UseSVE > 0);
 972   match(Set dst (XorV src1 src2));
 973   ins_cost(SVE_COST);
 974   format %{ "sve_eor  $dst, $src1, $src2\t# vector (sve)" %}
 975   ins_encode %{
 976     __ sve_eor(as_FloatRegister($dst$$reg),
 977          as_FloatRegister($src1$$reg),
 978          as_FloatRegister($src2$$reg));
 979   %}
 980   ins_pipe(pipe_slow);
 981 %}
 982 
 983 // vector and - predicated
 984 
 985 instruct vand_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 986   predicate(UseSVE > 0);
 987   match(Set dst_src1 (AndV (Binary dst_src1 src2) pg));


 988   ins_cost(SVE_COST);
 989   format %{ "sve_and $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
 990   ins_encode %{
 991     BasicType bt = Matcher::vector_element_basic_type(this);
 992     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
 993     __ sve_and(as_FloatRegister($dst_src1$$reg), size,
 994           as_PRegister($pg$$reg),
 995           as_FloatRegister($src2$$reg));
 996   %}
 997   ins_pipe(pipe_slow);
 998 %}
 999 
1000 // vector or - predicated
1001 
1002 instruct vor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1003   predicate(UseSVE > 0);
1004   match(Set dst_src1 (OrV (Binary dst_src1 src2) pg));
1005   ins_cost(SVE_COST);
1006   format %{ "sve_orr $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
1007   ins_encode %{
1008     BasicType bt = Matcher::vector_element_basic_type(this);
1009     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1010     __ sve_orr(as_FloatRegister($dst_src1$$reg), size,
1011           as_PRegister($pg$$reg),
1012           as_FloatRegister($src2$$reg));
1013   %}
1014   ins_pipe(pipe_slow);
1015 %}
1016 
1017 // vector xor - predicated
1018 
1019 instruct vxor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{


1020   predicate(UseSVE > 0);
1021   match(Set dst_src1 (XorV (Binary dst_src1 src2) pg));


1022   ins_cost(SVE_COST);
1023   format %{ "sve_eor $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
1024   ins_encode %{
1025     BasicType bt = Matcher::vector_element_basic_type(this);
1026     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1027     __ sve_eor(as_FloatRegister($dst_src1$$reg), size,
1028           as_PRegister($pg$$reg),
1029           as_FloatRegister($src2$$reg));
1030   %}
1031   ins_pipe(pipe_slow);
1032 %}
1033 
1034 // vector not
1035 
1036 instruct vnotI(vReg dst, vReg src, immI_M1 m1) %{
1037   predicate(UseSVE > 0);
1038   match(Set dst (XorV src (ReplicateB m1)));
1039   match(Set dst (XorV src (ReplicateS m1)));
1040   match(Set dst (XorV src (ReplicateI m1)));
1041   ins_cost(SVE_COST);
1042   format %{ "sve_not $dst, $src\t# vector (sve) B/H/S" %}
1043   ins_encode %{
1044     __ sve_not(as_FloatRegister($dst$$reg), __ D,
1045                ptrue, as_FloatRegister($src$$reg));
1046   %}
1047   ins_pipe(pipe_slow);
1048 %}
1049 
1050 instruct vnotL(vReg dst, vReg src, immL_M1 m1) %{
1051   predicate(UseSVE > 0);
1052   match(Set dst (XorV src (ReplicateL m1)));
1053   ins_cost(SVE_COST);
1054   format %{ "sve_not $dst, $src\t# vector (sve) D" %}
1055   ins_encode %{
1056     __ sve_not(as_FloatRegister($dst$$reg), __ D,
1057                ptrue, as_FloatRegister($src$$reg));
1058   %}
1059   ins_pipe(pipe_slow);
1060 %}
1061 
1062 // vector and_not
1063 
1064 instruct vand_notI(vReg dst, vReg src1, vReg src2, immI_M1 m1) %{
1065   predicate(UseSVE > 0);
1066   match(Set dst (AndV src1 (XorV src2 (ReplicateB m1))));
1067   match(Set dst (AndV src1 (XorV src2 (ReplicateS m1))));
1068   match(Set dst (AndV src1 (XorV src2 (ReplicateI m1))));
1069   ins_cost(SVE_COST);
1070   format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) B/H/S" %}
1071   ins_encode %{
1072     __ sve_bic(as_FloatRegister($dst$$reg),
1073                as_FloatRegister($src1$$reg),
1074                as_FloatRegister($src2$$reg));
1075   %}
1076   ins_pipe(pipe_slow);
1077 %}
1078 
1079 instruct vand_notL(vReg dst, vReg src1, vReg src2, immL_M1 m1) %{
1080   predicate(UseSVE > 0);
1081   match(Set dst (AndV src1 (XorV src2 (ReplicateL m1))));
1082   ins_cost(SVE_COST);
1083   format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) D" %}
1084   ins_encode %{
1085     __ sve_bic(as_FloatRegister($dst$$reg),
1086                as_FloatRegister($src1$$reg),
1087                as_FloatRegister($src2$$reg));
1088   %}
1089   ins_pipe(pipe_slow);
1090 %}
1091 

1092 // vector float div
1093 
1094 instruct vdivF(vReg dst_src1, vReg src2) %{
1095   predicate(UseSVE > 0);
1096   match(Set dst_src1 (DivVF dst_src1 src2));
1097   ins_cost(SVE_COST);
1098   format %{ "sve_fdiv  $dst_src1, $dst_src1, $src2\t# vector (sve) (S)" %}
1099   ins_encode %{
1100     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ S,
1101          ptrue, as_FloatRegister($src2$$reg));
1102   %}
1103   ins_pipe(pipe_slow);
1104 %}
1105 
1106 instruct vdivD(vReg dst_src1, vReg src2) %{
1107   predicate(UseSVE > 0);
1108   match(Set dst_src1 (DivVD dst_src1 src2));
1109   ins_cost(SVE_COST);
1110   format %{ "sve_fdiv  $dst_src1, $dst_src1, $src2\t# vector (sve) (D)" %}
1111   ins_encode %{
1112     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ D,
1113          ptrue, as_FloatRegister($src2$$reg));
1114   %}
1115   ins_pipe(pipe_slow);
1116 %}
1117 
1118 // vector float div - predicated
1119 
1120 instruct vfdivF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1121   predicate(UseSVE > 0);
1122   match(Set dst_src1 (DivVF (Binary dst_src1 src2) pg));
1123   ins_cost(SVE_COST);
1124   format %{ "sve_fdiv $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
1125   ins_encode %{
1126     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ S,
1127             as_PRegister($pg$$reg),
1128             as_FloatRegister($src2$$reg));
1129   %}
1130   ins_pipe(pipe_slow);
1131 %}
1132 
1133 instruct vfdivD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1134   predicate(UseSVE > 0);
1135   match(Set dst_src1 (DivVD (Binary dst_src1 src2) pg));
1136   ins_cost(SVE_COST);
1137   format %{ "sve_fdiv $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
1138   ins_encode %{
1139     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ D,
1140             as_PRegister($pg$$reg),
1141             as_FloatRegister($src2$$reg));
1142   %}
1143   ins_pipe(pipe_slow);
1144 %}
1145 
1146 // vector min/max
1147 
1148 instruct vmin(vReg dst_src1, vReg src2) %{
1149   predicate(UseSVE > 0);
1150   match(Set dst_src1 (MinV dst_src1 src2));
1151   ins_cost(SVE_COST);
1152   format %{ "sve_min $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
1153   ins_encode %{
1154     BasicType bt = Matcher::vector_element_basic_type(this);
1155     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1156     if (is_floating_point_type(bt)) {
1157       __ sve_fmin(as_FloatRegister($dst_src1$$reg), size,
1158                   ptrue, as_FloatRegister($src2$$reg));
1159     } else {
1160       assert(is_integral_type(bt), "unsupported type");
1161       __ sve_smin(as_FloatRegister($dst_src1$$reg), size,
1162                   ptrue, as_FloatRegister($src2$$reg));
1163     }
1164   %}
1165   ins_pipe(pipe_slow);
1166 %}
1167 
1168 instruct vmax(vReg dst_src1, vReg src2) %{
1169   predicate(UseSVE > 0);
1170   match(Set dst_src1 (MaxV dst_src1 src2));
1171   ins_cost(SVE_COST);
1172   format %{ "sve_max $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
1173   ins_encode %{
1174     BasicType bt = Matcher::vector_element_basic_type(this);
1175     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1176     if (is_floating_point_type(bt)) {
1177       __ sve_fmax(as_FloatRegister($dst_src1$$reg), size,
1178                   ptrue, as_FloatRegister($src2$$reg));
1179     } else {
1180       assert(is_integral_type(bt), "unsupported type");
1181       __ sve_smax(as_FloatRegister($dst_src1$$reg), size,
1182                   ptrue, as_FloatRegister($src2$$reg));
1183     }
1184   %}
1185   ins_pipe(pipe_slow);
1186 %}
1187 
1188 // vector min/max - predicated
1189 
1190 instruct vmin_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1191   predicate(UseSVE > 0);
1192   match(Set dst_src1 (MinV (Binary dst_src1 src2) pg));
1193   ins_cost(SVE_COST);
1194   format %{ "sve_min $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %}
1195   ins_encode %{
1196     BasicType bt = Matcher::vector_element_basic_type(this);
1197     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1198     if (is_floating_point_type(bt)) {
1199       __ sve_fmin(as_FloatRegister($dst_src1$$reg), size,
1200                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1201     } else {
1202       assert(is_integral_type(bt), "unsupported type");
1203       __ sve_smin(as_FloatRegister($dst_src1$$reg), size,
1204                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1205     }
1206   %}
1207   ins_pipe(pipe_slow);
1208 %}
1209 
1210 instruct vmax_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1211   predicate(UseSVE > 0);
1212   match(Set dst_src1 (MaxV (Binary dst_src1 src2) pg));
1213   ins_cost(SVE_COST);
1214   format %{ "sve_max $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %}
1215   ins_encode %{
1216     BasicType bt = Matcher::vector_element_basic_type(this);
1217     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1218     if (is_floating_point_type(bt)) {
1219       __ sve_fmax(as_FloatRegister($dst_src1$$reg), size,
1220                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1221     } else {
1222       assert(is_integral_type(bt), "unsupported type");
1223       __ sve_smax(as_FloatRegister($dst_src1$$reg), size,
1224                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1225     }
1226   %}
1227   ins_pipe(pipe_slow);
1228 %}
1229 
1230 // vector fmla
1231 
1232 // dst_src1 = dst_src1 + src2 * src3
1233 instruct vfmlaF(vReg dst_src1, vReg src2, vReg src3) %{
1234   predicate(UseFMA && UseSVE > 0);
1235   match(Set dst_src1 (FmaVF dst_src1 (Binary src2 src3)));
1236   ins_cost(SVE_COST);
1237   format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1238   ins_encode %{
1239     __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ S,
1240          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1241   %}
1242   ins_pipe(pipe_slow);
1243 %}
1244 
1245 // dst_src1 = dst_src1 + src2 * src3
1246 instruct vfmlaD(vReg dst_src1, vReg src2, vReg src3) %{
1247   predicate(UseFMA && UseSVE > 0);
1248   match(Set dst_src1 (FmaVD dst_src1 (Binary src2 src3)));
1249   ins_cost(SVE_COST);
1250   format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1251   ins_encode %{
1252     __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ D,
1253          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1254   %}
1255   ins_pipe(pipe_slow);
1256 %}
1257 
1258 // vector fmla - predicated
1259 
1260 // dst_src1 = dst_src1 * src2 + src3
1261 instruct vfmlaF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1262   predicate(UseFMA && UseSVE > 0);
1263   match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary src3 pg)));
1264   ins_cost(SVE_COST);
1265   format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) (S)" %}
1266   ins_encode %{
1267     __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
1268          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1269   %}
1270   ins_pipe(pipe_slow);
1271 %}
1272 
1273 // dst_src1 = dst_src1 * src2 + src3
1274 instruct vfmlaD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1275   predicate(UseFMA && UseSVE > 0);
1276   match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary src3 pg)));
1277   ins_cost(SVE_COST);
1278   format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) (D)" %}
1279   ins_encode %{
1280     __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
1281          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1282   %}
1283   ins_pipe(pipe_slow);
1284 %}
1285 
1286 // vector fmls
1287 
1288 // dst_src1 = dst_src1 + -src2 * src3
1289 // dst_src1 = dst_src1 + src2 * -src3
1290 instruct vfmlsF(vReg dst_src1, vReg src2, vReg src3) %{
1291   predicate(UseFMA && UseSVE > 0);
1292   match(Set dst_src1 (FmaVF dst_src1 (Binary (NegVF src2) src3)));
1293   match(Set dst_src1 (FmaVF dst_src1 (Binary src2 (NegVF src3))));
1294   ins_cost(SVE_COST);
1295   format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1296   ins_encode %{
1297     __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ S,
1298          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1299   %}
1300   ins_pipe(pipe_slow);
1301 %}
1302 
1303 // dst_src1 = dst_src1 + -src2 * src3
1304 // dst_src1 = dst_src1 + src2 * -src3
1305 instruct vfmlsD(vReg dst_src1, vReg src2, vReg src3) %{

1474     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ S,
1475       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1476   %}
1477   ins_pipe(pipe_slow);
1478 %}
1479 
1480 // dst_src1 = dst_src1 - src2 * src3
1481 instruct vmlsL(vReg dst_src1, vReg src2, vReg src3)
1482 %{
1483   predicate(UseSVE > 0);
1484   match(Set dst_src1 (SubVL dst_src1 (MulVL src2 src3)));
1485   ins_cost(SVE_COST);
1486   format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (D)" %}
1487   ins_encode %{
1488     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ D,
1489       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1490   %}
1491   ins_pipe(pipe_slow);
1492 %}
1493 

1494 // vector mul
1495 
1496 instruct vmulB(vReg dst_src1, vReg src2) %{
1497   predicate(UseSVE > 0);
1498   match(Set dst_src1 (MulVB dst_src1 src2));
1499   ins_cost(SVE_COST);
1500   format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (B)" %}
1501   ins_encode %{
1502     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ B,
1503          ptrue, as_FloatRegister($src2$$reg));
1504   %}
1505   ins_pipe(pipe_slow);
1506 %}
1507 
1508 instruct vmulS(vReg dst_src1, vReg src2) %{
1509   predicate(UseSVE > 0);
1510   match(Set dst_src1 (MulVS dst_src1 src2));
1511   ins_cost(SVE_COST);
1512   format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (H)" %}
1513   ins_encode %{

1550     __ sve_fmul(as_FloatRegister($dst$$reg), __ S,
1551          as_FloatRegister($src1$$reg),
1552          as_FloatRegister($src2$$reg));
1553   %}
1554   ins_pipe(pipe_slow);
1555 %}
1556 
1557 instruct vmulD(vReg dst, vReg src1, vReg src2) %{
1558   predicate(UseSVE > 0);
1559   match(Set dst (MulVD src1 src2));
1560   ins_cost(SVE_COST);
1561   format %{ "sve_fmul $dst, $src1, $src2\t # vector (sve) (D)" %}
1562   ins_encode %{
1563     __ sve_fmul(as_FloatRegister($dst$$reg), __ D,
1564          as_FloatRegister($src1$$reg),
1565          as_FloatRegister($src2$$reg));
1566   %}
1567   ins_pipe(pipe_slow);
1568 %}
1569 
1570 // vector mul - predicated
1571 
1572 instruct vmulB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1573   predicate(UseSVE > 0);
1574   match(Set dst_src1 (MulVB (Binary dst_src1 src2) pg));
1575   ins_cost(SVE_COST);
1576   format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
1577   ins_encode %{
1578     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ B,
1579             as_PRegister($pg$$reg),
1580             as_FloatRegister($src2$$reg));
1581   %}
1582   ins_pipe(pipe_slow);
1583 %}
1584 
1585 instruct vmulS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1586   predicate(UseSVE > 0);
1587   match(Set dst_src1 (MulVS (Binary dst_src1 src2) pg));
1588   ins_cost(SVE_COST);
1589   format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
1590   ins_encode %{
1591     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ H,
1592             as_PRegister($pg$$reg),
1593             as_FloatRegister($src2$$reg));
1594   %}
1595   ins_pipe(pipe_slow);
1596 %}
1597 
1598 instruct vmulI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1599   predicate(UseSVE > 0);
1600   match(Set dst_src1 (MulVI (Binary dst_src1 src2) pg));
1601   ins_cost(SVE_COST);
1602   format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
1603   ins_encode %{
1604     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ S,
1605             as_PRegister($pg$$reg),
1606             as_FloatRegister($src2$$reg));
1607   %}
1608   ins_pipe(pipe_slow);
1609 %}
1610 
1611 instruct vmulL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1612   predicate(UseSVE > 0);
1613   match(Set dst_src1 (MulVL (Binary dst_src1 src2) pg));
1614   ins_cost(SVE_COST);
1615   format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
1616   ins_encode %{
1617     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ D,
1618             as_PRegister($pg$$reg),
1619             as_FloatRegister($src2$$reg));
1620   %}
1621   ins_pipe(pipe_slow);
1622 %}
1623 
1624 instruct vmulF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1625   predicate(UseSVE > 0);
1626   match(Set dst_src1 (MulVF (Binary dst_src1 src2) pg));
1627   ins_cost(SVE_COST);
1628   format %{ "sve_fmul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
1629   ins_encode %{
1630     __ sve_fmul(as_FloatRegister($dst_src1$$reg), __ S,
1631             as_PRegister($pg$$reg),
1632             as_FloatRegister($src2$$reg));
1633   %}
1634   ins_pipe(pipe_slow);
1635 %}
1636 
1637 instruct vmulD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1638   predicate(UseSVE > 0);
1639   match(Set dst_src1 (MulVD (Binary dst_src1 src2) pg));
1640   ins_cost(SVE_COST);
1641   format %{ "sve_fmul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
1642   ins_encode %{
1643     __ sve_fmul(as_FloatRegister($dst_src1$$reg), __ D,
1644             as_PRegister($pg$$reg),
1645             as_FloatRegister($src2$$reg));
1646   %}
1647   ins_pipe(pipe_slow);
1648 %}
1649 
1650 // vector fneg
1651 
1652 instruct vnegF(vReg dst, vReg src) %{
1653   predicate(UseSVE > 0 &&
1654             !n->as_Vector()->is_predicated_vector());
1655   match(Set dst (NegVF src));
1656   ins_cost(SVE_COST);
1657   format %{ "sve_fneg $dst, $src\t# vector (sve) (S)" %}
1658   ins_encode %{
1659     __ sve_fneg(as_FloatRegister($dst$$reg), __ S,
1660          ptrue, as_FloatRegister($src$$reg));
1661   %}
1662   ins_pipe(pipe_slow);
1663 %}
1664 
1665 instruct vnegD(vReg dst, vReg src) %{
1666   predicate(UseSVE > 0 &&
1667             !n->as_Vector()->is_predicated_vector());
1668   match(Set dst (NegVD src));
1669   ins_cost(SVE_COST);
1670   format %{ "sve_fneg $dst, $src\t# vector (sve) (D)" %}
1671   ins_encode %{
1672     __ sve_fneg(as_FloatRegister($dst$$reg), __ D,
1673          ptrue, as_FloatRegister($src$$reg));
1674   %}
1675   ins_pipe(pipe_slow);
1676 %}
1677 
1678 // vector fneg - predicated
1679 
1680 instruct vnegF_masked(vReg dst_src, pRegGov pg) %{
1681   predicate(UseSVE > 0);
1682   match(Set dst_src (NegVF dst_src pg));
1683   ins_cost(SVE_COST);
1684   format %{ "sve_fneg $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
1685   ins_encode %{
1686     __ sve_fneg(as_FloatRegister($dst_src$$reg), __ S,
1687             as_PRegister($pg$$reg),
1688             as_FloatRegister($dst_src$$reg));
1689   %}
1690   ins_pipe(pipe_slow);
1691 %}
1692 
1693 instruct vnegD_masked(vReg dst_src, pRegGov pg) %{
1694   predicate(UseSVE > 0);
1695   match(Set dst_src (NegVD dst_src pg));
1696   ins_cost(SVE_COST);
1697   format %{ "sve_fneg $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
1698   ins_encode %{
1699     __ sve_fneg(as_FloatRegister($dst_src$$reg), __ D,
1700             as_PRegister($pg$$reg),
1701             as_FloatRegister($dst_src$$reg));
1702   %}
1703   ins_pipe(pipe_slow);
1704 %}
1705 
1706 // popcount vector
1707 
1708 instruct vpopcountI(vReg dst, vReg src) %{
1709   predicate(UseSVE > 0);
1710   match(Set dst (PopCountVI src));
1711   format %{ "sve_cnt $dst, $src\t# vector (sve) (S)\n\t" %}
1712   ins_encode %{
1713      __ sve_cnt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg));
1714   %}
1715   ins_pipe(pipe_slow);
1716 %}
1717 
1718 // vector mask compare
1719 
1720 instruct vmaskcmp(pRegGov dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %{
1721   predicate(UseSVE > 0);
1722   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
1723   effect(KILL cr);
1724   ins_cost(SVE_COST);
1725   format %{ "sve_cmp $dst, $src1, $src2\t# vector mask cmp (sve)" %}

1726   ins_encode %{
1727     BasicType bt = Matcher::vector_element_basic_type(this);
1728     __ sve_compare(as_PRegister($dst$$reg), bt, ptrue, as_FloatRegister($src1$$reg),
1729                    as_FloatRegister($src2$$reg), (int)$cond$$constant);


1730   %}
1731   ins_pipe(pipe_slow);
1732 %}
1733 
1734 instruct vmaskcmp_masked(pRegGov dst, vReg src1, vReg src2, immI cond, pRegGov pg, rFlagsReg cr) %{


1735   predicate(UseSVE > 0);
1736   match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond pg)));
1737   effect(KILL cr);
1738   ins_cost(SVE_COST);
1739   format %{ "sve_cmp $dst, $pg, $src1, $src2\t# vector mask cmp (sve)" %}

1740   ins_encode %{
1741     BasicType bt = Matcher::vector_element_basic_type(this);
1742     __ sve_compare(as_PRegister($dst$$reg), bt, as_PRegister($pg$$reg), as_FloatRegister($src1$$reg),
1743                    as_FloatRegister($src2$$reg), (int)$cond$$constant);



1744   %}
1745   ins_pipe(pipe_slow);
1746 %}
1747 
1748 // vector blend
1749 
1750 instruct vblend(vReg dst, vReg src1, vReg src2, pRegGov pg) %{

1751   predicate(UseSVE > 0);
1752   match(Set dst (VectorBlend (Binary src1 src2) pg));
1753   ins_cost(SVE_COST);
1754   format %{ "sve_sel $dst, $pg, $src2, $src1\t# vector blend (sve)" %}


1755   ins_encode %{
1756     Assembler::SIMD_RegVariant size =
1757                __ elemType_to_regVariant(Matcher::vector_element_basic_type(this));
1758     __ sve_sel(as_FloatRegister($dst$$reg), size, as_PRegister($pg$$reg),
1759                as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));


1760   %}
1761   ins_pipe(pipe_slow);
1762 %}
1763 
1764 // vector load mask
1765 
1766 instruct vloadmaskB(pRegGov dst, vReg src, rFlagsReg cr) %{
1767   predicate(UseSVE > 0 &&
1768             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
1769   match(Set dst (VectorLoadMask src));
1770   effect(KILL cr);
1771   ins_cost(SVE_COST);
1772   format %{ "vloadmaskB $dst, $src\t# vector load mask (sve) (B)" %}













1773   ins_encode %{
1774     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ B,
1775                ptrue, as_FloatRegister($src$$reg), 0);
1776   %}
1777   ins_pipe(pipe_slow);
1778 %}
1779 
1780 instruct vloadmask_extend(pRegGov dst, vReg src, vReg tmp, rFlagsReg cr) %{
1781   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() != T_BYTE);


1782   match(Set dst (VectorLoadMask src));
1783   effect(TEMP tmp, KILL cr);
1784   ins_cost(3 * SVE_COST);
1785   format %{ "vloadmask $dst, $src\t# vector load mask (sve) (H/S/D)" %}




















1786   ins_encode %{
1787     BasicType bt = Matcher::vector_element_basic_type(this);
1788     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1789     __ sve_vector_extend(as_FloatRegister($tmp$$reg), size, as_FloatRegister($src$$reg), __ B);
1790     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
1791   %}
1792   ins_pipe(pipe_slow);
1793 %}
1794 
1795 // vector store mask
1796 
1797 instruct vstoremaskB(vReg dst, pRegGov src, immI_1 size) %{
1798   predicate(UseSVE > 0);
1799   match(Set dst (VectorStoreMask src size));
1800   ins_cost(SVE_COST);
1801   format %{ "vstoremask $dst, $src\t# vector store mask (sve) (B)" %}
1802   ins_encode %{
1803     __ sve_cpy(as_FloatRegister($dst$$reg), __ B, as_PRegister($src$$reg), 1, false);

1804   %}
1805   ins_pipe(pipe_slow);
1806 %}
1807 
1808 instruct vstoremask_narrow(vReg dst, pRegGov src, vReg tmp, immI_gt_1 size) %{
1809   predicate(UseSVE > 0);
1810   match(Set dst (VectorStoreMask src size));
1811   effect(TEMP_DEF dst, TEMP tmp);
1812   ins_cost(3 * SVE_COST);
1813   format %{ "vstoremask $dst, $src\t# vector store mask (sve) (H/S/D)" %}






















1814   ins_encode %{
1815     Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant((int)$size$$constant);
1816     __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($src$$reg), 1, false);
1817     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
1818                          as_FloatRegister($dst$$reg), size, as_FloatRegister($tmp$$reg));



1819   %}
1820   ins_pipe(pipe_slow);
1821 %}
1822 
1823 // Combine LoadVector+VectorLoadMask when the vector element type is not T_BYTE






















1824 
1825 instruct vloadmask_loadV(pRegGov dst, indirect mem, vReg tmp, rFlagsReg cr) %{
1826   predicate(UseSVE > 0 &&
1827             n->as_Vector()->length_in_bytes() == MaxVectorSize &&
1828             type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);

1829   match(Set dst (VectorLoadMask (LoadVector mem)));
1830   effect(TEMP tmp, KILL cr);
1831   ins_cost(3 * SVE_COST);
1832   format %{ "sve_ld1b $tmp, $mem\n\t"
1833             "sve_cmpne $dst, $tmp, 0\t# load vector mask (sve) (H/S/D)" %}
1834   ins_encode %{
1835     // Load mask values which are boolean type, and extend them to the
1836     // expected vector element type. Convert the vector to predicate.
1837     BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
1838     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($tmp$$reg),
1839                           ptrue, T_BOOLEAN, to_vect_bt, $mem->opcode(),

1840                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1841     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ elemType_to_regVariant(to_vect_bt),
1842                ptrue, as_FloatRegister($tmp$$reg), 0);
1843   %}
1844   ins_pipe(pipe_slow);
1845 %}
1846 
1847 instruct vloadmask_loadV_partial(pRegGov dst, indirect mem, vReg vtmp, pRegGov ptmp, rFlagsReg cr) %{
1848   predicate(UseSVE > 0 &&
1849             n->as_Vector()->length_in_bytes() > 16 &&
1850             n->as_Vector()->length_in_bytes() < MaxVectorSize &&
1851             type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
1852   match(Set dst (VectorLoadMask (LoadVector mem)));
1853   effect(TEMP vtmp, TEMP ptmp, KILL cr);
1854   ins_cost(6 * SVE_COST);
1855   format %{ "vloadmask_loadV $dst, $mem\t# load vector mask partial (sve) (H/S/D)" %}
1856   ins_encode %{
1857     // Load valid mask values which are boolean type, and extend them to the
1858     // expected vector element type. Convert the vector to predicate.
1859     BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
1860     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(to_vect_bt);
1861     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this));
1862     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($vtmp$$reg),
1863                           as_PRegister($ptmp$$reg), T_BOOLEAN, to_vect_bt, $mem->opcode(),
1864                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1865     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($vtmp$$reg), 0);
1866   %}
1867   ins_pipe(pipe_slow);
1868 %}
1869 
1870 // Combine VectorStoreMask+StoreVector when the vector element type is not T_BYTE
1871 
1872 instruct storeV_vstoremask(indirect mem, pRegGov src, vReg tmp, immI_gt_1 esize) %{
1873   predicate(UseSVE > 0 &&
1874             Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
1875   match(Set mem (StoreVector mem (VectorStoreMask src esize)));
1876   effect(TEMP tmp);
1877   ins_cost(3 * SVE_COST);
1878   format %{ "sve_cpy $tmp, $src, 1\n\t"
1879             "sve_st1b $tmp, $mem\t# store vector mask (sve) (H/S/D)" %}
1880   ins_encode %{
1881     BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
1882     assert(type2aelembytes(from_vect_bt) == (int)$esize$$constant, "unsupported type.");
1883     Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
1884     __ sve_cpy(as_FloatRegister($tmp$$reg), size, as_PRegister($src$$reg), 1, false);

1885     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($tmp$$reg),
1886                           ptrue, T_BOOLEAN, from_vect_bt, $mem->opcode(),
1887                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1888   %}
1889   ins_pipe(pipe_slow);
1890 %}
1891 
1892 instruct storeV_vstoremask_partial(indirect mem, pRegGov src, vReg vtmp,
1893                                    immI_gt_1 esize, pRegGov ptmp, rFlagsReg cr) %{
1894   predicate(UseSVE > 0 &&
1895             n->as_StoreVector()->memory_size() > 16 &&
1896             type2aelembytes(n->as_StoreVector()->vect_type()->element_basic_type()) > 1 &&
1897             Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
1898   match(Set mem (StoreVector mem (VectorStoreMask src esize)));
1899   effect(TEMP vtmp, TEMP ptmp, KILL cr);
1900   format %{ "storeV_vstoremask $src, $mem\t# store vector mask partial (sve) (H/S/D)" %}
1901   ins_cost(6 * SVE_COST);

1902   ins_encode %{
1903     // Convert the valid src predicate to vector, and store the vector
1904     // elements as boolean values.
1905     BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
1906     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(from_vect_bt);
1907     __ sve_cpy(as_FloatRegister($vtmp$$reg), size, as_PRegister($src$$reg), 1, false);
1908     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
1909     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($vtmp$$reg),
1910                           as_PRegister($ptmp$$reg), T_BOOLEAN, from_vect_bt, $mem->opcode(),

1911                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1912   %}
1913   ins_pipe(pipe_slow);
1914 %}
1915 
1916 // vector add reduction
1917 
1918 instruct reduce_addI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
1919   predicate(UseSVE > 0 &&
1920             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1921   match(Set dst (AddReductionVI src1 src2));
1922   effect(TEMP_DEF dst, TEMP tmp);
1923   ins_cost(SVE_COST);
1924   format %{ "sve_reduce_addI $dst, $src1, $src2\t# addI reduction (sve) (may extend)" %}
1925   ins_encode %{
1926     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1927     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1928                            $src1$$Register, as_FloatRegister($src2$$reg),
1929                            ptrue, as_FloatRegister($tmp$$reg));
1930   %}
1931   ins_pipe(pipe_slow);
1932 %}
1933 
1934 instruct reduce_addL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
1935   predicate(UseSVE > 0 &&
1936             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1937   match(Set dst (AddReductionVL src1 src2));
1938   effect(TEMP_DEF dst, TEMP tmp);
1939   ins_cost(SVE_COST);
1940   format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction (sve)" %}
1941   ins_encode %{
1942     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1943                            $src1$$Register, as_FloatRegister($src2$$reg),
1944                            ptrue, as_FloatRegister($tmp$$reg));
1945   %}
1946   ins_pipe(pipe_slow);
1947 %}
1948 
1949 instruct reduce_addF(vRegF src1_dst, vReg src2) %{
1950   predicate(UseSVE > 0 &&
1951             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1952   match(Set src1_dst (AddReductionVF src1_dst src2));
1953   ins_cost(SVE_COST);
1954   format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (S)" %}
1955   ins_encode %{
1956     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
1957          ptrue, as_FloatRegister($src2$$reg));
1958   %}
1959   ins_pipe(pipe_slow);
1960 %}
1961 
1962 instruct reduce_addD(vRegD src1_dst, vReg src2) %{
1963   predicate(UseSVE > 0 &&
1964             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1965   match(Set src1_dst (AddReductionVD src1_dst src2));
1966   ins_cost(SVE_COST);
1967   format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (D)" %}
1968   ins_encode %{
1969     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
1970          ptrue, as_FloatRegister($src2$$reg));
1971   %}
1972   ins_pipe(pipe_slow);
1973 %}
1974 
1975 instruct reduce_addI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1976                              pRegGov ptmp, rFlagsReg cr) %{
1977   predicate(UseSVE > 0 &&
1978             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1979   match(Set dst (AddReductionVI src1 src2));
1980   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1981   ins_cost(2 * SVE_COST);
1982   format %{ "sve_reduce_addI $dst, $src1, $src2\t# addI reduction partial (sve) (may extend)" %}
1983   ins_encode %{
1984     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1985     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1986     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1987                           Matcher::vector_length(this, $src2));
1988     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1989                            $src1$$Register, as_FloatRegister($src2$$reg),
1990                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));








1991   %}
1992   ins_pipe(pipe_slow);
1993 %}
1994 
1995 instruct reduce_addL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1996                              pRegGov ptmp, rFlagsReg cr) %{
1997   predicate(UseSVE > 0 &&
1998             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1999   match(Set dst (AddReductionVL src1 src2));
2000   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2001   ins_cost(2 * SVE_COST);
2002   format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction partial (sve)" %}
2003   ins_encode %{
2004     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2005                           Matcher::vector_length(this, $src2));
2006     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2007                            $src1$$Register, as_FloatRegister($src2$$reg),
2008                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2009   %}
2010   ins_pipe(pipe_slow);
2011 %}
2012 
2013 instruct reduce_addF_partial(vRegF src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
2014   predicate(UseSVE > 0 &&
2015             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2016   match(Set src1_dst (AddReductionVF src1_dst src2));
2017   ins_cost(SVE_COST);
2018   effect(TEMP ptmp, KILL cr);
2019   format %{ "sve_reduce_addF $src1_dst, $src1_dst, $src2\t# addF reduction partial (sve) (S)" %}
2020   ins_encode %{
2021     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2022                           Matcher::vector_length(this, $src2));
2023     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
2024                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2025   %}
2026   ins_pipe(pipe_slow);
2027 %}
2028 
2029 instruct reduce_addD_partial(vRegD src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
2030   predicate(UseSVE > 0 &&
2031             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2032   match(Set src1_dst (AddReductionVD src1_dst src2));

2033   ins_cost(SVE_COST);
2034   effect(TEMP ptmp, KILL cr);
2035   format %{ "sve_reduce_addD $src1_dst, $src1_dst, $src2\t# addD reduction partial (sve) (D)" %}
2036   ins_encode %{
2037     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2038                           Matcher::vector_length(this, $src2));
2039     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
2040                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));


2041   %}
2042   ins_pipe(pipe_slow);
2043 %}
2044 
2045 // vector add reduction - predicated
2046 
2047 instruct reduce_addI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2048   predicate(UseSVE > 0 &&
2049             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2050   match(Set dst (AddReductionVI (Binary src1 src2) pg));
2051   effect(TEMP_DEF dst, TEMP tmp);
2052   ins_cost(SVE_COST);
2053   format %{ "sve_reduce_addI $dst, $src1, $pg, $src2\t# addI reduction predicated (sve) (may extend)" %}
2054   ins_encode %{
2055     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2056     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2057                            $src1$$Register, as_FloatRegister($src2$$reg),
2058                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2059   %}
2060   ins_pipe(pipe_slow);
2061 %}
2062 
2063 instruct reduce_addL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2064   predicate(UseSVE > 0 &&
2065             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2066   match(Set dst (AddReductionVL (Binary src1 src2) pg));
2067   effect(TEMP_DEF dst, TEMP tmp);
2068   ins_cost(SVE_COST);
2069   format %{ "sve_reduce_addL $dst, $src1, $pg, $src2\t# addL reduction predicated (sve)" %}
2070   ins_encode %{
2071     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2072                            $src1$$Register, as_FloatRegister($src2$$reg),
2073                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2074   %}
2075   ins_pipe(pipe_slow);
2076 %}
2077 
2078 instruct reduce_addF_masked(vRegF src1_dst, vReg src2, pRegGov pg) %{
2079   predicate(UseSVE > 0 &&
2080             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2081   match(Set src1_dst (AddReductionVF (Binary src1_dst src2) pg));
2082   ins_cost(SVE_COST);
2083   format %{ "sve_reduce_addF $src1_dst, $pg, $src2\t# addF reduction predicated (sve)" %}
2084   ins_encode %{


2085     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
2086                  as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
2087   %}
2088   ins_pipe(pipe_slow);
2089 %}
2090 
2091 instruct reduce_addD_masked(vRegD src1_dst, vReg src2, pRegGov pg) %{
2092   predicate(UseSVE > 0 &&
2093             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2094   match(Set src1_dst (AddReductionVD (Binary src1_dst src2) pg));
2095   ins_cost(SVE_COST);
2096   format %{ "sve_reduce_addD $src1_dst, $pg, $src2\t# addD reduction predicated (sve)" %}
2097   ins_encode %{
2098     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
2099                  as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
2100   %}
2101   ins_pipe(pipe_slow);
2102 %}
2103 
2104 instruct reduce_addI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2105                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2106   predicate(UseSVE > 0 &&
2107             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2108   match(Set dst (AddReductionVI (Binary src1 src2) pg));
2109   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2110   ins_cost(3 * SVE_COST);
2111   format %{ "sve_reduce_addI $dst, $src1, $pg, $src2\t# addI reduction predicated partial (sve) (may extend)" %}
2112   ins_encode %{
2113     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2114     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2115     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2116                           Matcher::vector_length(this, $src2));
2117     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2118                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2119     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2120                            $src1$$Register, as_FloatRegister($src2$$reg),
2121                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2122   %}
2123   ins_pipe(pipe_slow);
2124 %}
2125 
2126 instruct reduce_addL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2127                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2128   predicate(UseSVE > 0 &&
2129             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2130   match(Set dst (AddReductionVL (Binary src1 src2) pg));
2131   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2132   ins_cost(3 * SVE_COST);
2133   format %{ "sve_reduce_addL $dst, $src1, $pg, $src2\t# addL reduction predicated partial (sve)" %}
2134   ins_encode %{
2135     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2136                           Matcher::vector_length(this, $src2));
2137     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2138                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2139     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2140                            $src1$$Register, as_FloatRegister($src2$$reg),
2141                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2142   %}
2143   ins_pipe(pipe_slow);
2144 %}
2145 
2146 instruct reduce_addF_masked_partial(vRegF src1_dst, vReg src2, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2147   predicate(UseSVE > 0 &&
2148             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2149   match(Set src1_dst (AddReductionVF (Binary src1_dst src2) pg));
2150   effect(TEMP ptmp, KILL cr);
2151   ins_cost(SVE_COST);
2152   format %{ "sve_reduce_addF $src1_dst, $pg, $src2\t# addF reduction predicated partial (sve)" %}
2153   ins_encode %{
2154     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2155                           Matcher::vector_length(this, $src2));
2156     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2157                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2158     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
2159                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2160   %}
2161   ins_pipe(pipe_slow);
2162 %}
2163 
2164 instruct reduce_addD_masked_partial(vRegD src1_dst, vReg src2, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2165   predicate(UseSVE > 0 &&
2166             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2167   match(Set src1_dst (AddReductionVD (Binary src1_dst src2) pg));
2168   effect(TEMP ptmp, KILL cr);
2169   ins_cost(SVE_COST);
2170   format %{ "sve_reduce_addD $src1_dst, $pg, $src2\t# addD reduction predicated partial (sve)" %}
2171   ins_encode %{
2172     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2173                           Matcher::vector_length(this, $src2));
2174     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2175                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2176     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
2177                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2178   %}
2179   ins_pipe(pipe_slow);
2180 %}
2181 
2182 // vector and reduction
2183 
2184 instruct reduce_andI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
2185   predicate(UseSVE > 0 &&
2186             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2187             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2188   match(Set dst (AndReductionV src1 src2));
2189   effect(TEMP_DEF dst, TEMP tmp);
2190   ins_cost(SVE_COST);
2191   format %{ "sve_reduce_andI $dst, $src1, $src2\t# andI reduction (sve) (may extend)" %}
2192   ins_encode %{
2193     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2194     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2195                            $src1$$Register, as_FloatRegister($src2$$reg),
2196                            ptrue, as_FloatRegister($tmp$$reg));
2197   %}
2198   ins_pipe(pipe_slow);
2199 %}
2200 
2201 instruct reduce_andL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
2202   predicate(UseSVE > 0 &&
2203             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2204             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2205   match(Set dst (AndReductionV src1 src2));
2206   effect(TEMP_DEF dst, TEMP tmp);
2207   ins_cost(SVE_COST);
2208   format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction (sve)" %}
2209   ins_encode %{
2210     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2211                            $src1$$Register, as_FloatRegister($src2$$reg),
2212                            ptrue, as_FloatRegister($tmp$$reg));
2213   %}
2214   ins_pipe(pipe_slow);
2215 %}
2216 
2217 instruct reduce_andI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2218                              pRegGov ptmp, rFlagsReg cr) %{
2219   predicate(UseSVE > 0 &&
2220             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2221             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2222   match(Set dst (AndReductionV src1 src2));
2223   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2224   ins_cost(2 * SVE_COST);
2225   format %{ "sve_reduce_andI $dst, $src1, $src2\t# andI reduction partial (sve) (may extend)" %}
2226   ins_encode %{
2227     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2228     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2229     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2230                           Matcher::vector_length(this, $src2));
2231     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2232                            $src1$$Register, as_FloatRegister($src2$$reg),
2233                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));








2234   %}
2235   ins_pipe(pipe_slow);
2236 %}
2237 
2238 instruct reduce_andL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2239                              pRegGov ptmp, rFlagsReg cr) %{
2240   predicate(UseSVE > 0 &&
2241             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2242             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2243   match(Set dst (AndReductionV src1 src2));
2244   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2245   ins_cost(2 * SVE_COST);
2246   format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction partial (sve)" %}
2247   ins_encode %{
2248     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2249                           Matcher::vector_length(this, $src2));
2250     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2251                            $src1$$Register, as_FloatRegister($src2$$reg),
2252                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2253   %}
2254   ins_pipe(pipe_slow);
2255 %}
2256 
2257 // vector and reduction - predicated
2258 
2259 instruct reduce_andI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2260   predicate(UseSVE > 0 &&
2261             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2262             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2263   match(Set dst (AndReductionV (Binary src1 src2) pg));
2264   effect(TEMP_DEF dst, TEMP tmp);
2265   ins_cost(SVE_COST);
2266   format %{ "sve_reduce_andI $dst, $src1, $pg, $src2\t# andI reduction predicated (sve) (may extend)" %}
2267   ins_encode %{
2268     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2269     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2270                            $src1$$Register, as_FloatRegister($src2$$reg),
2271                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2272   %}
2273   ins_pipe(pipe_slow);
2274 %}
2275 
2276 instruct reduce_andL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2277   predicate(UseSVE > 0 &&
2278             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2279             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2280   match(Set dst (AndReductionV (Binary src1 src2) pg));
2281   effect(TEMP_DEF dst, TEMP tmp);
2282   ins_cost(SVE_COST);
2283   format %{ "sve_reduce_andL $dst, $src1, $pg, $src2\t# andL reduction predicated (sve)" %}
2284   ins_encode %{
2285     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2286                            $src1$$Register, as_FloatRegister($src2$$reg),
2287                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2288   %}
2289   ins_pipe(pipe_slow);
2290 %}
2291 
2292 instruct reduce_andI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2293                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2294   predicate(UseSVE > 0 &&
2295             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2296             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2297   match(Set dst (AndReductionV (Binary src1 src2) pg));
2298   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2299   ins_cost(3 * SVE_COST);
2300   format %{ "sve_reduce_andI $dst, $src1, $pg, $src2\t# andI reduction predicated partial (sve) (may extend)" %}
2301   ins_encode %{
2302     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2303     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2304     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2305                           Matcher::vector_length(this, $src2));
2306     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2307                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2308     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2309                            $src1$$Register, as_FloatRegister($src2$$reg),
2310                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2311   %}
2312   ins_pipe(pipe_slow);
2313 %}
2314 
2315 instruct reduce_andL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2316                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2317   predicate(UseSVE > 0 &&
2318             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2319             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2320   match(Set dst (AndReductionV (Binary src1 src2) pg));
2321   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2322   ins_cost(3 * SVE_COST);
2323   format %{ "sve_reduce_andL $dst, $src1, $pg, $src2\t# andL reduction predicated partial (sve)" %}
2324   ins_encode %{
2325     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2326                           Matcher::vector_length(this, $src2));
2327     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2328                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2329     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2330                            $src1$$Register, as_FloatRegister($src2$$reg),
2331                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2332   %}
2333   ins_pipe(pipe_slow);
2334 %}
2335 
2336 // vector or reduction
2337 
2338 instruct reduce_orI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
2339   predicate(UseSVE > 0 &&
2340             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2341             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2342   match(Set dst (OrReductionV src1 src2));
2343   effect(TEMP_DEF dst, TEMP tmp);
2344   ins_cost(SVE_COST);
2345   format %{ "sve_reduce_orI $dst, $src1, $src2\t# orI reduction (sve) (may extend)" %}
2346   ins_encode %{
2347     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2348     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2349                            $src1$$Register, as_FloatRegister($src2$$reg),
2350                            ptrue, as_FloatRegister($tmp$$reg));
2351   %}
2352   ins_pipe(pipe_slow);
2353 %}
2354 
2355 instruct reduce_orL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
2356   predicate(UseSVE > 0 &&
2357             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2358             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2359   match(Set dst (OrReductionV src1 src2));
2360   effect(TEMP_DEF dst, TEMP tmp);
2361   ins_cost(SVE_COST);
2362   format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction (sve)" %}
2363   ins_encode %{
2364     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2365                            $src1$$Register, as_FloatRegister($src2$$reg),
2366                            ptrue, as_FloatRegister($tmp$$reg));
2367   %}
2368   ins_pipe(pipe_slow);
2369 %}
2370 
2371 instruct reduce_orI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2372                              pRegGov ptmp, rFlagsReg cr) %{
2373   predicate(UseSVE > 0 &&
2374             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2375             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2376   match(Set dst (OrReductionV src1 src2));
2377   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2378   ins_cost(2 * SVE_COST);
2379   format %{ "sve_reduce_orI $dst, $src1, $src2\t# orI reduction partial (sve) (may extend)" %}
2380   ins_encode %{
2381     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2382     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2383     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2384                           Matcher::vector_length(this, $src2));
2385     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2386                            $src1$$Register, as_FloatRegister($src2$$reg),
2387                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));








2388   %}
2389   ins_pipe(pipe_slow);
2390 %}
2391 
2392 instruct reduce_orL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2393                              pRegGov ptmp, rFlagsReg cr) %{
2394   predicate(UseSVE > 0 &&
2395             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2396             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2397   match(Set dst (OrReductionV src1 src2));
2398   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2399   ins_cost(2 * SVE_COST);
2400   format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction partial (sve)" %}
2401   ins_encode %{
2402     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2403                           Matcher::vector_length(this, $src2));
2404     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2405                            $src1$$Register, as_FloatRegister($src2$$reg),
2406                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2407   %}
2408   ins_pipe(pipe_slow);
2409 %}
2410 
2411 // vector or reduction - predicated
2412 
2413 instruct reduce_orI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2414   predicate(UseSVE > 0 &&
2415             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2416             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2417   match(Set dst (OrReductionV (Binary src1 src2) pg));
2418   effect(TEMP_DEF dst, TEMP tmp);
2419   ins_cost(SVE_COST);
2420   format %{ "sve_reduce_orI $dst, $src1, $pg, $src2\t# orI reduction predicated (sve) (may extend)" %}
2421   ins_encode %{
2422     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2423     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2424                            $src1$$Register, as_FloatRegister($src2$$reg),
2425                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2426   %}
2427   ins_pipe(pipe_slow);
2428 %}
2429 
2430 instruct reduce_orL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2431   predicate(UseSVE > 0 &&
2432             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2433             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2434   match(Set dst (OrReductionV (Binary src1 src2) pg));
2435   effect(TEMP_DEF dst, TEMP tmp);
2436   ins_cost(SVE_COST);
2437   format %{ "sve_reduce_orL $dst, $src1, $pg, $src2\t# orL reduction predicated (sve)" %}
2438   ins_encode %{
2439     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2440                            $src1$$Register, as_FloatRegister($src2$$reg),
2441                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2442   %}
2443   ins_pipe(pipe_slow);
2444 %}
2445 
2446 instruct reduce_orI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2447                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2448   predicate(UseSVE > 0 &&
2449             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2450             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2451   match(Set dst (OrReductionV (Binary src1 src2) pg));
2452   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2453   ins_cost(3 * SVE_COST);
2454   format %{ "sve_reduce_orI $dst, $src1, $pg, $src2\t# orI reduction predicated partial (sve) (may extend)" %}
2455   ins_encode %{
2456     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2457     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2458     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2459                           Matcher::vector_length(this, $src2));
2460     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2461                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2462     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2463                            $src1$$Register, as_FloatRegister($src2$$reg),
2464                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2465   %}
2466   ins_pipe(pipe_slow);
2467 %}
2468 
2469 instruct reduce_orL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2470                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2471   predicate(UseSVE > 0 &&
2472             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2473             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2474   match(Set dst (OrReductionV (Binary src1 src2) pg));
2475   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2476   ins_cost(3 * SVE_COST);
2477   format %{ "sve_reduce_orL $dst, $src1, $pg, $src2\t# orL reduction predicated partial (sve)" %}
2478   ins_encode %{
2479     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2480                           Matcher::vector_length(this, $src2));
2481     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2482                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2483     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2484                            $src1$$Register, as_FloatRegister($src2$$reg),
2485                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2486   %}
2487   ins_pipe(pipe_slow);
2488 %}
2489 
2490 // vector xor reduction
2491 
2492 instruct reduce_eorI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
2493   predicate(UseSVE > 0 &&
2494             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2495             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2496   match(Set dst (XorReductionV src1 src2));
2497   effect(TEMP_DEF dst, TEMP tmp);
2498   ins_cost(SVE_COST);
2499   format %{ "sve_reduce_eorI $dst, $src1, $src2\t# eorI reduction (sve) (may extend)" %}
2500   ins_encode %{
2501     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2502     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2503                            $src1$$Register, as_FloatRegister($src2$$reg),
2504                            ptrue, as_FloatRegister($tmp$$reg));
2505   %}
2506   ins_pipe(pipe_slow);
2507 %}
2508 
2509 instruct reduce_eorL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
2510   predicate(UseSVE > 0 &&
2511             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2512             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2513   match(Set dst (XorReductionV src1 src2));
2514   effect(TEMP_DEF dst, TEMP tmp);
2515   ins_cost(SVE_COST);
2516   format %{ "sve_reduce_eorL $dst, $src1, $src2\t# eorL reduction (sve)" %}
2517   ins_encode %{
2518     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2519                            $src1$$Register, as_FloatRegister($src2$$reg),
2520                            ptrue, as_FloatRegister($tmp$$reg));
2521   %}
2522   ins_pipe(pipe_slow);
2523 %}
2524 
2525 instruct reduce_eorI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2526                              pRegGov ptmp, rFlagsReg cr) %{
2527   predicate(UseSVE > 0 &&
2528             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2529             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2530   match(Set dst (XorReductionV src1 src2));
2531   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2532   ins_cost(2 * SVE_COST);
2533   format %{ "sve_reduce_eorI $dst, $src1, $src2\t# eorI reduction partial (sve) (may extend)" %}
2534   ins_encode %{
2535     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2536     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2537     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2538                           Matcher::vector_length(this, $src2));
2539     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2540                            $src1$$Register, as_FloatRegister($src2$$reg),
2541                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2542   %}
2543   ins_pipe(pipe_slow);
2544 %}
2545 
2546 instruct reduce_eorL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2547                              pRegGov ptmp, rFlagsReg cr) %{
2548   predicate(UseSVE > 0 &&
2549             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2550             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2551   match(Set dst (XorReductionV src1 src2));
2552   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2553   ins_cost(2 * SVE_COST);
2554   format %{ "sve_reduce_eorL $dst, $src1, $src2\t# eorL reduction partial (sve)" %}
2555   ins_encode %{
2556     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2557                           Matcher::vector_length(this, $src2));
2558     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2559                            $src1$$Register, as_FloatRegister($src2$$reg),
2560                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));

2561   %}
2562   ins_pipe(pipe_slow);
2563 %}
2564 
2565 // vector xor reduction - predicated
2566 
2567 instruct reduce_eorI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2568   predicate(UseSVE > 0 &&
2569             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2570             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2571   match(Set dst (XorReductionV (Binary src1 src2) pg));
2572   effect(TEMP_DEF dst, TEMP tmp);
2573   ins_cost(SVE_COST);
2574   format %{ "sve_reduce_eorI $dst, $src1, $pg, $src2\t# eorI reduction predicated (sve) (may extend)" %}
2575   ins_encode %{
2576     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2577     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2578                            $src1$$Register, as_FloatRegister($src2$$reg),
2579                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));








2580   %}
2581   ins_pipe(pipe_slow);
2582 %}
2583 
2584 instruct reduce_eorL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2585   predicate(UseSVE > 0 &&
2586             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2587             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2588   match(Set dst (XorReductionV (Binary src1 src2) pg));
2589   effect(TEMP_DEF dst, TEMP tmp);
2590   ins_cost(SVE_COST);
2591   format %{ "sve_reduce_eorL $dst, $src1, $pg, $src2\t# eorL reduction predicated (sve)" %}
2592   ins_encode %{
2593     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2594                            $src1$$Register, as_FloatRegister($src2$$reg),
2595                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));












2596   %}
2597   ins_pipe(pipe_slow);
2598 %}
2599 
2600 instruct reduce_eorI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2601                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2602   predicate(UseSVE > 0 &&
2603             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2604             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2605   match(Set dst (XorReductionV (Binary src1 src2) pg));
2606   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2607   ins_cost(3 * SVE_COST);
2608   format %{ "sve_reduce_eorI $dst, $src1, $pg, $src2\t# eorI reduction predicated partial (sve) (may extend)" %}
2609   ins_encode %{
2610     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2611     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2612     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2613                           Matcher::vector_length(this, $src2));
2614     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2615                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2616     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2617                            $src1$$Register, as_FloatRegister($src2$$reg),
2618                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2619   %}
2620   ins_pipe(pipe_slow);
2621 %}
2622 
2623 instruct reduce_eorL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2624                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2625   predicate(UseSVE > 0 &&
2626             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2627             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2628   match(Set dst (XorReductionV (Binary src1 src2) pg));
2629   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2630   ins_cost(3 * SVE_COST);
2631   format %{ "sve_reduce_eorL $dst, $src1, $pg, $src2\t# eorL reduction predicated partial (sve)" %}
2632   ins_encode %{
2633     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2634                           Matcher::vector_length(this, $src2));
2635     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2636                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2637     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2638                            $src1$$Register, as_FloatRegister($src2$$reg),
2639                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2640   %}
2641   ins_pipe(pipe_slow);
2642 %}
2643 

2644 // vector max reduction
2645 
2646 instruct reduce_maxI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
2647   predicate(UseSVE > 0 &&
2648             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2649             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2650             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
2651   match(Set dst (MaxReductionV src1 src2));
2652   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2653   ins_cost(SVE_COST);
2654   format %{ "sve_reduce_maxI $dst, $src1, $src2\t# maxI reduction (sve)" %}
2655   ins_encode %{
2656     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2657     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2658                            $src1$$Register, as_FloatRegister($src2$$reg),
2659                            ptrue, as_FloatRegister($tmp$$reg));


2660   %}
2661   ins_pipe(pipe_slow);
2662 %}
2663 
2664 instruct reduce_maxL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
2665   predicate(UseSVE > 0 &&
2666             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2667             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);


2668   match(Set dst (MaxReductionV src1 src2));
2669   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2670   ins_cost(SVE_COST);
2671   format %{ "sve_reduce_maxL $dst, $src1, $src2\t# maxL reduction (sve)" %}
2672   ins_encode %{
2673     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2674                            $src1$$Register, as_FloatRegister($src2$$reg),
2675                            ptrue, as_FloatRegister($tmp$$reg));






2676   %}
2677   ins_pipe(pipe_slow);
2678 %}
2679 
2680 instruct reduce_maxI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2681                              pRegGov ptmp, rFlagsReg cr) %{
2682   predicate(UseSVE > 0 &&
2683             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2684             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2685             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
2686   match(Set dst (MaxReductionV src1 src2));
2687   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2688   ins_cost(2 * SVE_COST);
2689   format %{ "sve_reduce_maxI $dst, $src1, $src2\t# maxI reduction partial (sve)" %}
2690   ins_encode %{
2691     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2692     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2693     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2694                           Matcher::vector_length(this, $src2));
2695     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2696                            $src1$$Register, as_FloatRegister($src2$$reg),
2697                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2698   %}
2699   ins_pipe(pipe_slow);
2700 %}
2701 
2702 instruct reduce_maxL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2703                              pRegGov ptmp, rFlagsReg cr) %{
2704   predicate(UseSVE > 0 &&
2705             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2706             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2707   match(Set dst (MaxReductionV src1 src2));
2708   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2709   ins_cost(2 * SVE_COST);
2710   format %{ "sve_reduce_maxL $dst, $src1, $src2\t# maxL reduction  partial (sve)" %}
2711   ins_encode %{
2712     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2713                           Matcher::vector_length(this, $src2));
2714     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2715                            $src1$$Register, as_FloatRegister($src2$$reg),
2716                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));


2717   %}
2718   ins_pipe(pipe_slow);
2719 %}
2720 
2721 instruct reduce_maxF(vRegF dst, vRegF src1, vReg src2) %{
2722   predicate(UseSVE > 0 &&
2723             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
2724             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2725   match(Set dst (MaxReductionV src1 src2));
2726   ins_cost(INSN_COST);
2727   effect(TEMP_DEF dst);
2728   format %{ "sve_reduce_maxF $dst, $src1, $src2\t# maxF reduction (sve)" %}

2729   ins_encode %{
2730     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src2$$reg));

2731     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2732   %}
2733   ins_pipe(pipe_slow);
2734 %}
2735 
2736 instruct reduce_maxF_partial(vRegF dst, vRegF src1, vReg src2,
2737                              pRegGov ptmp, rFlagsReg cr) %{
2738   predicate(UseSVE > 0 &&
2739             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
2740             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2741   match(Set dst (MaxReductionV src1 src2));
2742   ins_cost(INSN_COST);
2743   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
2744   format %{ "sve_reduce_maxF $dst, $src1, $src2\t# maxF reduction partial (sve)" %}
2745   ins_encode %{
2746     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2747                           Matcher::vector_length(this, $src2));
2748     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));

2749     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2750   %}
2751   ins_pipe(pipe_slow);
2752 %}
2753 
2754 instruct reduce_maxD(vRegD dst, vRegD src1, vReg src2) %{
2755   predicate(UseSVE > 0 &&
2756             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
2757             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2758   match(Set dst (MaxReductionV src1 src2));
2759   ins_cost(INSN_COST);
2760   effect(TEMP_DEF dst);
2761   format %{ "sve_reduce_maxD $dst, $src1, $src2\t# maxD reduction (sve)" %}

2762   ins_encode %{
2763     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));

2764     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2765   %}
2766   ins_pipe(pipe_slow);
2767 %}
2768 
2769 instruct reduce_maxD_partial(vRegD dst, vRegD src1, vReg src2,
2770                              pRegGov ptmp, rFlagsReg cr) %{
2771   predicate(UseSVE > 0 &&
2772             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
2773             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2774   match(Set dst (MaxReductionV src1 src2));
2775   ins_cost(INSN_COST);
2776   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
2777   format %{ "sve_reduce_maxD $dst, $src1, $src2\t# maxD reduction partial (sve)" %}
2778   ins_encode %{
2779     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2780                           Matcher::vector_length(this, $src2));
2781     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));

2782     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2783   %}
2784   ins_pipe(pipe_slow);
2785 %}
2786 
2787 // vector max reduction - predicated
2788 
2789 instruct reduce_maxI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp,
2790                            pRegGov pg, rFlagsReg cr) %{
2791   predicate(UseSVE > 0 &&
2792             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2793             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2794             is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
2795   match(Set dst (MaxReductionV (Binary src1 src2) pg));
2796   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2797   ins_cost(SVE_COST);
2798   format %{ "sve_reduce_maxI $dst, $src1, $pg, $src2\t# maxI reduction predicated (sve)" %}
2799   ins_encode %{
2800     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2801     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2802                            $src1$$Register, as_FloatRegister($src2$$reg),
2803                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));


2804   %}
2805   ins_pipe(pipe_slow);
2806 %}
2807 
2808 instruct reduce_maxL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp,
2809                           pRegGov pg, rFlagsReg cr) %{
2810   predicate(UseSVE > 0 &&
2811             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2812             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2813   match(Set dst (MaxReductionV (Binary src1 src2) pg));
2814   effect(TEMP_DEF dst, TEMP tmp, KILL cr);

2815   ins_cost(SVE_COST);
2816   format %{ "sve_reduce_maxL $dst, $src1, $pg, $src2\t# maxL reduction predicated (sve)" %}
2817   ins_encode %{
2818     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2819                            $src1$$Register, as_FloatRegister($src2$$reg),
2820                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2821   %}
2822   ins_pipe(pipe_slow);
2823 %}
2824 
2825 instruct reduce_maxI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2826                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2827   predicate(UseSVE > 0 &&
2828             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2829             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2830             is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
2831   match(Set dst (MaxReductionV (Binary src1 src2) pg));
2832   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2833   ins_cost(3 * SVE_COST);
2834   format %{ "sve_reduce_maxI $dst, $src1, $pg, $src2\t# maxI reduction predicated partial (sve)" %}
2835   ins_encode %{
2836     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2837     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2838     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2839                           Matcher::vector_length(this, $src2));
2840     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2841                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2842     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2843                            $src1$$Register, as_FloatRegister($src2$$reg),
2844                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2845   %}
2846   ins_pipe(pipe_slow);
2847 %}
2848 
2849 instruct reduce_maxL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2850                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2851   predicate(UseSVE > 0 &&
2852             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2853             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2854   match(Set dst (MaxReductionV (Binary src1 src2) pg));
2855   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2856   ins_cost(3 * SVE_COST);
2857   format %{ "sve_reduce_maxL $dst, $src1, $pg, $src2\t# maxL reduction predicated partial (sve)" %}
2858   ins_encode %{
2859     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2860                           Matcher::vector_length(this, $src2));
2861     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2862                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2863     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2864                            $src1$$Register, as_FloatRegister($src2$$reg),
2865                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2866   %}
2867   ins_pipe(pipe_slow);
2868 %}
2869 
2870 instruct reduce_maxF_masked(vRegF dst, vRegF src1, vReg src2, pRegGov pg) %{
2871   predicate(UseSVE > 0 &&
2872             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
2873             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2874   match(Set dst (MaxReductionV (Binary src1 src2) pg));
2875   ins_cost(SVE_COST);
2876   format %{ "sve_reduce_maxF $dst, $src1, $pg, $src2\t# maxF reduction predicated (sve)" %}
2877   ins_encode %{
2878     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
2879     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2880   %}
2881   ins_pipe(pipe_slow);
2882 %}
2883 
2884 instruct reduce_maxD_masked(vRegD dst, vRegD src1, vReg src2, pRegGov pg) %{
2885   predicate(UseSVE > 0 &&
2886             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
2887             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2888   match(Set dst (MaxReductionV (Binary src1 src2) pg));
2889   ins_cost(SVE_COST);
2890   format %{ "sve_reduce_maxD $dst, $src1, $pg, $src2\t# maxD reduction predicated (sve)" %}
2891   ins_encode %{
2892     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
2893     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2894   %}
2895   ins_pipe(pipe_slow);
2896 %}
2897 
2898 instruct reduce_maxF_masked_partial(vRegF dst, vRegF src1, vReg src2, pRegGov pg,
2899                                     pRegGov ptmp, rFlagsReg cr) %{
2900   predicate(UseSVE > 0 &&
2901             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
2902             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2903   match(Set dst (MaxReductionV (Binary src1 src2) pg));
2904   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
2905   ins_cost(3 * SVE_COST);
2906   format %{ "sve_reduce_maxF $dst, $src1, $pg, $src2\t# maxF reduction predicated partial (sve)" %}
2907   ins_encode %{
2908     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2909                           Matcher::vector_length(this, $src2));
2910     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2911                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2912     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S,
2913                as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2914     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2915   %}
2916   ins_pipe(pipe_slow);
2917 %}
2918 
2919 instruct reduce_maxD_masked_partial(vRegD dst, vRegD src1, vReg src2, pRegGov pg,
2920                                     pRegGov ptmp, rFlagsReg cr) %{
2921   predicate(UseSVE > 0 &&
2922             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
2923             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2924   match(Set dst (MaxReductionV (Binary src1 src2) pg));
2925   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
2926   ins_cost(3 * SVE_COST);
2927   format %{ "sve_reduce_maxD $dst, $src1, $pg, $src2\t# maxD reduction predicated partial (sve)" %}
2928   ins_encode %{
2929     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2930                           Matcher::vector_length(this, $src2));
2931     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2932                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2933     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D,
2934                as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2935     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2936   %}
2937   ins_pipe(pipe_slow);
2938 %}
2939 
2940 // vector min reduction
2941 
2942 instruct reduce_minI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
2943   predicate(UseSVE > 0 &&
2944             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2945             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2946             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
2947   match(Set dst (MinReductionV src1 src2));
2948   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2949   ins_cost(SVE_COST);
2950   format %{ "sve_reduce_minI $dst, $src1, $src2\t# minI reduction (sve)" %}
2951   ins_encode %{
2952     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2953     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2954                            $src1$$Register, as_FloatRegister($src2$$reg),
2955                            ptrue, as_FloatRegister($tmp$$reg));
2956   %}
2957   ins_pipe(pipe_slow);
2958 %}
2959 
2960 instruct reduce_minL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
2961   predicate(UseSVE > 0 &&
2962             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2963             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2964   match(Set dst (MinReductionV src1 src2));
2965   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2966   ins_cost(SVE_COST);
2967   format %{ "sve_reduce_minL $dst, $src1, $src2\t# minL reduction (sve)" %}
2968   ins_encode %{
2969     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2970                            $src1$$Register, as_FloatRegister($src2$$reg),
2971                            ptrue, as_FloatRegister($tmp$$reg));
2972   %}
2973   ins_pipe(pipe_slow);
2974 %}
2975 
2976 instruct reduce_minI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2977                              pRegGov ptmp, rFlagsReg cr) %{
2978   predicate(UseSVE > 0 &&
2979             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2980             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2981             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
2982   match(Set dst (MinReductionV src1 src2));
2983   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2984   ins_cost(2 * SVE_COST);
2985   format %{ "sve_reduce_minI $dst, $src1, $src2\t# minI reduction partial (sve)" %}
2986   ins_encode %{
2987     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2988     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2989     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2990                           Matcher::vector_length(this, $src2));
2991     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2992                            $src1$$Register, as_FloatRegister($src2$$reg),
2993                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2994   %}
2995   ins_pipe(pipe_slow);
2996 %}
2997 
2998 instruct reduce_minL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2999                              pRegGov ptmp, rFlagsReg cr) %{
3000   predicate(UseSVE > 0 &&
3001             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
3002             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3003   match(Set dst (MinReductionV src1 src2));
3004   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
3005   ins_cost(2 * SVE_COST);
3006   format %{ "sve_reduce_minL $dst, $src1, $src2\t# minL reduction  partial (sve)" %}
3007   ins_encode %{
3008     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
3009                           Matcher::vector_length(this, $src2));
3010     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
3011                            $src1$$Register, as_FloatRegister($src2$$reg),
3012                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));


3013   %}
3014   ins_pipe(pipe_slow);
3015 %}
3016 
3017 instruct reduce_minF(vRegF dst, vRegF src1, vReg src2) %{
3018   predicate(UseSVE > 0 &&
3019             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3020             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3021   match(Set dst (MinReductionV src1 src2));
3022   ins_cost(INSN_COST);
3023   effect(TEMP_DEF dst);
3024   format %{ "sve_reduce_minF $dst, $src1, $src2\t# minF reduction (sve)" %}

3025   ins_encode %{
3026     __ sve_fminv(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src2$$reg));

3027     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3028   %}
3029   ins_pipe(pipe_slow);
3030 %}
3031 
3032 instruct reduce_minF_partial(vRegF dst, vRegF src1, vReg src2,
3033                              pRegGov ptmp, rFlagsReg cr) %{
3034   predicate(UseSVE > 0 &&
3035             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3036             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3037   match(Set dst (MinReductionV src1 src2));
3038   ins_cost(INSN_COST);
3039   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3040   format %{ "sve_reduce_minF $dst, $src1, $src2\t# minF reduction partial (sve)" %}
3041   ins_encode %{
3042     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
3043                           Matcher::vector_length(this, $src2));
3044     __ sve_fminv(as_FloatRegister($dst$$reg), __ S, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));

3045     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3046   %}
3047   ins_pipe(pipe_slow);
3048 %}
3049 
3050 instruct reduce_minD(vRegD dst, vRegD src1, vReg src2) %{
3051   predicate(UseSVE > 0 &&
3052             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3053             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3054   match(Set dst (MinReductionV src1 src2));
3055   ins_cost(INSN_COST);
3056   effect(TEMP_DEF dst);
3057   format %{ "sve_reduce_minD $dst, $src1, $src2\t# minD reduction (sve)" %}

3058   ins_encode %{
3059     __ sve_fminv(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
3060     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3061   %}
3062   ins_pipe(pipe_slow);
3063 %}
3064 
3065 instruct reduce_minD_partial(vRegD dst, vRegD src1, vReg src2,
3066                              pRegGov ptmp, rFlagsReg cr) %{
3067   predicate(UseSVE > 0 &&
3068             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3069             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3070   match(Set dst (MinReductionV src1 src2));
3071   ins_cost(INSN_COST);
3072   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3073   format %{ "sve_reduce_minD $dst, $src1, $src2\t# minD reduction partial (sve)" %}
3074   ins_encode %{
3075     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
3076                           Matcher::vector_length(this, $src2));
3077     __ sve_fminv(as_FloatRegister($dst$$reg), __ D, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
3078     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3079   %}
3080   ins_pipe(pipe_slow);
3081 %}
3082 
3083 // vector min reduction - predicated
3084 
3085 instruct reduce_minI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp,
3086                            pRegGov pg, rFlagsReg cr) %{
3087   predicate(UseSVE > 0 &&
3088             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
3089             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
3090             is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
3091   match(Set dst (MinReductionV (Binary src1 src2) pg));
3092   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3093   ins_cost(SVE_COST);
3094   format %{ "sve_reduce_minI $dst, $src1, $pg, $src2\t# minI reduction predicated (sve)" %}
3095   ins_encode %{
3096     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
3097     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
3098                            $src1$$Register, as_FloatRegister($src2$$reg),
3099                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
3100   %}
3101   ins_pipe(pipe_slow);
3102 %}
3103 
3104 instruct reduce_minL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp,
3105                           pRegGov pg, rFlagsReg cr) %{
3106   predicate(UseSVE > 0 &&
3107             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
3108             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3109   match(Set dst (MinReductionV (Binary src1 src2) pg));
3110   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3111   ins_cost(SVE_COST);
3112   format %{ "sve_reduce_minL $dst, $src1, $pg, $src2\t# minL reduction predicated (sve)" %}
3113   ins_encode %{
3114     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
3115                            $src1$$Register, as_FloatRegister($src2$$reg),
3116                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
3117   %}
3118   ins_pipe(pipe_slow);
3119 %}
3120 
3121 instruct reduce_minI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
3122                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
3123   predicate(UseSVE > 0 &&
3124             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
3125             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
3126             is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
3127   match(Set dst (MinReductionV (Binary src1 src2) pg));
3128   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
3129   ins_cost(3 * SVE_COST);
3130   format %{ "sve_reduce_minI $dst, $src1, $pg, $src2\t# minI reduction predicated partial (sve)" %}
3131   ins_encode %{
3132     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
3133     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
3134     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
3135                           Matcher::vector_length(this, $src2));
3136     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
3137                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
3138     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
3139                            $src1$$Register, as_FloatRegister($src2$$reg),
3140                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
3141   %}
3142   ins_pipe(pipe_slow);
3143 %}
3144 
3145 instruct reduce_minL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
3146                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
3147   predicate(UseSVE > 0 &&
3148             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
3149             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3150   match(Set dst (MinReductionV (Binary src1 src2) pg));
3151   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
3152   ins_cost(3 * SVE_COST);
3153   format %{ "sve_reduce_minL $dst, $src1, $pg, $src2\t# minL reduction predicated partial (sve)" %}
3154   ins_encode %{
3155     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
3156                           Matcher::vector_length(this, $src2));
3157     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
3158                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
3159     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
3160                            $src1$$Register, as_FloatRegister($src2$$reg),
3161                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
3162   %}
3163   ins_pipe(pipe_slow);
3164 %}
3165 
3166 instruct reduce_minF_masked(vRegF dst, vRegF src1, vReg src2, pRegGov pg) %{
3167   predicate(UseSVE > 0 &&
3168             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3169             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3170   match(Set dst (MinReductionV (Binary src1 src2) pg));
3171   ins_cost(SVE_COST);
3172   format %{ "sve_reduce_minF $dst, $src1, $pg, $src2\t# minF reduction predicated (sve)" %}
3173   ins_encode %{
3174     __ sve_fminv(as_FloatRegister($dst$$reg), __ S, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
3175     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3176   %}
3177   ins_pipe(pipe_slow);
3178 %}
3179 
3180 instruct reduce_minD_masked(vRegD dst, vRegD src1, vReg src2, pRegGov pg) %{
3181   predicate(UseSVE > 0 &&
3182             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3183             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3184   match(Set dst (MinReductionV (Binary src1 src2) pg));
3185   ins_cost(SVE_COST);
3186   format %{ "sve_reduce_minD $dst, $src1, $pg, $src2\t# minD reduction predicated (sve)" %}
3187   ins_encode %{
3188     __ sve_fminv(as_FloatRegister($dst$$reg), __ D, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
3189     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3190   %}
3191   ins_pipe(pipe_slow);
3192 %}
3193 
3194 instruct reduce_minF_masked_partial(vRegF dst, vRegF src1, vReg src2, pRegGov pg,
3195                                     pRegGov ptmp, rFlagsReg cr) %{
3196   predicate(UseSVE > 0 &&
3197             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3198             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3199   match(Set dst (MinReductionV (Binary src1 src2) pg));
3200   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3201   ins_cost(3 * SVE_COST);
3202   format %{ "sve_reduce_minF $dst, $src1, $pg, $src2\t# minF reduction predicated partial (sve)" %}
3203   ins_encode %{
3204     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
3205                           Matcher::vector_length(this, $src2));
3206     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
3207                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
3208     __ sve_fminv(as_FloatRegister($dst$$reg), __ S,
3209                as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
3210     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3211   %}
3212   ins_pipe(pipe_slow);
3213 %}
3214 
3215 instruct reduce_minD_masked_partial(vRegD dst, vRegD src1, vReg src2, pRegGov pg,
3216                                     pRegGov ptmp, rFlagsReg cr) %{
3217   predicate(UseSVE > 0 &&
3218             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3219             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3220   match(Set dst (MinReductionV (Binary src1 src2) pg));
3221   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3222   ins_cost(3 * SVE_COST);
3223   format %{ "sve_reduce_minD $dst, $src1, $pg, $src2\t# minD reduction predicated partial (sve)" %}
3224   ins_encode %{
3225     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
3226                           Matcher::vector_length(this, $src2));
3227     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
3228                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
3229     __ sve_fminv(as_FloatRegister($dst$$reg), __ D,
3230                as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
3231     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3232   %}
3233   ins_pipe(pipe_slow);
3234 %}
3235 
3236 // vector Math.rint, floor, ceil
3237 
3238 instruct vroundD(vReg dst, vReg src, immI rmode) %{
3239   predicate(UseSVE > 0 &&
3240             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
3241   match(Set dst (RoundDoubleModeV src rmode));
3242   format %{ "sve_frint $dst, $src, $rmode\t# vector (sve) (D)" %}
3243   ins_encode %{
3244     switch ($rmode$$constant) {
3245       case RoundDoubleModeNode::rmode_rint:
3246         __ sve_frintn(as_FloatRegister($dst$$reg), __ D,
3247              ptrue, as_FloatRegister($src$$reg));
3248         break;
3249       case RoundDoubleModeNode::rmode_floor:
3250         __ sve_frintm(as_FloatRegister($dst$$reg), __ D,

3640 %}
3641 
3642 instruct vlsrI_imm(vReg dst, vReg src, immI shift) %{
3643   predicate(UseSVE > 0);
3644   match(Set dst (URShiftVI src (RShiftCntV shift)));
3645   ins_cost(SVE_COST);
3646   format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (S)" %}
3647   ins_encode %{
3648     int con = (int)$shift$$constant;
3649     if (con == 0) {
3650       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3651            as_FloatRegister($src$$reg));
3652       return;
3653     }
3654     __ sve_lsr(as_FloatRegister($dst$$reg), __ S,
3655          as_FloatRegister($src$$reg), con);
3656   %}
3657   ins_pipe(pipe_slow);
3658 %}
3659 
3660 instruct vlsrL_imm(vReg dst, vReg src, immI shift) %{
3661   predicate(UseSVE > 0);
3662   match(Set dst (URShiftVL src (RShiftCntV shift)));
3663   ins_cost(SVE_COST);
3664   format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (D)" %}
3665   ins_encode %{
3666     int con = (int)$shift$$constant;
3667     if (con == 0) {
3668       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3669            as_FloatRegister($src$$reg));
3670       return;
3671     }
3672     __ sve_lsr(as_FloatRegister($dst$$reg), __ D,
3673          as_FloatRegister($src$$reg), con);
3674   %}
3675   ins_pipe(pipe_slow);
3676 %}
3677 
3678 instruct vlslB_imm(vReg dst, vReg src, immI shift) %{
3679   predicate(UseSVE > 0);
3680   match(Set dst (LShiftVB src (LShiftCntV shift)));
3681   ins_cost(SVE_COST);
3682   format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (B)" %}
3683   ins_encode %{
3684     int con = (int)$shift$$constant;
3685     if (con >= 8) {
3686       __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3687            as_FloatRegister($src$$reg));
3688       return;
3689     }
3690     __ sve_lsl(as_FloatRegister($dst$$reg), __ B,
3691          as_FloatRegister($src$$reg), con);
3692   %}
3693   ins_pipe(pipe_slow);
3694 %}
3695 
3696 instruct vlslS_imm(vReg dst, vReg src, immI shift) %{
3697   predicate(UseSVE > 0);
3698   match(Set dst (LShiftVS src (LShiftCntV shift)));
3699   ins_cost(SVE_COST);
3700   format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (H)" %}
3701   ins_encode %{
3702     int con = (int)$shift$$constant;
3703     if (con >= 16) {
3704       __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3705            as_FloatRegister($src$$reg));
3706       return;
3707     }
3708     __ sve_lsl(as_FloatRegister($dst$$reg), __ H,
3709          as_FloatRegister($src$$reg), con);
3710   %}
3711   ins_pipe(pipe_slow);
3712 %}
3713 
3714 instruct vlslI_imm(vReg dst, vReg src, immI shift) %{
3715   predicate(UseSVE > 0);
3716   match(Set dst (LShiftVI src (LShiftCntV shift)));
3717   ins_cost(SVE_COST);
3718   format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (S)" %}
3719   ins_encode %{
3720     int con = (int)$shift$$constant;
3721     __ sve_lsl(as_FloatRegister($dst$$reg), __ S,
3722          as_FloatRegister($src$$reg), con);
3723   %}
3724   ins_pipe(pipe_slow);
3725 %}
3726 
3727 instruct vlslL_imm(vReg dst, vReg src, immI shift) %{
3728   predicate(UseSVE > 0);
3729   match(Set dst (LShiftVL src (LShiftCntV shift)));
3730   ins_cost(SVE_COST);
3731   format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (D)" %}
3732   ins_encode %{
3733     int con = (int)$shift$$constant;
3734     __ sve_lsl(as_FloatRegister($dst$$reg), __ D,
3735          as_FloatRegister($src$$reg), con);
3736   %}
3737   ins_pipe(pipe_slow);
3738 %}
3739 
3740 instruct vshiftcntB(vReg dst, iRegIorL2I cnt) %{
3741   predicate(UseSVE > 0 &&
3742             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE));
3743   match(Set dst (LShiftCntV cnt));
3744   match(Set dst (RShiftCntV cnt));
3745   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (B)" %}
3746   ins_encode %{
3747     __ sve_dup(as_FloatRegister($dst$$reg), __ B, as_Register($cnt$$reg));
3748   %}
3749   ins_pipe(pipe_slow);
3750 %}
3751 
3752 instruct vshiftcntS(vReg dst, iRegIorL2I cnt) %{
3753   predicate(UseSVE > 0 &&
3754             (n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
3755             (n->bottom_type()->is_vect()->element_basic_type() == T_CHAR)));
3756   match(Set dst (LShiftCntV cnt));
3757   match(Set dst (RShiftCntV cnt));
3758   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (H)" %}
3759   ins_encode %{
3760     __ sve_dup(as_FloatRegister($dst$$reg), __ H, as_Register($cnt$$reg));
3761   %}
3762   ins_pipe(pipe_slow);
3763 %}
3764 
3765 instruct vshiftcntI(vReg dst, iRegIorL2I cnt) %{
3766   predicate(UseSVE > 0 &&
3767             (n->bottom_type()->is_vect()->element_basic_type() == T_INT));
3768   match(Set dst (LShiftCntV cnt));
3769   match(Set dst (RShiftCntV cnt));
3770   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (S)" %}
3771   ins_encode %{
3772     __ sve_dup(as_FloatRegister($dst$$reg), __ S, as_Register($cnt$$reg));
3773   %}
3774   ins_pipe(pipe_slow);
3775 %}
3776 
3777 instruct vshiftcntL(vReg dst, iRegIorL2I cnt) %{
3778   predicate(UseSVE > 0 &&
3779             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
3780   match(Set dst (LShiftCntV cnt));
3781   match(Set dst (RShiftCntV cnt));
3782   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (D)" %}
3783   ins_encode %{
3784     __ sve_dup(as_FloatRegister($dst$$reg), __ D, as_Register($cnt$$reg));
3785   %}
3786   ins_pipe(pipe_slow);
3787 %}
3788 
3789 // vector shift - predicated
3790 
3791 instruct vasrB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3792   predicate(UseSVE > 0);
3793   match(Set dst_src1 (RShiftVB (Binary dst_src1 src2) pg));
3794   ins_cost(SVE_COST);
3795   format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
3796   ins_encode %{
3797     __ sve_asr(as_FloatRegister($dst_src1$$reg), __ B,
3798             as_PRegister($pg$$reg),
3799             as_FloatRegister($src2$$reg));
3800   %}
3801   ins_pipe(pipe_slow);
3802 %}
3803 
3804 instruct vasrS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3805   predicate(UseSVE > 0);
3806   match(Set dst_src1 (RShiftVS (Binary dst_src1 src2) pg));
3807   ins_cost(SVE_COST);
3808   format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
3809   ins_encode %{
3810     __ sve_asr(as_FloatRegister($dst_src1$$reg), __ H,
3811             as_PRegister($pg$$reg),
3812             as_FloatRegister($src2$$reg));
3813   %}
3814   ins_pipe(pipe_slow);
3815 %}
3816 
3817 instruct vasrI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3818   predicate(UseSVE > 0);
3819   match(Set dst_src1 (RShiftVI (Binary dst_src1 src2) pg));
3820   ins_cost(SVE_COST);
3821   format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
3822   ins_encode %{
3823     __ sve_asr(as_FloatRegister($dst_src1$$reg), __ S,
3824             as_PRegister($pg$$reg),
3825             as_FloatRegister($src2$$reg));
3826   %}
3827   ins_pipe(pipe_slow);
3828 %}
3829 
3830 instruct vasrL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3831   predicate(UseSVE > 0);
3832   match(Set dst_src1 (RShiftVL (Binary dst_src1 src2) pg));
3833   ins_cost(SVE_COST);
3834   format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
3835   ins_encode %{
3836     __ sve_asr(as_FloatRegister($dst_src1$$reg), __ D,
3837             as_PRegister($pg$$reg),
3838             as_FloatRegister($src2$$reg));
3839   %}
3840   ins_pipe(pipe_slow);
3841 %}
3842 
3843 instruct vlslB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3844   predicate(UseSVE > 0);
3845   match(Set dst_src1 (LShiftVB (Binary dst_src1 src2) pg));
3846   ins_cost(SVE_COST);
3847   format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
3848   ins_encode %{
3849     __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ B,
3850             as_PRegister($pg$$reg),
3851             as_FloatRegister($src2$$reg));
3852   %}
3853   ins_pipe(pipe_slow);
3854 %}
3855 
3856 instruct vlslS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3857   predicate(UseSVE > 0);
3858   match(Set dst_src1 (LShiftVS (Binary dst_src1 src2) pg));
3859   ins_cost(SVE_COST);
3860   format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
3861   ins_encode %{
3862     __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ H,
3863             as_PRegister($pg$$reg),
3864             as_FloatRegister($src2$$reg));
3865   %}
3866   ins_pipe(pipe_slow);
3867 %}
3868 
3869 instruct vlslI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3870   predicate(UseSVE > 0);
3871   match(Set dst_src1 (LShiftVI (Binary dst_src1 src2) pg));
3872   ins_cost(SVE_COST);
3873   format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
3874   ins_encode %{
3875     __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ S,
3876             as_PRegister($pg$$reg),
3877             as_FloatRegister($src2$$reg));
3878   %}
3879   ins_pipe(pipe_slow);
3880 %}
3881 
3882 instruct vlslL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3883   predicate(UseSVE > 0);
3884   match(Set dst_src1 (LShiftVL (Binary dst_src1 src2) pg));
3885   ins_cost(SVE_COST);
3886   format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
3887   ins_encode %{
3888     __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ D,
3889             as_PRegister($pg$$reg),
3890             as_FloatRegister($src2$$reg));
3891   %}
3892   ins_pipe(pipe_slow);
3893 %}
3894 
3895 instruct vlsrB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3896   predicate(UseSVE > 0);
3897   match(Set dst_src1 (URShiftVB (Binary dst_src1 src2) pg));
3898   ins_cost(SVE_COST);
3899   format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
3900   ins_encode %{
3901     __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ B,
3902             as_PRegister($pg$$reg),
3903             as_FloatRegister($src2$$reg));
3904   %}
3905   ins_pipe(pipe_slow);
3906 %}
3907 
3908 instruct vlsrS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3909   predicate(UseSVE > 0);
3910   match(Set dst_src1 (URShiftVS (Binary dst_src1 src2) pg));
3911   ins_cost(SVE_COST);
3912   format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
3913   ins_encode %{
3914     __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ H,
3915             as_PRegister($pg$$reg),
3916             as_FloatRegister($src2$$reg));
3917   %}
3918   ins_pipe(pipe_slow);
3919 %}
3920 
3921 instruct vlsrI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3922   predicate(UseSVE > 0);
3923   match(Set dst_src1 (URShiftVI (Binary dst_src1 src2) pg));
3924   ins_cost(SVE_COST);
3925   format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
3926   ins_encode %{
3927     __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ S,
3928             as_PRegister($pg$$reg),
3929             as_FloatRegister($src2$$reg));
3930   %}
3931   ins_pipe(pipe_slow);
3932 %}
3933 
3934 instruct vlsrL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3935   predicate(UseSVE > 0);
3936   match(Set dst_src1 (URShiftVL (Binary dst_src1 src2) pg));
3937   ins_cost(SVE_COST);
3938   format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
3939   ins_encode %{
3940     __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ D,
3941             as_PRegister($pg$$reg),
3942             as_FloatRegister($src2$$reg));
3943   %}
3944   ins_pipe(pipe_slow);
3945 %}
3946 
3947 instruct vasrB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
3948   predicate(UseSVE > 0);
3949   match(Set dst_src (RShiftVB (Binary dst_src (RShiftCntV shift)) pg));
3950   ins_cost(SVE_COST);
3951   format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %}
3952   ins_encode %{
3953     int con = (int)$shift$$constant;
3954     assert(con > 0 && con < 8, "invalid shift immediate");
3955     __ sve_asr(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con);
3956   %}
3957   ins_pipe(pipe_slow);
3958 %}
3959 
3960 instruct vasrS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
3961   predicate(UseSVE > 0);
3962   match(Set dst_src (RShiftVS (Binary dst_src (RShiftCntV shift)) pg));
3963   ins_cost(SVE_COST);
3964   format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %}
3965   ins_encode %{
3966     int con = (int)$shift$$constant;
3967     assert(con > 0 && con < 16, "invalid shift immediate");
3968     __ sve_asr(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con);
3969   %}
3970   ins_pipe(pipe_slow);
3971 %}
3972 
3973 instruct vasrI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
3974   predicate(UseSVE > 0);
3975   match(Set dst_src (RShiftVI (Binary dst_src (RShiftCntV shift)) pg));
3976   ins_cost(SVE_COST);
3977   format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %}
3978   ins_encode %{
3979     int con = (int)$shift$$constant;
3980     assert(con > 0 && con < 32, "invalid shift immediate");
3981     __ sve_asr(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con);
3982   %}
3983   ins_pipe(pipe_slow);
3984 %}
3985 
3986 instruct vasrL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
3987   predicate(UseSVE > 0);
3988   match(Set dst_src (RShiftVL (Binary dst_src (RShiftCntV shift)) pg));
3989   ins_cost(SVE_COST);
3990   format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %}
3991   ins_encode %{
3992     int con = (int)$shift$$constant;
3993     assert(con > 0 && con < 64, "invalid shift immediate");
3994     __ sve_asr(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con);





3995   %}
3996   ins_pipe(pipe_slow);
3997 %}
3998 
3999 instruct vlsrB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4000   predicate(UseSVE > 0);
4001   match(Set dst_src (URShiftVB (Binary dst_src (RShiftCntV shift)) pg));
4002   ins_cost(SVE_COST);
4003   format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %}
4004   ins_encode %{
4005     int con = (int)$shift$$constant;
4006     assert(con > 0 && con < 8, "invalid shift immediate");
4007     __ sve_lsr(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con);





4008   %}
4009   ins_pipe(pipe_slow);
4010 %}
4011 
4012 instruct vlsrS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4013   predicate(UseSVE > 0);
4014   match(Set dst_src (URShiftVS (Binary dst_src (RShiftCntV shift)) pg));
4015   ins_cost(SVE_COST);
4016   format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %}
4017   ins_encode %{
4018     int con = (int)$shift$$constant;
4019     assert(con > 0 && con < 16, "invalid shift immediate");
4020     __ sve_lsr(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con);





4021   %}
4022   ins_pipe(pipe_slow);
4023 %}
4024 
4025 instruct vlsrI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4026   predicate(UseSVE > 0);
4027   match(Set dst_src (URShiftVI (Binary dst_src (RShiftCntV shift)) pg));
4028   ins_cost(SVE_COST);
4029   format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %}
4030   ins_encode %{
4031     int con = (int)$shift$$constant;
4032     assert(con > 0 && con < 32, "invalid shift immediate");
4033     __ sve_lsr(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con);
4034   %}
4035   ins_pipe(pipe_slow);
4036 %}
4037 
4038 instruct vlsrL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4039   predicate(UseSVE > 0);
4040   match(Set dst_src (URShiftVL (Binary dst_src (RShiftCntV shift)) pg));
4041   ins_cost(SVE_COST);
4042   format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %}
4043   ins_encode %{
4044     int con = (int)$shift$$constant;
4045     assert(con > 0 && con < 64, "invalid shift immediate");
4046     __ sve_lsr(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con);
4047   %}
4048   ins_pipe(pipe_slow);
4049 %}
4050 
4051 instruct vlslB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4052   predicate(UseSVE > 0);
4053   match(Set dst_src (LShiftVB (Binary dst_src (LShiftCntV shift)) pg));
4054   ins_cost(SVE_COST);
4055   format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %}

4056   ins_encode %{
4057     int con = (int)$shift$$constant;
4058     assert(con >= 0 && con < 8, "invalid shift immediate");
4059     __ sve_lsl(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con);
4060   %}
4061   ins_pipe(pipe_slow);
4062 %}
4063 
4064 instruct vlslS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4065   predicate(UseSVE > 0);
4066   match(Set dst_src (LShiftVS (Binary dst_src (LShiftCntV shift)) pg));
4067   ins_cost(SVE_COST);
4068   format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %}


4069   ins_encode %{
4070     int con = (int)$shift$$constant;
4071     assert(con >= 0 && con < 16, "invalid shift immediate");
4072     __ sve_lsl(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con);
4073   %}
4074   ins_pipe(pipe_slow);
4075 %}
4076 
4077 instruct vlslI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4078   predicate(UseSVE > 0);
4079   match(Set dst_src (LShiftVI (Binary dst_src (LShiftCntV shift)) pg));
4080   ins_cost(SVE_COST);
4081   format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %}

4082   ins_encode %{
4083     int con = (int)$shift$$constant;
4084     assert(con >= 0 && con < 32, "invalid shift immediate");
4085     __ sve_lsl(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con);
4086   %}
4087   ins_pipe(pipe_slow);
4088 %}
4089 
4090 instruct vlslL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4091   predicate(UseSVE > 0);
4092   match(Set dst_src (LShiftVL (Binary dst_src (LShiftCntV shift)) pg));
4093   ins_cost(SVE_COST);
4094   format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %}

4095   ins_encode %{
4096     int con = (int)$shift$$constant;
4097     assert(con >= 0 && con < 64, "invalid shift immediate");
4098     __ sve_lsl(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con);
4099   %}
4100   ins_pipe(pipe_slow);
4101 %}
4102 
4103 // vector sqrt
4104 
4105 instruct vsqrtF(vReg dst, vReg src) %{
4106   predicate(UseSVE > 0 &&
4107             !n->as_Vector()->is_predicated_vector());
4108   match(Set dst (SqrtVF src));
4109   ins_cost(SVE_COST);
4110   format %{ "sve_fsqrt $dst, $src\t# vector (sve) (S)" %}
4111   ins_encode %{
4112     __ sve_fsqrt(as_FloatRegister($dst$$reg), __ S,
4113          ptrue, as_FloatRegister($src$$reg));
4114   %}
4115   ins_pipe(pipe_slow);
4116 %}
4117 
4118 instruct vsqrtD(vReg dst, vReg src) %{
4119   predicate(UseSVE > 0 &&
4120             !n->as_Vector()->is_predicated_vector());
4121   match(Set dst (SqrtVD src));
4122   ins_cost(SVE_COST);
4123   format %{ "sve_fsqrt $dst, $src\t# vector (sve) (D)" %}
4124   ins_encode %{
4125     __ sve_fsqrt(as_FloatRegister($dst$$reg), __ D,
4126          ptrue, as_FloatRegister($src$$reg));
4127   %}
4128   ins_pipe(pipe_slow);
4129 %}
4130 
4131 // vector sqrt - predicated
4132 
4133 instruct vsqrtF_masked(vReg dst_src, pRegGov pg) %{
4134   predicate(UseSVE > 0);
4135   match(Set dst_src (SqrtVF dst_src pg));
4136   ins_cost(SVE_COST);
4137   format %{ "sve_fsqrt $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
4138   ins_encode %{
4139     __ sve_fsqrt(as_FloatRegister($dst_src$$reg), __ S,
4140             as_PRegister($pg$$reg),
4141             as_FloatRegister($dst_src$$reg));
4142   %}
4143   ins_pipe(pipe_slow);
4144 %}
4145 
4146 instruct vsqrtD_masked(vReg dst_src, pRegGov pg) %{
4147   predicate(UseSVE > 0);
4148   match(Set dst_src (SqrtVD dst_src pg));
4149   ins_cost(SVE_COST);
4150   format %{ "sve_fsqrt $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
4151   ins_encode %{
4152     __ sve_fsqrt(as_FloatRegister($dst_src$$reg), __ D,
4153             as_PRegister($pg$$reg),
4154             as_FloatRegister($dst_src$$reg));
4155   %}
4156   ins_pipe(pipe_slow);
4157 %}
4158 
4159 // vector sub
4160 
4161 instruct vsubB(vReg dst, vReg src1, vReg src2) %{
4162   predicate(UseSVE > 0);
4163   match(Set dst (SubVB src1 src2));
4164   ins_cost(SVE_COST);
4165   format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (B)" %}
4166   ins_encode %{
4167     __ sve_sub(as_FloatRegister($dst$$reg), __ B,
4168          as_FloatRegister($src1$$reg),
4169          as_FloatRegister($src2$$reg));
4170   %}
4171   ins_pipe(pipe_slow);
4172 %}
4173 
4174 instruct vsubS(vReg dst, vReg src1, vReg src2) %{
4175   predicate(UseSVE > 0);
4176   match(Set dst (SubVS src1 src2));
4177   ins_cost(SVE_COST);
4178   format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (H)" %}

4219     __ sve_fsub(as_FloatRegister($dst$$reg), __ S,
4220          as_FloatRegister($src1$$reg),
4221          as_FloatRegister($src2$$reg));
4222   %}
4223   ins_pipe(pipe_slow);
4224 %}
4225 
4226 instruct vsubD(vReg dst, vReg src1, vReg src2) %{
4227   predicate(UseSVE > 0);
4228   match(Set dst (SubVD src1 src2));
4229   ins_cost(SVE_COST);
4230   format %{ "sve_fsub $dst, $src1, $src2\t # vector (sve) (D)" %}
4231   ins_encode %{
4232     __ sve_fsub(as_FloatRegister($dst$$reg), __ D,
4233          as_FloatRegister($src1$$reg),
4234          as_FloatRegister($src2$$reg));
4235   %}
4236   ins_pipe(pipe_slow);
4237 %}
4238 
4239 // vector sub - predicated
4240 
4241 instruct vsubB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4242   predicate(UseSVE > 0);
4243   match(Set dst_src1 (SubVB (Binary dst_src1 src2) pg));
4244   ins_cost(SVE_COST);
4245   format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}

4246   ins_encode %{
4247     __ sve_sub(as_FloatRegister($dst_src1$$reg), __ B,
4248             as_PRegister($pg$$reg),
4249             as_FloatRegister($src2$$reg));
4250   %}
4251   ins_pipe(pipe_slow);
4252 %}
4253 
4254 instruct vsubS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4255   predicate(UseSVE > 0);
4256   match(Set dst_src1 (SubVS (Binary dst_src1 src2) pg));
4257   ins_cost(SVE_COST);
4258   format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
4259   ins_encode %{
4260     __ sve_sub(as_FloatRegister($dst_src1$$reg), __ H,
4261             as_PRegister($pg$$reg),
4262             as_FloatRegister($src2$$reg));
4263   %}
4264   ins_pipe(pipe_slow);
4265 %}
4266 
4267 instruct vsubI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4268   predicate(UseSVE > 0);
4269   match(Set dst_src1 (SubVI (Binary dst_src1 src2) pg));


4270   ins_cost(SVE_COST);
4271   format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
4272   ins_encode %{
4273     __ sve_sub(as_FloatRegister($dst_src1$$reg), __ S,
4274             as_PRegister($pg$$reg),
4275             as_FloatRegister($src2$$reg));
4276   %}
4277   ins_pipe(pipe_slow);
4278 %}
4279 
4280 instruct vsubL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4281   predicate(UseSVE > 0);
4282   match(Set dst_src1 (SubVL (Binary dst_src1 src2) pg));
4283   ins_cost(SVE_COST);
4284   format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}



4285   ins_encode %{
4286     __ sve_sub(as_FloatRegister($dst_src1$$reg), __ D,
4287             as_PRegister($pg$$reg),
4288             as_FloatRegister($src2$$reg));
4289   %}
4290   ins_pipe(pipe_slow);
4291 %}
4292 
4293 instruct vsubF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4294   predicate(UseSVE > 0);
4295   match(Set dst_src1 (SubVF (Binary dst_src1 src2) pg));
4296   ins_cost(SVE_COST);
4297   format %{ "sve_fsub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}




4298   ins_encode %{
4299     __ sve_fsub(as_FloatRegister($dst_src1$$reg), __ S,
4300             as_PRegister($pg$$reg),
4301             as_FloatRegister($src2$$reg));
4302   %}
4303   ins_pipe(pipe_slow);
4304 %}
4305 
4306 instruct vsubD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4307   predicate(UseSVE > 0);
4308   match(Set dst_src1 (SubVD (Binary dst_src1 src2) pg));
4309   ins_cost(SVE_COST);
4310   format %{ "sve_fsub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}




4311   ins_encode %{
4312     __ sve_fsub(as_FloatRegister($dst_src1$$reg), __ D,
4313             as_PRegister($pg$$reg),
4314             as_FloatRegister($src2$$reg));
4315   %}
4316   ins_pipe(pipe_slow);
4317 %}
4318 
4319 // ------------------------------ Vector mask cast --------------------------
4320 
4321 instruct vmaskcast(pRegGov dst_src) %{
4322   predicate(UseSVE > 0 &&
4323             n->bottom_type()->is_vect()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
4324             n->bottom_type()->is_vect()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
4325   match(Set dst_src (VectorMaskCast dst_src));
4326   ins_cost(0);
4327   format %{ "vmaskcast $dst_src\t# empty (sve)" %}
4328   ins_encode %{
4329     // empty





4330   %}
4331   ins_pipe(pipe_class_empty);
4332 %}
4333 
4334 instruct vmaskcast_extend(pRegGov dst, pReg src)
4335 %{
4336   predicate(UseSVE > 0 &&
4337             (Matcher::vector_length_in_bytes(n) == 2 * Matcher::vector_length_in_bytes(n->in(1)) ||
4338              Matcher::vector_length_in_bytes(n) == 4 * Matcher::vector_length_in_bytes(n->in(1)) ||
4339              Matcher::vector_length_in_bytes(n) == 8 * Matcher::vector_length_in_bytes(n->in(1))));
4340   match(Set dst (VectorMaskCast src));
4341   ins_cost(SVE_COST * 3);
4342   format %{ "sve_vmaskcast_extend  $dst, $src\t# extend predicate $src" %}
4343   ins_encode %{
4344     __ sve_vmaskcast_extend(as_PRegister($dst$$reg), as_PRegister($src$$reg),
4345                             Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
4346   %}
4347   ins_pipe(pipe_slow);
4348 %}
4349 
4350 instruct vmaskcast_narrow(pRegGov dst, pReg src)
4351 %{
4352   predicate(UseSVE > 0 &&
4353             (Matcher::vector_length_in_bytes(n) * 2 == Matcher::vector_length_in_bytes(n->in(1)) ||
4354              Matcher::vector_length_in_bytes(n) * 4 == Matcher::vector_length_in_bytes(n->in(1)) ||
4355              Matcher::vector_length_in_bytes(n) * 8 == Matcher::vector_length_in_bytes(n->in(1))));
4356   match(Set dst (VectorMaskCast src));
4357   ins_cost(SVE_COST * 3);
4358   format %{ "sve_vmaskcast_narrow  $dst, $src\t# narrow predicate $src" %}
4359   ins_encode %{
4360     __ sve_vmaskcast_narrow(as_PRegister($dst$$reg), as_PRegister($src$$reg),
4361                             Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
4362   %}
4363   ins_pipe(pipe_slow);
4364 %}
4365 
4366 // ------------------------------ Vector cast -------------------------------
4367 
4368 instruct vcvtBtoX_extend(vReg dst, vReg src)
4369 %{
4370   predicate(UseSVE > 0);
4371   match(Set dst (VectorCastB2X src));

4372   ins_cost(2 * SVE_COST);
4373   format %{ "sve_vectorcast_b2x  $dst, $src\t# convert B to X vector (extend)" %}

4374   ins_encode %{
4375     BasicType to_bt = Matcher::vector_element_basic_type(this);
4376     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4377     __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ B);
4378     if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
4379       __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
4380     }
4381   %}
4382   ins_pipe(pipe_slow);
4383 %}
4384 
4385 instruct vcvtStoB(vReg dst, vReg src, vReg tmp)
4386 %{
4387   predicate(UseSVE > 0 &&
4388             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
4389   match(Set dst (VectorCastS2X src));
4390   effect(TEMP tmp);
4391   ins_cost(2 * SVE_COST);
4392   format %{ "sve_vectorcast_s2b  $dst, $src\t# convert H to B vector" %}

4393   ins_encode %{
4394     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
4395                          as_FloatRegister($src$$reg), __ H, as_FloatRegister($tmp$$reg));
4396   %}
4397   ins_pipe(pipe_slow);
4398 %}
4399 
4400 instruct vcvtStoX_extend(vReg dst, vReg src)
4401 %{
4402   predicate(UseSVE > 0 &&
4403             type2aelembytes(Matcher::vector_element_basic_type(n)) > 2);
4404   match(Set dst (VectorCastS2X src));
4405   ins_cost(2 * SVE_COST);
4406   format %{ "sve_vectorcast_s2x  $dst, $src\t# convert H to X vector (extend)" %}


4407   ins_encode %{
4408     BasicType to_bt = Matcher::vector_element_basic_type(this);
4409     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4410     __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ H);
4411     if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
4412       __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
4413     }
4414   %}
4415   ins_pipe(pipe_slow);
4416 %}
4417 
4418 instruct vcvtItoB(vReg dst, vReg src, vReg tmp)
4419 %{
4420   predicate(UseSVE > 0 &&
4421             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
4422   match(Set dst (VectorCastI2X src));
4423   effect(TEMP_DEF dst, TEMP tmp);
4424   ins_cost(3 * SVE_COST);
4425   format %{ "sve_vectorcast_i2b  $dst, $src\t# convert I to B vector" %}


4426   ins_encode %{
4427     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
4428                          as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));

4429   %}
4430   ins_pipe(pipe_slow);
4431 %}
4432 
4433 instruct vcvtItoS(vReg dst, vReg src, vReg tmp)
4434 %{
4435   predicate(UseSVE > 0 &&
4436             n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
4437   match(Set dst (VectorCastI2X src));
4438   effect(TEMP tmp);
4439   ins_cost(2 * SVE_COST);
4440   format %{ "sve_vectorcast_i2s $dst, $src\t# convert I to H vector" %}

4441   ins_encode %{
4442     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ H,
4443                          as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
4444   %}
4445   ins_pipe(pipe_slow);
4446 %}
4447 
4448 instruct vcvtItoL(vReg dst, vReg src)
4449 %{
4450   predicate(UseSVE > 0 &&
4451             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
4452   match(Set dst (VectorCastI2X src));
4453   ins_cost(SVE_COST);
4454   format %{ "sve_vectorcast_i2l  $dst, $src\t# convert I to L vector" %}
4455   ins_encode %{
4456     __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
4457   %}
4458   ins_pipe(pipe_slow);
4459 %}
4460 
4461 instruct vcvtItoF(vReg dst, vReg src)
4462 %{
4463   predicate(UseSVE > 0 &&
4464             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4465   match(Set dst (VectorCastI2X src));
4466   ins_cost(SVE_COST);
4467   format %{ "sve_vectorcast_i2f  $dst, $src\t# convert I to F vector" %}
4468   ins_encode %{
4469     __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
4470   %}
4471   ins_pipe(pipe_slow);
4472 %}
4473 
4474 instruct vcvtItoD(vReg dst, vReg src)
4475 %{
4476   predicate(UseSVE > 0 &&
4477             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
4478   match(Set dst (VectorCastI2X src));
4479   ins_cost(2 * SVE_COST);
4480   format %{ "sve_vectorcast_i2d  $dst, $src\t# convert I to D vector" %}

4481   ins_encode %{
4482     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
4483     __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ D);
4484   %}
4485   ins_pipe(pipe_slow);
4486 %}
4487 
4488 instruct vcvtLtoX_narrow(vReg dst, vReg src, vReg tmp)




















4489 %{
4490   predicate(UseSVE > 0 && is_integral_type(Matcher::vector_element_basic_type(n)));

4491   match(Set dst (VectorCastL2X src));
4492   effect(TEMP_DEF dst, TEMP tmp);


















4493   ins_cost(2 * SVE_COST);
4494   format %{ "sve_vectorcast_l2x  $dst, $src\t# convert L to B/H/S vector (narrow)" %}

4495   ins_encode %{
4496     BasicType to_bt = Matcher::vector_element_basic_type(this);
4497     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4498     __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
4499                          as_FloatRegister($src$$reg), __ D, as_FloatRegister($tmp$$reg));
4500   %}
4501   ins_pipe(pipe_slow);
4502 %}
4503 
4504 instruct vcvtLtoF(vReg dst, vReg src, vReg tmp)
4505 %{
4506   predicate(UseSVE > 0 &&
4507             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4508   match(Set dst (VectorCastL2X src));
4509   effect(TEMP_DEF dst, TEMP tmp);
4510   ins_cost(3 * SVE_COST);
4511   format %{ "sve_vectorcast_l2f  $dst, $src\t# convert L to F vector" %}


4512   ins_encode %{
4513     __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
4514     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
4515                          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
4516 
4517   %}
4518   ins_pipe(pipe_slow);
4519 %}
4520 
4521 instruct vcvtLtoD(vReg dst, vReg src)
4522 %{
4523   predicate(UseSVE > 0 &&
4524             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
4525   match(Set dst (VectorCastL2X src));
4526   ins_cost(SVE_COST);
4527   format %{ "sve_vectorcast_l2d  $dst, $src\t# convert L to D vector" %}
4528   ins_encode %{
4529     __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
4530   %}
4531   ins_pipe(pipe_slow);
4532 %}
4533 
4534 instruct vcvtFtoX_narrow(vReg dst, vReg src, vReg tmp)




















4535 %{
4536   predicate(UseSVE > 0 &&
4537             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4538              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT));
4539   match(Set dst (VectorCastF2X src));
4540   effect(TEMP_DEF dst, TEMP tmp);
4541   ins_cost(3 * SVE_COST);
4542   format %{ "sve_vectorcast_f2x  $dst, $src\t# convert F to B/H vector" %}































4543   ins_encode %{
4544     BasicType to_bt = Matcher::vector_element_basic_type(this);
4545     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4546     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
4547     __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
4548                          as_FloatRegister($dst$$reg), __ S, as_FloatRegister($tmp$$reg));














4549   %}
4550   ins_pipe(pipe_slow);
4551 %}
4552 
4553 instruct vcvtFtoX_extend(vReg dst, vReg src)
4554 %{
4555   predicate(UseSVE > 0 &&
4556             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
4557              n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
4558   match(Set dst (VectorCastF2X src));
4559   ins_cost(SVE_COST);
4560   format %{ "sve_vectorcast_f2x  $dst, $src\t# convert F to I/L vector" %}




4561   ins_encode %{
4562     BasicType to_bt = Matcher::vector_element_basic_type(this);
4563     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
4564     if (to_bt == T_LONG) {
4565       __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($dst$$reg), __ S);
4566     }
4567   %}
4568   ins_pipe(pipe_slow);
4569 %}
4570 
4571 instruct vcvtFtoD(vReg dst, vReg src)
4572 %{
4573   predicate(UseSVE > 0 &&
4574             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
4575   match(Set dst (VectorCastF2X src));
4576   ins_cost(2 * SVE_COST);
4577   format %{ "sve_vectorcast_f2d  $dst, $dst\t# convert F to D vector" %}




4578   ins_encode %{
4579     __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
4580     __ sve_fcvt(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);


4581   %}
4582   ins_pipe(pipe_slow);
4583 %}
4584 
4585 instruct vcvtDtoX_narrow(vReg dst, vReg src, vReg tmp)
4586 %{
4587   predicate(UseSVE > 0 &&
4588             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4589              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
4590              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4591   match(Set dst (VectorCastD2X src));
4592   effect(TEMP_DEF dst, TEMP tmp);
4593   ins_cost(3 * SVE_COST);
4594   format %{ "sve_vectorcast_d2x  $dst, $src\t# convert D to X vector (narrow)" %}


4595   ins_encode %{
4596     BasicType to_bt = Matcher::vector_element_basic_type(this);
4597     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4598     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
4599     __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
4600                          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
4601   %}
4602   ins_pipe(pipe_slow);
4603 %}
4604 
4605 instruct vcvtDtoL(vReg dst, vReg src)
4606 %{
4607   predicate(UseSVE > 0 &&
4608             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
4609   match(Set dst (VectorCastD2X src));
4610   ins_cost(SVE_COST);
4611   format %{ "sve_vectorcast_d2l  $dst, $src\t# convert D to L vector" %}
4612   ins_encode %{
4613     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
4614   %}
4615   ins_pipe(pipe_slow);
4616 %}
4617 
4618 instruct vcvtDtoF(vReg dst, vReg src, vReg tmp)
4619 %{
4620   predicate(UseSVE > 0 &&
4621             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4622   match(Set dst (VectorCastD2X src));
4623   effect(TEMP_DEF dst, TEMP tmp);
4624   ins_cost(3 * SVE_COST);
4625   format %{ "sve_vectorcast_d2f  $dst, S, $dst\t# convert D to F vector" %}


4626   ins_encode %{
4627     __ sve_fcvt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
4628     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
4629                          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
4630   %}
4631   ins_pipe(pipe_slow);
4632 %}
4633 
4634 // ------------------------------ Vector extract ---------------------------------
4635 
4636 instruct extractB(iRegINoSp dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
4637 %{
4638   predicate(UseSVE > 0);
4639   match(Set dst (ExtractB src idx));
4640   effect(TEMP pgtmp, KILL cr);
4641   ins_cost(2 * SVE_COST);
4642   format %{ "sve_extract $dst, B, $pgtmp, $src, $idx\n\t"
4643             "sbfmw $dst, $dst, 0U, 7U\t# extract from vector(B)" %}
4644   ins_encode %{
4645     __ sve_extract(as_Register($dst$$reg), __ B, as_PRegister($pgtmp$$reg),
4646                    as_FloatRegister($src$$reg), (int)($idx$$constant));
4647     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 7U);
4648   %}
4649   ins_pipe(pipe_slow);
4650 %}
4651 
4652 instruct extractS(iRegINoSp dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
4653 %{
4654   predicate(UseSVE > 0);
4655   match(Set dst (ExtractS src idx));
4656   effect(TEMP pgtmp, KILL cr);
4657   ins_cost(2 * SVE_COST);
4658   format %{ "sve_extract $dst, H, $pgtmp, $src, $idx\n\t"
4659             "sbfmw $dst, $dst, 0U, 15U\t# extract from vector(S)" %}
4660   ins_encode %{
4661     __ sve_extract(as_Register($dst$$reg), __ H, as_PRegister($pgtmp$$reg),
4662                    as_FloatRegister($src$$reg), (int)($idx$$constant));
4663     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
4664   %}
4665   ins_pipe(pipe_slow);
4666 %}
4667 
4668 
4669 instruct extractI(iRegINoSp dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
4670 %{
4671   predicate(UseSVE > 0);
4672   match(Set dst (ExtractI src idx));
4673   effect(TEMP pgtmp, KILL cr);
4674   ins_cost(2 * SVE_COST);
4675   format %{ "sve_extract $dst, S, $pgtmp, $src, $idx\t# extract from vector(I)" %}
4676   ins_encode %{
4677     __ sve_extract(as_Register($dst$$reg), __ S, as_PRegister($pgtmp$$reg),
4678                    as_FloatRegister($src$$reg), (int)($idx$$constant));
4679   %}
4680   ins_pipe(pipe_slow);
4681 %}
4682 
4683 instruct extractL(iRegLNoSp dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
4684 %{
4685   predicate(UseSVE > 0);
4686   match(Set dst (ExtractL src idx));
4687   effect(TEMP pgtmp, KILL cr);
4688   ins_cost(2 * SVE_COST);
4689   format %{ "sve_extract $dst, D, $pgtmp, $src, $idx\t# extract from vector(L)" %}
4690   ins_encode %{
4691     __ sve_extract(as_Register($dst$$reg), __ D, as_PRegister($pgtmp$$reg),
4692                    as_FloatRegister($src$$reg), (int)($idx$$constant));
4693   %}
4694   ins_pipe(pipe_slow);
4695 %}
4696 
4697 instruct extractF(vRegF dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
4698 %{
4699   predicate(UseSVE > 0);
4700   match(Set dst (ExtractF src idx));
4701   effect(TEMP pgtmp, KILL cr);
4702   ins_cost(2 * SVE_COST);
4703   format %{ "sve_extract $dst, S, $pgtmp, $src, $idx\t# extract from vector(F)" %}
4704   ins_encode %{
4705     __ sve_extract(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg),
4706                    as_FloatRegister($src$$reg), (int)($idx$$constant));
4707   %}
4708   ins_pipe(pipe_slow);
4709 %}
4710 
4711 instruct extractD(vRegD dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
4712 %{
4713   predicate(UseSVE > 0);
4714   match(Set dst (ExtractD src idx));
4715   effect(TEMP pgtmp, KILL cr);
4716   ins_cost(2 * SVE_COST);
4717   format %{ "sve_extract $dst, D, $pgtmp, $src, $idx\t# extract from vector(D)" %}
4718   ins_encode %{
4719     __ sve_extract(as_FloatRegister($dst$$reg), __ D, as_PRegister($pgtmp$$reg),
4720                    as_FloatRegister($src$$reg), (int)($idx$$constant));
4721   %}
4722   ins_pipe(pipe_slow);
4723 %}
4724 
4725 // ------------------------------- VectorTest ----------------------------------
4726 
4727 instruct vtest_alltrue(iRegINoSp dst, pRegGov src1, pRegGov src2, pReg ptmp, rFlagsReg cr)
4728 %{
4729   predicate(UseSVE > 0 &&
4730             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
4731             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
4732   match(Set dst (VectorTest src1 src2));
4733   effect(TEMP ptmp, KILL cr);
4734   ins_cost(SVE_COST);
4735   format %{ "sve_eors $ptmp, $src1, $src2\t# $src2 is all true mask\n"
4736             "csetw $dst, EQ\t# VectorTest (sve) - alltrue" %}
4737   ins_encode %{
4738     __ sve_eors(as_PRegister($ptmp$$reg), ptrue,
4739                 as_PRegister($src1$$reg), as_PRegister($src2$$reg));



4740     __ csetw(as_Register($dst$$reg), Assembler::EQ);
4741   %}
4742   ins_pipe(pipe_slow);
4743 %}
4744 
4745 instruct vtest_anytrue(iRegINoSp dst, pRegGov src1, pRegGov src2, rFlagsReg cr)
4746 %{
4747   predicate(UseSVE > 0 &&
4748             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
4749             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
4750   match(Set dst (VectorTest src1 src2));
4751   effect(KILL cr);
4752   ins_cost(SVE_COST);
4753   format %{ "sve_ptest $src1\n\t"
4754             "csetw $dst, NE\t# VectorTest (sve) - anytrue" %}
4755   ins_encode %{
4756     // "src2" is not used for sve.
4757     __ sve_ptest(ptrue, as_PRegister($src1$$reg));



4758     __ csetw(as_Register($dst$$reg), Assembler::NE);
4759   %}
4760   ins_pipe(pipe_slow);
4761 %}
4762 
4763 instruct vtest_alltrue_partial(iRegINoSp dst, pRegGov src1, pRegGov src2, pRegGov ptmp, rFlagsReg cr)
4764 %{
4765   predicate(UseSVE > 0 &&
4766             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
4767             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
4768   match(Set dst (VectorTest src1 src2));
4769   effect(TEMP ptmp, KILL cr);
4770   ins_cost(SVE_COST);
4771   format %{ "vtest_alltrue_partial $dst, $src1, $src2\t# VectorTest partial (sve) - alltrue" %}
4772   ins_encode %{

4773     BasicType bt = Matcher::vector_element_basic_type(this, $src1);
4774     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
4775     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size,
4776                           Matcher::vector_length(this, $src1));
4777     __ sve_eors(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
4778           as_PRegister($src1$$reg), as_PRegister($src2$$reg));
4779     __ csetw(as_Register($dst$$reg), Assembler::EQ);
4780   %}
4781   ins_pipe(pipe_slow);
4782 %}
4783 
4784 instruct vtest_anytrue_partial(iRegINoSp dst, pRegGov src1, pRegGov src2, pRegGov ptmp, rFlagsReg cr)
4785 %{
4786   predicate(UseSVE > 0 &&
4787             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
4788             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
4789   match(Set dst (VectorTest src1 src2));
4790   effect(TEMP ptmp, KILL cr);
4791   ins_cost(SVE_COST);
4792   format %{ "vtest_anytrue_partial $dst, $src1, $src2\t# VectorTest partial (sve) - anytrue" %}
4793   ins_encode %{

4794     BasicType bt = Matcher::vector_element_basic_type(this, $src1);
4795     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
4796     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size,
4797                           Matcher::vector_length(this, $src1));
4798     __ sve_ands(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
4799           as_PRegister($src1$$reg), as_PRegister($src2$$reg));
4800     __ csetw(as_Register($dst$$reg), Assembler::NE);
4801   %}
4802   ins_pipe(pipe_slow);
4803 %}
4804 
4805 // ------------------------------ Vector insert ---------------------------------
4806 
4807 instruct insertI_small(vReg dst, vReg src, iRegIorL2I val, immI idx, pRegGov pgtmp, rFlagsReg cr)
4808 %{
4809   predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
4810             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4811              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
4812              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4813   match(Set dst (VectorInsert (Binary src val) idx));
4814   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
4815   ins_cost(4 * SVE_COST);
4816   format %{ "sve_index $dst, -16, 1\t# (B/H/S)\n\t"
4817             "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
4818             "sve_orr $dst, $src, $src\n\t"
4819             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
4820   ins_encode %{
4821     BasicType bt = Matcher::vector_element_basic_type(this, $src);
4822     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
4823     __ sve_index(as_FloatRegister($dst$$reg), size, -16, 1);
4824     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
4825                as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
4826     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4827     __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
4828   %}
4829   ins_pipe(pipe_slow);
4830 %}
4831 
4832 instruct insertF_small(vReg dst, vReg src, vRegF val, immI idx, pRegGov pgtmp, rFlagsReg cr)
4833 %{
4834   predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
4835             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4836   match(Set dst (VectorInsert (Binary src val) idx));
4837   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
4838   ins_cost(4 * SVE_COST);
4839   format %{ "sve_index $dst, S, -16, 1\n\t"
4840             "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
4841             "sve_orr $dst, $src, $src\n\t"
4842             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
4843   ins_encode %{
4844     __ sve_index(as_FloatRegister($dst$$reg), __ S, -16, 1);
4845     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
4846                as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
4847     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4848     __ sve_cpy(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
4849   %}
4850   ins_pipe(pipe_slow);
4851 %}
4852 
4853 instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr)
4854 %{
4855   predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
4856             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4857              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
4858              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4859   match(Set dst (VectorInsert (Binary src val) idx));
4860   effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
4861   ins_cost(5 * SVE_COST);
4862   format %{ "sve_index $tmp1, 0, 1\t# (B/H/S)\n\t"
4863             "sve_dup $dst, $idx\t# (B/H/S)\n\t"
4864             "sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
4865             "sve_orr $dst, $src, $src\n\t"
4866             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
4867   ins_encode %{
4868     BasicType bt = Matcher::vector_element_basic_type(this, $src);
4869     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
4870     __ sve_index(as_FloatRegister($tmp1$$reg), size, 0, 1);
4871     __ sve_dup(as_FloatRegister($dst$$reg), size, (int)($idx$$constant));
4872     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
4873                as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
4874     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4875     __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
4876   %}
4877   ins_pipe(pipe_slow);
4878 %}
4879 
4880 instruct insertL(vReg dst, vReg src, iRegL val, immI idx, pRegGov pgtmp, rFlagsReg cr)
4881 %{
4882   predicate(UseSVE > 0 &&
4883             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
4884   match(Set dst (VectorInsert (Binary src val) idx));
4885   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
4886   ins_cost(4 * SVE_COST);
4887   format %{ "sve_index $dst, D, -16, 1\n\t"
4888             "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
4889             "sve_orr $dst, $src, $src\n\t"
4890             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (L)" %}
4891   ins_encode %{
4892     __ sve_index(as_FloatRegister($dst$$reg), __ D, -16, 1);
4893     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
4894                as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
4895     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4896     __ sve_cpy(as_FloatRegister($dst$$reg), __ D, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
4897   %}
4898   ins_pipe(pipe_slow);
4899 %}
4900 
4901 instruct insertD(vReg dst, vReg src, vRegD val, immI idx, pRegGov pgtmp, rFlagsReg cr)
4902 %{
4903   predicate(UseSVE > 0 &&
4904             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
4905   match(Set dst (VectorInsert (Binary src val) idx));
4906   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
4907   ins_cost(4 * SVE_COST);
4908   format %{ "sve_index $dst, D, -16, 1\n\t"
4909             "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
4910             "sve_orr $dst, $src, $src\n\t"
4911             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (D)" %}
4912   ins_encode %{
4913     __ sve_index(as_FloatRegister($dst$$reg), __ D, -16, 1);
4914     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
4915                as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
4916     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4917     __ sve_cpy(as_FloatRegister($dst$$reg), __ D, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
4918   %}
4919   ins_pipe(pipe_slow);
4920 %}
4921 
4922 instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr)
4923 %{
4924   predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
4925             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4926   match(Set dst (VectorInsert (Binary src val) idx));
4927   effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
4928   ins_cost(5 * SVE_COST);
4929   format %{ "sve_index $tmp1, S, 0, 1\n\t"
4930             "sve_dup $dst, S, $idx\n\t"
4931             "sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
4932             "sve_orr $dst, $src, $src\n\t"
4933             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
4934   ins_encode %{
4935     __ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
4936     __ sve_dup(as_FloatRegister($dst$$reg), __ S, (int)($idx$$constant));
4937     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
4938                as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
4939     __ sve_orr(as_FloatRegister($dst$$reg),
4940                as_FloatRegister($src$$reg),
4941                as_FloatRegister($src$$reg));
4942     __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
4943                as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
4944   %}
4945   ins_pipe(pipe_slow);
4946 %}
4947 
4948 // ------------------------------ Vector shuffle -------------------------------
4949 
4950 instruct loadshuffle(vReg dst, vReg src) %{
4951   predicate(UseSVE > 0);

4952   match(Set dst (VectorLoadShuffle src));
4953   ins_cost(SVE_COST);
4954   format %{ "sve_loadshuffle $dst, $src\t# vector load shuffle (B/H/S/D)" %}
4955   ins_encode %{
4956     BasicType bt = Matcher::vector_element_basic_type(this);
4957     if (bt == T_BYTE) {
4958       if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
4959         __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
4960                    as_FloatRegister($src$$reg));
4961       }
4962     } else {
4963       __ sve_vector_extend(as_FloatRegister($dst$$reg),  __ elemType_to_regVariant(bt),
4964                            as_FloatRegister($src$$reg), __ B);
4965     }
4966   %}
4967   ins_pipe(pipe_slow);
4968 %}
4969 














































4970 // ------------------------------ Vector rearrange -------------------------------
4971 
4972 instruct rearrange(vReg dst, vReg src, vReg shuffle)
4973 %{
4974   predicate(UseSVE > 0);
4975   match(Set dst (VectorRearrange src shuffle));
4976   ins_cost(SVE_COST);
4977   format %{ "sve_tbl $dst, $src, $shuffle\t# vector rearrange" %}
4978   ins_encode %{
4979     BasicType bt = Matcher::vector_element_basic_type(this, $src);
4980     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
4981     __ sve_tbl(as_FloatRegister($dst$$reg), size,
4982                as_FloatRegister($src$$reg), as_FloatRegister($shuffle$$reg));
4983   %}
4984   ins_pipe(pipe_slow);
4985 %}
4986 
4987 // ------------------------------ Vector Load Gather ---------------------------------
4988 
4989 instruct gatherI(vReg dst, indirect mem, vReg idx) %{
4990   predicate(UseSVE > 0 &&
4991             n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
4992             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
4993              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
4994   match(Set dst (LoadVectorGather mem idx));
4995   ins_cost(SVE_COST);
4996   format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (S)" %}
4997   ins_encode %{
4998     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), ptrue,
4999                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5000   %}
5001   ins_pipe(pipe_slow);
5002 %}
5003 
5004 instruct gatherL(vReg dst, indirect mem, vReg idx) %{
5005   predicate(UseSVE > 0 &&
5006             n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
5007             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5008              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5009   match(Set dst (LoadVectorGather mem idx));
5010   ins_cost(2 * SVE_COST);
5011   format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (D)" %}

5012   ins_encode %{
5013     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5014     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), ptrue, as_Register($mem$$base),
5015                        as_FloatRegister($idx$$reg));
5016   %}
5017   ins_pipe(pipe_slow);
5018 %}
5019 
5020 // ------------------------------ Vector Load Gather Partial-------------------------------
5021 
5022 instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5023   predicate(UseSVE > 0 &&
5024             n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
5025             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5026              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5027   match(Set dst (LoadVectorGather mem idx));
5028   effect(TEMP ptmp, KILL cr);
5029   ins_cost(2 * SVE_COST + INSN_COST);
5030   format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (S)" %}

5031   ins_encode %{
5032     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this));
5033     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),

5034                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5035   %}
5036   ins_pipe(pipe_slow);
5037 %}
5038 
5039 instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5040   predicate(UseSVE > 0 &&
5041             n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
5042             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5043              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5044   match(Set dst (LoadVectorGather mem idx));
5045   effect(TEMP ptmp, KILL cr);
5046   ins_cost(3 * SVE_COST + INSN_COST);
5047   format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (D)" %}
5048   ins_encode %{
5049     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
5050                           Matcher::vector_length(this));
5051     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5052     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
5053                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5054   %}
5055   ins_pipe(pipe_slow);
5056 %}
5057 
5058 // ------------------------------ Vector Load Gather Predicated -------------------------------
5059 
5060 instruct gatherI_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
5061   predicate(UseSVE > 0 &&
5062             n->as_LoadVector()->memory_size() == MaxVectorSize &&
5063             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5064              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5065   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
5066   ins_cost(SVE_COST);
5067   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (S)" %}
5068   ins_encode %{
5069     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
5070                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5071   %}
5072   ins_pipe(pipe_slow);
5073 %}
5074 
5075 instruct gatherL_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
5076   predicate(UseSVE > 0 &&
5077             n->as_LoadVector()->memory_size() == MaxVectorSize &&
5078             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5079              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5080   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
5081   ins_cost(2 * SVE_COST);
5082   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (D)" %}
5083   ins_encode %{
5084     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5085     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
5086                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5087   %}
5088   ins_pipe(pipe_slow);
5089 %}
5090 
5091 // ------------------------------ Vector Load Gather Predicated Partial -------------------------------
5092 
5093 instruct gatherI_masked_partial(vReg dst, indirect mem, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
5094   predicate(UseSVE > 0 &&
5095             n->as_LoadVector()->memory_size() < MaxVectorSize &&
5096             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5097              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5098   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
5099   effect(TEMP ptmp, KILL cr);
5100   ins_cost(3 * SVE_COST);
5101   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated partial (S)" %}
5102   ins_encode %{
5103     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
5104                           Matcher::vector_length(this));
5105     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
5106                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
5107     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
5108                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5109   %}
5110   ins_pipe(pipe_slow);
5111 %}
5112 
5113 instruct gatherL_masked_partial(vReg dst, indirect mem, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
5114   predicate(UseSVE > 0 &&
5115             n->as_LoadVector()->memory_size() < MaxVectorSize &&
5116             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5117              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5118   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
5119   effect(TEMP ptmp, KILL cr);
5120   ins_cost(4 * SVE_COST);
5121   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated partial (D)" %}
5122   ins_encode %{
5123     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this));
5124     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
5125                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
5126     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5127     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
5128                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5129   %}
5130   ins_pipe(pipe_slow);
5131 %}
5132 
5133 // ------------------------------ Vector Store Scatter -------------------------------
5134 
5135 instruct scatterI(indirect mem, vReg src, vReg idx) %{
5136   predicate(UseSVE > 0 &&
5137             n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
5138             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5139              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5140   match(Set mem (StoreVectorScatter mem (Binary src idx)));
5141   ins_cost(SVE_COST);
5142   format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (S)" %}
5143   ins_encode %{
5144     __ sve_st1w_scatter(as_FloatRegister($src$$reg), ptrue,
5145                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5146   %}
5147   ins_pipe(pipe_slow);
5148 %}
5149 
5150 instruct scatterL(indirect mem, vReg src, vReg idx) %{
5151   predicate(UseSVE > 0 &&
5152             n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
5153             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5154              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5155   match(Set mem (StoreVectorScatter mem (Binary src idx)));
5156   ins_cost(2 * SVE_COST);
5157   format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (D)" %}

5158   ins_encode %{
5159     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));

5160     __ sve_st1d_scatter(as_FloatRegister($src$$reg), ptrue,
5161                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5162   %}
5163   ins_pipe(pipe_slow);
5164 %}
5165 
5166 // ------------------------------ Vector Store Scatter Partial -------------------------------
5167 
5168 instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5169   predicate(UseSVE > 0 &&
5170             n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
5171             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5172              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5173   match(Set mem (StoreVectorScatter mem (Binary src idx)));
5174   effect(TEMP ptmp, KILL cr);
5175   ins_cost(2 * SVE_COST + INSN_COST);
5176   format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (S)" %}

5177   ins_encode %{
5178     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
5179                           Matcher::vector_length(this, $src));
5180     __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
5181                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5182   %}
5183   ins_pipe(pipe_slow);
5184 %}
5185 
5186 instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5187   predicate(UseSVE > 0 &&
5188             n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
5189             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5190              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5191   match(Set mem (StoreVectorScatter mem (Binary src idx)));
5192   effect(TEMP ptmp, KILL cr);
5193   ins_cost(3 * SVE_COST + INSN_COST);
5194   format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (D)" %}


5195   ins_encode %{
5196     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
5197                           Matcher::vector_length(this, $src));
5198     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5199     __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
5200                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5201   %}
5202   ins_pipe(pipe_slow);
5203 %}
5204 
5205 // ------------------------------ Vector Store Scatter Predicated -------------------------------
5206 
5207 instruct scatterI_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
5208   predicate(UseSVE > 0 &&
5209             n->as_StoreVector()->memory_size() == MaxVectorSize &&
5210             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5211              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5212   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
5213   ins_cost(SVE_COST);
5214   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicate (S)" %}
5215   ins_encode %{
5216     __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
5217                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5218   %}
5219   ins_pipe(pipe_slow);
5220 %}
5221 
5222 instruct scatterL_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
5223   predicate(UseSVE > 0 &&
5224             n->as_StoreVector()->memory_size() == MaxVectorSize &&
5225             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5226              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5227   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
5228   ins_cost(2 * SVE_COST);
5229   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated (D)" %}
5230   ins_encode %{
5231     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5232     __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
5233                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5234   %}
5235   ins_pipe(pipe_slow);
5236 %}
5237 
5238 // ------------------------------ Vector Store Scatter Predicated Partial -------------------------------
5239 
5240 instruct scatterI_masked_partial(indirect mem, vReg src, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
5241   predicate(UseSVE > 0 &&
5242             n->as_StoreVector()->memory_size() < MaxVectorSize &&
5243             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5244              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5245   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
5246   effect(TEMP ptmp, KILL cr);
5247   ins_cost(3 * SVE_COST);
5248   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated partial (S)" %}
5249   ins_encode %{
5250     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
5251                           Matcher::vector_length(this, $src));
5252     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
5253                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
5254     __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
5255                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5256   %}
5257   ins_pipe(pipe_slow);
5258 %}
5259 
5260 instruct scatterL_masked_partial(indirect mem, vReg src, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
5261   predicate(UseSVE > 0 &&
5262             n->as_StoreVector()->memory_size() < MaxVectorSize &&
5263             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5264              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5265   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
5266   effect(TEMP ptmp, KILL cr);
5267   ins_cost(4 * SVE_COST);
5268   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated partial (D)" %}
5269   ins_encode %{
5270     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
5271                           Matcher::vector_length(this, $src));
5272     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
5273                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
5274     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5275     __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
5276                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5277   %}
5278   ins_pipe(pipe_slow);
5279 %}
5280 
5281 // ------------------------------ Vector Load Const -------------------------------
5282 
5283 instruct loadconB(vReg dst, immI0 src) %{
5284   predicate(UseSVE > 0 &&
5285             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
5286   match(Set dst (VectorLoadConst src));
5287   ins_cost(SVE_COST);
5288   format %{ "sve_index $dst, 0, 1\t# generate iota indices" %}
5289   ins_encode %{
5290     __ sve_index(as_FloatRegister($dst$$reg), __ B, 0, 1);
5291   %}
5292   ins_pipe(pipe_slow);
5293 %}
5294 
5295 // Intrisics for String.indexOf(char)
5296 
5297 
5298 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
5299                                   iRegI_R0 result, vReg ztmp1, vReg ztmp2,

5315 
5316 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
5317                                   iRegI_R0 result, vReg ztmp1, vReg ztmp2,
5318                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
5319 %{
5320   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
5321   predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
5322   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
5323 
5324   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
5325 
5326   ins_encode %{
5327     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
5328                                as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
5329                                as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), false /* isL */);
5330   %}
5331   ins_pipe(pipe_class_memory);
5332 %}
5333 
5334 // ---------------------------- Vector mask reductions ---------------------------
5335 instruct vmask_truecount(iRegINoSp dst, pReg src) %{

5336   predicate(UseSVE > 0 &&
5337             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
5338   match(Set dst (VectorMaskTrueCount src));
5339   ins_cost(SVE_COST);

5340   format %{ "vmask_truecount $dst, $src\t# vector mask truecount (sve)" %}
5341   ins_encode %{
5342     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5343     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5344     __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($src$$reg));
5345   %}
5346   ins_pipe(pipe_slow);
5347 %}
5348 
5349 instruct vmask_firsttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
5350   predicate(UseSVE > 0 &&
5351             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
5352   match(Set dst (VectorMaskFirstTrue src));
5353   effect(TEMP ptmp);
5354   ins_cost(2 * SVE_COST);
5355   format %{ "vmask_firsttrue $dst, $src\t# vector mask firsttrue (sve)" %}
5356   ins_encode %{
5357     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5358     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5359     __ sve_brkb(as_PRegister($ptmp$$reg), ptrue, as_PRegister($src$$reg), false);
5360     __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($ptmp$$reg));
5361   %}
5362   ins_pipe(pipe_slow);
5363 %}
5364 
5365 instruct vmask_lasttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
5366   predicate(UseSVE > 0 &&
5367             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
5368   match(Set dst (VectorMaskLastTrue src));
5369   effect(TEMP ptmp);
5370   ins_cost(3 * SVE_COST);
5371   format %{ "vmask_lasttrue $dst, $src\t# vector mask lasttrue (sve)" %}
5372   ins_encode %{
5373     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5374     __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($src$$reg), as_PRegister($ptmp$$reg));
5375   %}
5376   ins_pipe(pipe_slow);
5377 %}
5378 
5379 instruct vmask_truecount_partial(iRegINoSp dst, pReg src, pRegGov pgtmp, rFlagsReg cr) %{
5380   predicate(UseSVE > 0 &&
5381             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
5382   match(Set dst (VectorMaskTrueCount src));
5383   effect(TEMP pgtmp, KILL cr);
5384   ins_cost(2 * SVE_COST);
5385   format %{ "vmask_truecount_partial $dst, $src\t# vector mask truecount partial (sve)" %}
5386   ins_encode %{
5387     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5388     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5389     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), size, Matcher::vector_length(this, $src));
5390     __ sve_cntp($dst$$Register, size, as_PRegister($pgtmp$$reg), as_PRegister($src$$reg));
5391   %}
5392   ins_pipe(pipe_slow);
5393 %}
5394 
5395 instruct vmask_firsttrue_partial(iRegINoSp dst, pReg src, pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
5396   predicate(UseSVE > 0 &&
5397             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
5398   match(Set dst (VectorMaskFirstTrue src));
5399   effect(TEMP pgtmp, TEMP ptmp, KILL cr);
5400   ins_cost(3 * SVE_COST);
5401   format %{ "vmask_firsttrue_partial $dst, $src\t# vector mask firsttrue partial (sve)" %}
5402   ins_encode %{
5403     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5404     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5405     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), size,
5406                           Matcher::vector_length(this, $src));
5407     __ sve_brkb(as_PRegister($ptmp$$reg), as_PRegister($pgtmp$$reg), as_PRegister($src$$reg), false);
5408     __ sve_cntp($dst$$Register, size, as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg));
5409   %}
5410   ins_pipe(pipe_slow);
5411 %}
5412 
5413 instruct vmask_lasttrue_partial(iRegINoSp dst, pReg src, pReg ptmp, rFlagsReg cr) %{
5414   predicate(UseSVE > 0 &&
5415             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
5416   match(Set dst (VectorMaskLastTrue src));
5417   effect(TEMP ptmp, KILL cr);
5418   ins_cost(5 * SVE_COST);
5419   format %{ "vmask_lasttrue_partial $dst, $src\t# vector mask lasttrue partial (sve)" %}











































































































5420   ins_encode %{
5421     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5422     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5423     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
5424     __ sve_and(as_PRegister($ptmp$$reg), ptrue, as_PRegister($ptmp$$reg), as_PRegister($src$$reg));
5425     __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg));


5426   %}
5427   ins_pipe(pipe_slow);
5428 %}
< prev index next >