1 //
   2 // Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
   3 // Copyright (c) 2020, 2021, Arm Limited. All rights reserved.
   4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5 //
   6 // This code is free software; you can redistribute it and/or modify it
   7 // under the terms of the GNU General Public License version 2 only, as
   8 // published by the Free Software Foundation.
   9 //
  10 // This code is distributed in the hope that it will be useful, but WITHOUT
  11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 // version 2 for more details (a copy is included in the LICENSE file that
  14 // accompanied this code).
  15 //
  16 // You should have received a copy of the GNU General Public License version
  17 // 2 along with this work; if not, write to the Free Software Foundation,
  18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21 // or visit www.oracle.com if you need additional information or have any
  22 // questions.
  23 //
  24 //
  25 
  26 // This file is automatically generated by running "m4 aarch64_sve_ad.m4". Do not edit ----
  27 
  28 // AArch64 SVE Architecture Description File
  29 
  30 
  31 // 4 bit signed offset -- for predicated load/store
  32 
  33 operand vmemA_immIOffset4()
  34 %{
  35   // (esize / msize) = 1
  36   predicate(Address::offset_ok_for_sve_immed(n->get_int(), 4,
  37             Matcher::scalable_vector_reg_size(T_BYTE)));
  38   match(ConI);
  39 
  40   op_cost(0);
  41   format %{ %}
  42   interface(CONST_INTER);
  43 %}
  44 
  45 operand vmemA_immLOffset4()
  46 %{
  47   // (esize / msize) = 1
  48   predicate(Address::offset_ok_for_sve_immed(n->get_long(), 4,
  49             Matcher::scalable_vector_reg_size(T_BYTE)));
  50   match(ConL);
  51 
  52   op_cost(0);
  53   format %{ %}
  54   interface(CONST_INTER);
  55 %}
  56 
  57 operand vmemA_indOffI4(iRegP reg, vmemA_immIOffset4 off)
  58 %{
  59   constraint(ALLOC_IN_RC(ptr_reg));
  60   match(AddP reg off);
  61   op_cost(0);
  62   format %{ "[$reg, $off]" %}
  63   interface(MEMORY_INTER) %{
  64     base($reg);
  65     index(0xffffffff);
  66     scale(0x0);
  67     disp($off);
  68   %}
  69 %}
  70 
  71 operand vmemA_indOffL4(iRegP reg, vmemA_immLOffset4 off)
  72 %{
  73   constraint(ALLOC_IN_RC(ptr_reg));
  74   match(AddP reg off);
  75   op_cost(0);
  76   format %{ "[$reg, $off]" %}
  77   interface(MEMORY_INTER) %{
  78     base($reg);
  79     index(0xffffffff);
  80     scale(0x0);
  81     disp($off);
  82   %}
  83 %}
  84 
  85 // The indOff of vmemA is valid only when the vector element (load to/store from)
  86 // size equals to memory element (load from/store to) size.
  87 opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);
  88 
  89 source_hpp %{
  90   bool op_sve_supported(int opcode, int vlen, BasicType bt);
  91   bool masked_op_sve_supported(int opcode, int vlen, BasicType bt);
  92 %}
  93 
  94 source %{
  95 
  96   typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
  97                                                              PRegister Pg, const Address &adr);
  98 
  99   // Predicated load/store, with optional ptrue to all elements of given predicate register.
 100   static void loadStoreA_predicated(C2_MacroAssembler masm, bool is_store, FloatRegister reg,
 101                                     PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
 102                                     int opcode, Register base, int index, int size, int disp) {
 103     sve_mem_insn_predicate insn;
 104     int mesize = type2aelembytes(mem_elem_bt);
 105     if (index == -1) {
 106       assert(size == 0, "unsupported address mode: scale size = %d", size);
 107       switch(mesize) {
 108       case 1:
 109         insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
 110         break;
 111       case 2:
 112         insn = is_store ? &C2_MacroAssembler::sve_st1h : &C2_MacroAssembler::sve_ld1h;
 113         break;
 114       case 4:
 115         insn = is_store ? &C2_MacroAssembler::sve_st1w : &C2_MacroAssembler::sve_ld1w;
 116         break;
 117       case 8:
 118         insn = is_store ? &C2_MacroAssembler::sve_st1d : &C2_MacroAssembler::sve_ld1d;
 119         break;
 120       default:
 121         assert(false, "unsupported");
 122         ShouldNotReachHere();
 123       }
 124       int imm4 = disp / mesize / Matcher::scalable_vector_reg_size(vector_elem_bt);
 125       (masm.*insn)(reg, Assembler::elemType_to_regVariant(vector_elem_bt), pg, Address(base, imm4));
 126     } else {
 127       assert(false, "unimplemented");
 128       ShouldNotReachHere();
 129     }
 130   }
 131 
 132   bool op_sve_supported(int opcode, int vlen, BasicType bt) {
 133     int length_in_bytes = vlen * type2aelembytes(bt);
 134     switch (opcode) {
 135       case Op_MulAddVS2VI:
 136       // No multiply reduction instructions
 137       case Op_MulReductionVD:
 138       case Op_MulReductionVF:
 139       case Op_MulReductionVI:
 140       case Op_MulReductionVL:
 141       // Others
 142       case Op_ExtractC:
 143       case Op_ExtractUB:
 144         return false;
 145       // Vector API specific
 146       case Op_VectorLoadShuffle:
 147       case Op_VectorRearrange:
 148         return vlen >= 4 && length_in_bytes <= MaxVectorSize;
 149       case Op_LoadVector:
 150       case Op_StoreVector:
 151         return Matcher::vector_size_supported(bt, vlen);
 152       default:
 153         break;
 154     }
 155     // By default, we only support vector operations with no less than 8 bytes and 2 elements.
 156     return 8 <= length_in_bytes && length_in_bytes <= MaxVectorSize && vlen >= 2;
 157   }
 158 
 159   bool masked_op_sve_supported(int opcode, int vlen, BasicType bt) {
 160     if (opcode == Op_VectorRearrange) {
 161       return false;
 162     }
 163     return op_sve_supported(opcode, vlen, bt);
 164   }
 165 
 166 %}
 167 
 168 definitions %{
 169   int_def SVE_COST             (200, 200);
 170 %}
 171 
 172 
 173 // All SVE instructions
 174 
 175 // vector load/store
 176 
 177 // Unpredicated vector load/store
 178 instruct loadV(vReg dst, vmemA mem) %{
 179   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() >= 16 &&
 180             n->as_LoadVector()->memory_size() == MaxVectorSize);
 181   match(Set dst (LoadVector mem));
 182   ins_cost(4 * SVE_COST);
 183   format %{ "sve_ldr $dst, $mem\t# vector (sve)" %}
 184   ins_encode %{
 185     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 186     BasicType bt = Matcher::vector_element_basic_type(this);
 187     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg, ptrue,
 188                           bt, bt, $mem->opcode(),
 189                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 190   %}
 191   ins_pipe(pipe_slow);
 192 %}
 193 
 194 instruct storeV(vReg src, vmemA mem) %{
 195   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() >= 16 &&
 196             n->as_StoreVector()->memory_size() == MaxVectorSize);
 197   match(Set mem (StoreVector mem src));
 198   ins_cost(4 * SVE_COST);
 199   format %{ "sve_str $mem, $src\t# vector (sve)" %}
 200   ins_encode %{
 201     FloatRegister src_reg = as_FloatRegister($src$$reg);
 202     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 203     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg, ptrue,
 204                           bt, bt, $mem->opcode(),
 205                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 206   %}
 207   ins_pipe(pipe_slow);
 208 %}
 209 
 210 // Load Vector (16 bits)
 211 instruct loadV2_vreg(vReg dst, vmem2 mem)
 212 %{
 213   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 2);
 214   match(Set dst (LoadVector mem));
 215   ins_cost(4 * INSN_COST);
 216   format %{ "ldrh   $dst,$mem\t# vector (16 bits)" %}
 217   ins_encode( aarch64_enc_ldrvH(dst, mem) );
 218   ins_pipe(vload_reg_mem64);
 219 %}
 220 
 221 // Store Vector (16 bits)
 222 instruct storeV2_vreg(vReg src, vmem2 mem)
 223 %{
 224   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 2);
 225   match(Set mem (StoreVector mem src));
 226   ins_cost(4 * INSN_COST);
 227   format %{ "strh   $mem,$src\t# vector (16 bits)" %}
 228   ins_encode( aarch64_enc_strvH(src, mem) );
 229   ins_pipe(vstore_reg_mem64);
 230 %}
 231 
 232 // Load Vector (32 bits)
 233 instruct loadV4_vreg(vReg dst, vmem4 mem)
 234 %{
 235   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 4);
 236   match(Set dst (LoadVector mem));
 237   ins_cost(4 * INSN_COST);
 238   format %{ "ldrs   $dst,$mem\t# vector (32 bits)" %}
 239   ins_encode( aarch64_enc_ldrvS(dst, mem) );
 240   ins_pipe(vload_reg_mem64);
 241 %}
 242 
 243 // Store Vector (32 bits)
 244 instruct storeV4_vreg(vReg src, vmem4 mem)
 245 %{
 246   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 4);
 247   match(Set mem (StoreVector mem src));
 248   ins_cost(4 * INSN_COST);
 249   format %{ "strs   $mem,$src\t# vector (32 bits)" %}
 250   ins_encode( aarch64_enc_strvS(src, mem) );
 251   ins_pipe(vstore_reg_mem64);
 252 %}
 253 
 254 // Load Vector (64 bits)
 255 instruct loadV8_vreg(vReg dst, vmem8 mem)
 256 %{
 257   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 8);
 258   match(Set dst (LoadVector mem));
 259   ins_cost(4 * INSN_COST);
 260   format %{ "ldrd   $dst,$mem\t# vector (64 bits)" %}
 261   ins_encode( aarch64_enc_ldrvD(dst, mem) );
 262   ins_pipe(vload_reg_mem64);
 263 %}
 264 
 265 // Store Vector (64 bits)
 266 instruct storeV8_vreg(vReg src, vmem8 mem)
 267 %{
 268   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 8);
 269   match(Set mem (StoreVector mem src));
 270   ins_cost(4 * INSN_COST);
 271   format %{ "strd   $mem,$src\t# vector (64 bits)" %}
 272   ins_encode( aarch64_enc_strvD(src, mem) );
 273   ins_pipe(vstore_reg_mem64);
 274 %}
 275 
 276 // Load Vector (128 bits)
 277 instruct loadV16_vreg(vReg dst, vmem16 mem)
 278 %{
 279   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 16);
 280   match(Set dst (LoadVector mem));
 281   ins_cost(4 * INSN_COST);
 282   format %{ "ldrq   $dst,$mem\t# vector (128 bits)" %}
 283   ins_encode( aarch64_enc_ldrvQ(dst, mem) );
 284   ins_pipe(vload_reg_mem128);
 285 %}
 286 
 287 // Store Vector (128 bits)
 288 instruct storeV16_vreg(vReg src, vmem16 mem)
 289 %{
 290   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 16);
 291   match(Set mem (StoreVector mem src));
 292   ins_cost(4 * INSN_COST);
 293   format %{ "strq   $mem,$src\t# vector (128 bits)" %}
 294   ins_encode( aarch64_enc_strvQ(src, mem) );
 295   ins_pipe(vstore_reg_mem128);
 296 %}
 297 
 298 // Predicated vector load/store, based on the vector length of the node.
 299 // Only load/store values in the range of the memory_size. This is needed
 300 // when the memory_size is lower than the hardware supported max vector size.
 301 // And this might happen for Vector API mask vector load/store.
 302 instruct loadV_partial(vReg dst, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
 303   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() > 16 &&
 304             n->as_LoadVector()->memory_size() < MaxVectorSize);
 305   match(Set dst (LoadVector mem));
 306   effect(TEMP pgtmp, KILL cr);
 307   ins_cost(6 * SVE_COST);
 308   format %{ "sve_whilelo_zr_imm $pgtmp, vector_length\n\t"
 309             "sve_ldr $dst, $pgtmp, $mem\t# load vector partial" %}
 310   ins_encode %{
 311     BasicType bt = Matcher::vector_element_basic_type(this);
 312     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 313                           Matcher::vector_length(this));
 314     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 315     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg,
 316                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 317                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 318   %}
 319   ins_pipe(pipe_slow);
 320 %}
 321 
 322 instruct storeV_partial(vReg src, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
 323   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() > 16 &&
 324             n->as_StoreVector()->memory_size() < MaxVectorSize);
 325   match(Set mem (StoreVector mem src));
 326   effect(TEMP pgtmp, KILL cr);
 327   ins_cost(5 * SVE_COST);
 328   format %{ "sve_whilelo_zr_imm $pgtmp, vector_length\n\t"
 329             "sve_str $src, $pgtmp, $mem\t# store vector partial" %}
 330   ins_encode %{
 331     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 332     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 333                           Matcher::vector_length(this, $src));
 334     FloatRegister src_reg = as_FloatRegister($src$$reg);
 335     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg,
 336                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 337                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 338   %}
 339   ins_pipe(pipe_slow);
 340 %}
 341 
 342 // vector load/store - predicated
 343 
 344 instruct loadV_masked(vReg dst, vmemA mem, pRegGov pg) %{
 345   predicate(UseSVE > 0 &&
 346             n->as_LoadVector()->memory_size() == MaxVectorSize);
 347   match(Set dst (LoadVectorMasked mem pg));
 348   ins_cost(4 * SVE_COST);
 349   format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated (sve)" %}
 350   ins_encode %{
 351     BasicType bt = Matcher::vector_element_basic_type(this);
 352     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
 353                           as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
 354                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 355   %}
 356   ins_pipe(pipe_slow);
 357 %}
 358 
 359 instruct loadV_masked_partial(vReg dst, vmemA mem, pRegGov pg, pRegGov pgtmp, rFlagsReg cr) %{
 360   predicate(UseSVE > 0 &&
 361             n->as_LoadVector()->memory_size() < MaxVectorSize);
 362   match(Set dst (LoadVectorMasked mem pg));
 363   effect(TEMP pgtmp, KILL cr);
 364   ins_cost(6 * SVE_COST);
 365   format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated partial (sve)" %}
 366   ins_encode %{
 367     BasicType bt = Matcher::vector_element_basic_type(this);
 368     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 369                           Matcher::vector_length(this));
 370     __ sve_and(as_PRegister($pgtmp$$reg), as_PRegister($pgtmp$$reg),
 371                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
 372     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
 373                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 374                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 375   %}
 376   ins_pipe(pipe_slow);
 377 %}
 378 
 379 instruct storeV_masked(vReg src, vmemA mem, pRegGov pg) %{
 380   predicate(UseSVE > 0 &&
 381             n->as_StoreVector()->memory_size() == MaxVectorSize);
 382   match(Set mem (StoreVectorMasked mem (Binary src pg)));
 383   ins_cost(4 * SVE_COST);
 384   format %{ "sve_str $mem, $pg, $src\t# store vector predicated (sve)" %}
 385   ins_encode %{
 386     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 387     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
 388                           as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
 389                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 390   %}
 391   ins_pipe(pipe_slow);
 392 %}
 393 
 394 instruct storeV_masked_partial(vReg src, vmemA mem, pRegGov pg, pRegGov pgtmp, rFlagsReg cr) %{
 395   predicate(UseSVE > 0 &&
 396             n->as_StoreVector()->memory_size() < MaxVectorSize);
 397   match(Set mem (StoreVectorMasked mem (Binary src pg)));
 398   effect(TEMP pgtmp, KILL cr);
 399   ins_cost(6 * SVE_COST);
 400   format %{ "sve_str $mem, $pg, $src\t# store vector predicated partial (sve)" %}
 401   ins_encode %{
 402     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 403     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 404                           Matcher::vector_length(this, $src));
 405     __ sve_and(as_PRegister($pgtmp$$reg), as_PRegister($pgtmp$$reg),
 406                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
 407     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
 408                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 409                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 410   %}
 411   ins_pipe(pipe_slow);
 412 %}
 413 
 414 // maskAll
 415 
 416 instruct vmaskAll_immI(pRegGov dst, immI src) %{
 417   predicate(UseSVE > 0);
 418   match(Set dst (MaskAll src));
 419   ins_cost(SVE_COST);
 420   format %{ "sve_ptrue/sve_pfalse $dst\t# mask all (sve) (B/H/S)" %}
 421   ins_encode %{
 422     int con = (int)$src$$constant;
 423     if (con == 0) {
 424       __ sve_pfalse(as_PRegister($dst$$reg));
 425     } else {
 426       assert(con == -1, "invalid constant value for mask");
 427       BasicType bt = Matcher::vector_element_basic_type(this);
 428       __ sve_ptrue(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt));
 429     }
 430   %}
 431   ins_pipe(pipe_slow);
 432 %}
 433 
 434 instruct vmaskAllI(pRegGov dst, iRegIorL2I src, vReg tmp, rFlagsReg cr) %{
 435   predicate(UseSVE > 0);
 436   match(Set dst (MaskAll src));
 437   effect(TEMP tmp, KILL cr);
 438   ins_cost(2 * SVE_COST);
 439   format %{ "sve_dup $tmp, $src\n\t"
 440             "sve_cmpne $dst, $tmp, 0\t# mask all (sve) (B/H/S)" %}
 441   ins_encode %{
 442     BasicType bt = Matcher::vector_element_basic_type(this);
 443     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
 444     __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
 445     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
 446   %}
 447   ins_pipe(pipe_slow);
 448 %}
 449 
 450 instruct vmaskAll_immL(pRegGov dst, immL src) %{
 451   predicate(UseSVE > 0);
 452   match(Set dst (MaskAll src));
 453   ins_cost(SVE_COST);
 454   format %{ "sve_ptrue/sve_pfalse $dst\t# mask all (sve) (D)" %}
 455   ins_encode %{
 456     long con = (long)$src$$constant;
 457     if (con == 0) {
 458       __ sve_pfalse(as_PRegister($dst$$reg));
 459     } else {
 460       assert(con == -1, "invalid constant value for mask");
 461       BasicType bt = Matcher::vector_element_basic_type(this);
 462       __ sve_ptrue(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt));
 463     }
 464   %}
 465   ins_pipe(pipe_slow);
 466 %}
 467 
 468 instruct vmaskAllL(pRegGov dst, iRegL src, vReg tmp, rFlagsReg cr) %{
 469   predicate(UseSVE > 0);
 470   match(Set dst (MaskAll src));
 471   effect(TEMP tmp, KILL cr);
 472   ins_cost(2 * SVE_COST);
 473   format %{ "sve_dup $tmp, $src\n\t"
 474             "sve_cmpne $dst, $tmp, 0\t# mask all (sve) (D)" %}
 475   ins_encode %{
 476     BasicType bt = Matcher::vector_element_basic_type(this);
 477     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
 478     __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
 479     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
 480   %}
 481   ins_pipe(pipe_slow);
 482 %}
 483 
 484 // mask logical and/or/xor
 485 
 486 instruct vmask_and(pRegGov pd, pRegGov pn, pRegGov pm) %{
 487   predicate(UseSVE > 0);
 488   match(Set pd (AndVMask pn pm));
 489   ins_cost(SVE_COST);
 490   format %{ "sve_and $pd, $pn, $pm\t# predicate (sve)" %}
 491   ins_encode %{
 492     __ sve_and(as_PRegister($pd$$reg), ptrue,
 493                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 494   %}
 495   ins_pipe(pipe_slow);
 496 %}
 497 
 498 instruct vmask_or(pRegGov pd, pRegGov pn, pRegGov pm) %{
 499   predicate(UseSVE > 0);
 500   match(Set pd (OrVMask pn pm));
 501   ins_cost(SVE_COST);
 502   format %{ "sve_orr $pd, $pn, $pm\t# predicate (sve)" %}
 503   ins_encode %{
 504     __ sve_orr(as_PRegister($pd$$reg), ptrue,
 505                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 506   %}
 507   ins_pipe(pipe_slow);
 508 %}
 509 
 510 instruct vmask_xor(pRegGov pd, pRegGov pn, pRegGov pm) %{
 511   predicate(UseSVE > 0);
 512   match(Set pd (XorVMask pn pm));
 513   ins_cost(SVE_COST);
 514   format %{ "sve_eor $pd, $pn, $pm\t# predicate (sve)" %}
 515   ins_encode %{
 516     __ sve_eor(as_PRegister($pd$$reg), ptrue,
 517                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 518   %}
 519   ins_pipe(pipe_slow);
 520 %}
 521 
 522 // mask logical and_not
 523 
 524 instruct vmask_and_notI(pRegGov pd, pRegGov pn, pRegGov pm, immI_M1 m1) %{
 525   predicate(UseSVE > 0);
 526   match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
 527   ins_cost(SVE_COST);
 528   format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) (B/H/S)" %}
 529   ins_encode %{
 530     __ sve_bic(as_PRegister($pd$$reg), ptrue,
 531                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 532   %}
 533   ins_pipe(pipe_slow);
 534 %}
 535 
 536 instruct vmask_and_notL(pRegGov pd, pRegGov pn, pRegGov pm, immL_M1 m1) %{
 537   predicate(UseSVE > 0);
 538   match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
 539   ins_cost(SVE_COST);
 540   format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) (D)" %}
 541   ins_encode %{
 542     __ sve_bic(as_PRegister($pd$$reg), ptrue,
 543                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 544   %}
 545   ins_pipe(pipe_slow);
 546 %}
 547 
 548 // vector reinterpret
 549 
 550 instruct reinterpret(vReg dst) %{
 551   predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() ==
 552                           n->in(1)->bottom_type()->is_vect()->length_in_bytes());  // src == dst
 553   match(Set dst (VectorReinterpret dst));
 554   ins_cost(0);
 555   format %{ "# reinterpret $dst\t# do nothing" %}
 556   ins_encode %{
 557     // empty
 558   %}
 559   ins_pipe(pipe_class_empty);
 560 %}
 561 
 562 instruct reinterpretResize(vReg dst, vReg src, pRegGov pgtmp, rFlagsReg cr) %{
 563   predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() !=
 564                           n->in(1)->bottom_type()->is_vect()->length_in_bytes());  // src != dst
 565   match(Set dst (VectorReinterpret src));
 566   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
 567   ins_cost(3 * SVE_COST);
 568   format %{ "reinterpretResize $dst, $src\t# vector (sve)" %}
 569   ins_encode %{
 570     uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
 571     uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
 572     uint length_in_bytes_resize = length_in_bytes_src < length_in_bytes_dst ?
 573                                   length_in_bytes_src : length_in_bytes_dst;
 574     assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
 575            "invalid vector length");
 576     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ B, length_in_bytes_resize);
 577     __ sve_dup(as_FloatRegister($dst$$reg), __ B, 0);
 578     __ sve_sel(as_FloatRegister($dst$$reg), __ B, as_PRegister($pgtmp$$reg),
 579                as_FloatRegister($src$$reg), as_FloatRegister($dst$$reg));
 580   %}
 581   ins_pipe(pipe_slow);
 582 %}
 583 
 584 // vector mask reinterpret
 585 
 586 instruct vmask_reinterpret_same_esize(pRegGov dst_src) %{
 587   predicate(UseSVE > 0 &&
 588             n->as_Vector()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
 589             n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
 590   match(Set dst_src (VectorReinterpret dst_src));
 591   ins_cost(0);
 592   format %{ "# vmask_reinterpret $dst_src\t# do nothing" %}
 593   ins_encode %{
 594     // empty
 595   %}
 596   ins_pipe(pipe_class_empty);
 597 %}
 598 
 599 instruct vmask_reinterpret_diff_esize(pRegGov dst, pRegGov src, vReg tmp, rFlagsReg cr) %{
 600   predicate(UseSVE > 0 &&
 601             n->as_Vector()->length() != n->in(1)->bottom_type()->is_vect()->length() &&
 602             n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
 603   match(Set dst (VectorReinterpret src));
 604   effect(TEMP tmp, KILL cr);
 605   ins_cost(2 * SVE_COST);
 606   format %{ "# vmask_reinterpret $dst, $src\t# vector (sve)" %}
 607   ins_encode %{
 608     BasicType from_bt = Matcher::vector_element_basic_type(this, $src);
 609     Assembler::SIMD_RegVariant from_size = __ elemType_to_regVariant(from_bt);
 610     BasicType to_bt = Matcher::vector_element_basic_type(this);
 611     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
 612     __ sve_cpy(as_FloatRegister($tmp$$reg), from_size, as_PRegister($src$$reg), -1, false);
 613     __ sve_cmp(Assembler::EQ, as_PRegister($dst$$reg), to_size, ptrue, as_FloatRegister($tmp$$reg), -1);
 614   %}
 615   ins_pipe(pipe_slow);
 616 %}
 617 
 618 // vector abs
 619 
 620 instruct vabsB(vReg dst, vReg src) %{
 621   predicate(UseSVE > 0 &&
 622             !n->as_Vector()->is_predicated_vector());
 623   match(Set dst (AbsVB src));
 624   ins_cost(SVE_COST);
 625   format %{ "sve_abs $dst, $src\t# vector (sve) (B)" %}
 626   ins_encode %{
 627     __ sve_abs(as_FloatRegister($dst$$reg), __ B,
 628          ptrue, as_FloatRegister($src$$reg));
 629   %}
 630   ins_pipe(pipe_slow);
 631 %}
 632 
 633 instruct vabsS(vReg dst, vReg src) %{
 634   predicate(UseSVE > 0 &&
 635             !n->as_Vector()->is_predicated_vector());
 636   match(Set dst (AbsVS src));
 637   ins_cost(SVE_COST);
 638   format %{ "sve_abs $dst, $src\t# vector (sve) (H)" %}
 639   ins_encode %{
 640     __ sve_abs(as_FloatRegister($dst$$reg), __ H,
 641          ptrue, as_FloatRegister($src$$reg));
 642   %}
 643   ins_pipe(pipe_slow);
 644 %}
 645 
 646 instruct vabsI(vReg dst, vReg src) %{
 647   predicate(UseSVE > 0 &&
 648             !n->as_Vector()->is_predicated_vector());
 649   match(Set dst (AbsVI src));
 650   ins_cost(SVE_COST);
 651   format %{ "sve_abs $dst, $src\t# vector (sve) (S)" %}
 652   ins_encode %{
 653     __ sve_abs(as_FloatRegister($dst$$reg), __ S,
 654          ptrue, as_FloatRegister($src$$reg));
 655   %}
 656   ins_pipe(pipe_slow);
 657 %}
 658 
 659 instruct vabsL(vReg dst, vReg src) %{
 660   predicate(UseSVE > 0 &&
 661             !n->as_Vector()->is_predicated_vector());
 662   match(Set dst (AbsVL src));
 663   ins_cost(SVE_COST);
 664   format %{ "sve_abs $dst, $src\t# vector (sve) (D)" %}
 665   ins_encode %{
 666     __ sve_abs(as_FloatRegister($dst$$reg), __ D,
 667          ptrue, as_FloatRegister($src$$reg));
 668   %}
 669   ins_pipe(pipe_slow);
 670 %}
 671 
 672 instruct vabsF(vReg dst, vReg src) %{
 673   predicate(UseSVE > 0 &&
 674             !n->as_Vector()->is_predicated_vector());
 675   match(Set dst (AbsVF src));
 676   ins_cost(SVE_COST);
 677   format %{ "sve_fabs $dst, $src\t# vector (sve) (S)" %}
 678   ins_encode %{
 679     __ sve_fabs(as_FloatRegister($dst$$reg), __ S,
 680          ptrue, as_FloatRegister($src$$reg));
 681   %}
 682   ins_pipe(pipe_slow);
 683 %}
 684 
 685 instruct vabsD(vReg dst, vReg src) %{
 686   predicate(UseSVE > 0 &&
 687             !n->as_Vector()->is_predicated_vector());
 688   match(Set dst (AbsVD src));
 689   ins_cost(SVE_COST);
 690   format %{ "sve_fabs $dst, $src\t# vector (sve) (D)" %}
 691   ins_encode %{
 692     __ sve_fabs(as_FloatRegister($dst$$reg), __ D,
 693          ptrue, as_FloatRegister($src$$reg));
 694   %}
 695   ins_pipe(pipe_slow);
 696 %}
 697 
 698 // vector abs - predicated
 699 
 700 instruct vabsB_masked(vReg dst_src, pRegGov pg) %{
 701   predicate(UseSVE > 0);
 702   match(Set dst_src (AbsVB dst_src pg));
 703   ins_cost(SVE_COST);
 704   format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (B)" %}
 705   ins_encode %{
 706     __ sve_abs(as_FloatRegister($dst_src$$reg), __ B,
 707             as_PRegister($pg$$reg),
 708             as_FloatRegister($dst_src$$reg));
 709   %}
 710   ins_pipe(pipe_slow);
 711 %}
 712 
 713 instruct vabsS_masked(vReg dst_src, pRegGov pg) %{
 714   predicate(UseSVE > 0);
 715   match(Set dst_src (AbsVS dst_src pg));
 716   ins_cost(SVE_COST);
 717   format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (H)" %}
 718   ins_encode %{
 719     __ sve_abs(as_FloatRegister($dst_src$$reg), __ H,
 720             as_PRegister($pg$$reg),
 721             as_FloatRegister($dst_src$$reg));
 722   %}
 723   ins_pipe(pipe_slow);
 724 %}
 725 
 726 instruct vabsI_masked(vReg dst_src, pRegGov pg) %{
 727   predicate(UseSVE > 0);
 728   match(Set dst_src (AbsVI dst_src pg));
 729   ins_cost(SVE_COST);
 730   format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
 731   ins_encode %{
 732     __ sve_abs(as_FloatRegister($dst_src$$reg), __ S,
 733             as_PRegister($pg$$reg),
 734             as_FloatRegister($dst_src$$reg));
 735   %}
 736   ins_pipe(pipe_slow);
 737 %}
 738 
 739 instruct vabsL_masked(vReg dst_src, pRegGov pg) %{
 740   predicate(UseSVE > 0);
 741   match(Set dst_src (AbsVL dst_src pg));
 742   ins_cost(SVE_COST);
 743   format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
 744   ins_encode %{
 745     __ sve_abs(as_FloatRegister($dst_src$$reg), __ D,
 746             as_PRegister($pg$$reg),
 747             as_FloatRegister($dst_src$$reg));
 748   %}
 749   ins_pipe(pipe_slow);
 750 %}
 751 
 752 instruct vabsF_masked(vReg dst_src, pRegGov pg) %{
 753   predicate(UseSVE > 0);
 754   match(Set dst_src (AbsVF dst_src pg));
 755   ins_cost(SVE_COST);
 756   format %{ "sve_fabs $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
 757   ins_encode %{
 758     __ sve_fabs(as_FloatRegister($dst_src$$reg), __ S,
 759             as_PRegister($pg$$reg),
 760             as_FloatRegister($dst_src$$reg));
 761   %}
 762   ins_pipe(pipe_slow);
 763 %}
 764 
 765 instruct vabsD_masked(vReg dst_src, pRegGov pg) %{
 766   predicate(UseSVE > 0);
 767   match(Set dst_src (AbsVD dst_src pg));
 768   ins_cost(SVE_COST);
 769   format %{ "sve_fabs $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
 770   ins_encode %{
 771     __ sve_fabs(as_FloatRegister($dst_src$$reg), __ D,
 772             as_PRegister($pg$$reg),
 773             as_FloatRegister($dst_src$$reg));
 774   %}
 775   ins_pipe(pipe_slow);
 776 %}
 777 
 778 // vector add
 779 
 780 instruct vaddB(vReg dst, vReg src1, vReg src2) %{
 781   predicate(UseSVE > 0);
 782   match(Set dst (AddVB src1 src2));
 783   ins_cost(SVE_COST);
 784   format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (B)" %}
 785   ins_encode %{
 786     __ sve_add(as_FloatRegister($dst$$reg), __ B,
 787          as_FloatRegister($src1$$reg),
 788          as_FloatRegister($src2$$reg));
 789   %}
 790   ins_pipe(pipe_slow);
 791 %}
 792 
 793 instruct vaddS(vReg dst, vReg src1, vReg src2) %{
 794   predicate(UseSVE > 0);
 795   match(Set dst (AddVS src1 src2));
 796   ins_cost(SVE_COST);
 797   format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (H)" %}
 798   ins_encode %{
 799     __ sve_add(as_FloatRegister($dst$$reg), __ H,
 800          as_FloatRegister($src1$$reg),
 801          as_FloatRegister($src2$$reg));
 802   %}
 803   ins_pipe(pipe_slow);
 804 %}
 805 
 806 instruct vaddI(vReg dst, vReg src1, vReg src2) %{
 807   predicate(UseSVE > 0);
 808   match(Set dst (AddVI src1 src2));
 809   ins_cost(SVE_COST);
 810   format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (S)" %}
 811   ins_encode %{
 812     __ sve_add(as_FloatRegister($dst$$reg), __ S,
 813          as_FloatRegister($src1$$reg),
 814          as_FloatRegister($src2$$reg));
 815   %}
 816   ins_pipe(pipe_slow);
 817 %}
 818 
 819 instruct vaddL(vReg dst, vReg src1, vReg src2) %{
 820   predicate(UseSVE > 0);
 821   match(Set dst (AddVL src1 src2));
 822   ins_cost(SVE_COST);
 823   format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (D)" %}
 824   ins_encode %{
 825     __ sve_add(as_FloatRegister($dst$$reg), __ D,
 826          as_FloatRegister($src1$$reg),
 827          as_FloatRegister($src2$$reg));
 828   %}
 829   ins_pipe(pipe_slow);
 830 %}
 831 
 832 instruct vaddF(vReg dst, vReg src1, vReg src2) %{
 833   predicate(UseSVE > 0);
 834   match(Set dst (AddVF src1 src2));
 835   ins_cost(SVE_COST);
 836   format %{ "sve_fadd $dst, $src1, $src2\t # vector (sve) (S)" %}
 837   ins_encode %{
 838     __ sve_fadd(as_FloatRegister($dst$$reg), __ S,
 839          as_FloatRegister($src1$$reg),
 840          as_FloatRegister($src2$$reg));
 841   %}
 842   ins_pipe(pipe_slow);
 843 %}
 844 
 845 instruct vaddD(vReg dst, vReg src1, vReg src2) %{
 846   predicate(UseSVE > 0);
 847   match(Set dst (AddVD src1 src2));
 848   ins_cost(SVE_COST);
 849   format %{ "sve_fadd $dst, $src1, $src2\t # vector (sve) (D)" %}
 850   ins_encode %{
 851     __ sve_fadd(as_FloatRegister($dst$$reg), __ D,
 852          as_FloatRegister($src1$$reg),
 853          as_FloatRegister($src2$$reg));
 854   %}
 855   ins_pipe(pipe_slow);
 856 %}
 857 
 858 // vector add - predicated
 859 
 860 instruct vaddB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 861   predicate(UseSVE > 0);
 862   match(Set dst_src1 (AddVB (Binary dst_src1 src2) pg));
 863   ins_cost(SVE_COST);
 864   format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
 865   ins_encode %{
 866     __ sve_add(as_FloatRegister($dst_src1$$reg), __ B,
 867             as_PRegister($pg$$reg),
 868             as_FloatRegister($src2$$reg));
 869   %}
 870   ins_pipe(pipe_slow);
 871 %}
 872 
 873 instruct vaddS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 874   predicate(UseSVE > 0);
 875   match(Set dst_src1 (AddVS (Binary dst_src1 src2) pg));
 876   ins_cost(SVE_COST);
 877   format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
 878   ins_encode %{
 879     __ sve_add(as_FloatRegister($dst_src1$$reg), __ H,
 880             as_PRegister($pg$$reg),
 881             as_FloatRegister($src2$$reg));
 882   %}
 883   ins_pipe(pipe_slow);
 884 %}
 885 
 886 instruct vaddI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 887   predicate(UseSVE > 0);
 888   match(Set dst_src1 (AddVI (Binary dst_src1 src2) pg));
 889   ins_cost(SVE_COST);
 890   format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
 891   ins_encode %{
 892     __ sve_add(as_FloatRegister($dst_src1$$reg), __ S,
 893             as_PRegister($pg$$reg),
 894             as_FloatRegister($src2$$reg));
 895   %}
 896   ins_pipe(pipe_slow);
 897 %}
 898 
 899 instruct vaddL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 900   predicate(UseSVE > 0);
 901   match(Set dst_src1 (AddVL (Binary dst_src1 src2) pg));
 902   ins_cost(SVE_COST);
 903   format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
 904   ins_encode %{
 905     __ sve_add(as_FloatRegister($dst_src1$$reg), __ D,
 906             as_PRegister($pg$$reg),
 907             as_FloatRegister($src2$$reg));
 908   %}
 909   ins_pipe(pipe_slow);
 910 %}
 911 
 912 instruct vaddF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 913   predicate(UseSVE > 0);
 914   match(Set dst_src1 (AddVF (Binary dst_src1 src2) pg));
 915   ins_cost(SVE_COST);
 916   format %{ "sve_fadd $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
 917   ins_encode %{
 918     __ sve_fadd(as_FloatRegister($dst_src1$$reg), __ S,
 919             as_PRegister($pg$$reg),
 920             as_FloatRegister($src2$$reg));
 921   %}
 922   ins_pipe(pipe_slow);
 923 %}
 924 
 925 instruct vaddD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 926   predicate(UseSVE > 0);
 927   match(Set dst_src1 (AddVD (Binary dst_src1 src2) pg));
 928   ins_cost(SVE_COST);
 929   format %{ "sve_fadd $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
 930   ins_encode %{
 931     __ sve_fadd(as_FloatRegister($dst_src1$$reg), __ D,
 932             as_PRegister($pg$$reg),
 933             as_FloatRegister($src2$$reg));
 934   %}
 935   ins_pipe(pipe_slow);
 936 %}
 937 
 938 // vector and
 939 
 940 instruct vand(vReg dst, vReg src1, vReg src2) %{
 941   predicate(UseSVE > 0);
 942   match(Set dst (AndV src1 src2));
 943   ins_cost(SVE_COST);
 944   format %{ "sve_and  $dst, $src1, $src2\t# vector (sve)" %}
 945   ins_encode %{
 946     __ sve_and(as_FloatRegister($dst$$reg),
 947          as_FloatRegister($src1$$reg),
 948          as_FloatRegister($src2$$reg));
 949   %}
 950   ins_pipe(pipe_slow);
 951 %}
 952 
 953 // vector or
 954 
 955 instruct vor(vReg dst, vReg src1, vReg src2) %{
 956   predicate(UseSVE > 0);
 957   match(Set dst (OrV src1 src2));
 958   ins_cost(SVE_COST);
 959   format %{ "sve_orr  $dst, $src1, $src2\t# vector (sve)" %}
 960   ins_encode %{
 961     __ sve_orr(as_FloatRegister($dst$$reg),
 962          as_FloatRegister($src1$$reg),
 963          as_FloatRegister($src2$$reg));
 964   %}
 965   ins_pipe(pipe_slow);
 966 %}
 967 
 968 // vector xor
 969 
 970 instruct vxor(vReg dst, vReg src1, vReg src2) %{
 971   predicate(UseSVE > 0);
 972   match(Set dst (XorV src1 src2));
 973   ins_cost(SVE_COST);
 974   format %{ "sve_eor  $dst, $src1, $src2\t# vector (sve)" %}
 975   ins_encode %{
 976     __ sve_eor(as_FloatRegister($dst$$reg),
 977          as_FloatRegister($src1$$reg),
 978          as_FloatRegister($src2$$reg));
 979   %}
 980   ins_pipe(pipe_slow);
 981 %}
 982 
 983 // vector and - predicated
 984 
 985 instruct vand_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 986   predicate(UseSVE > 0);
 987   match(Set dst_src1 (AndV (Binary dst_src1 src2) pg));
 988   ins_cost(SVE_COST);
 989   format %{ "sve_and $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
 990   ins_encode %{
 991     BasicType bt = Matcher::vector_element_basic_type(this);
 992     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
 993     __ sve_and(as_FloatRegister($dst_src1$$reg), size,
 994           as_PRegister($pg$$reg),
 995           as_FloatRegister($src2$$reg));
 996   %}
 997   ins_pipe(pipe_slow);
 998 %}
 999 
1000 // vector or - predicated
1001 
1002 instruct vor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1003   predicate(UseSVE > 0);
1004   match(Set dst_src1 (OrV (Binary dst_src1 src2) pg));
1005   ins_cost(SVE_COST);
1006   format %{ "sve_orr $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
1007   ins_encode %{
1008     BasicType bt = Matcher::vector_element_basic_type(this);
1009     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1010     __ sve_orr(as_FloatRegister($dst_src1$$reg), size,
1011           as_PRegister($pg$$reg),
1012           as_FloatRegister($src2$$reg));
1013   %}
1014   ins_pipe(pipe_slow);
1015 %}
1016 
1017 // vector xor - predicated
1018 
1019 instruct vxor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1020   predicate(UseSVE > 0);
1021   match(Set dst_src1 (XorV (Binary dst_src1 src2) pg));
1022   ins_cost(SVE_COST);
1023   format %{ "sve_eor $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
1024   ins_encode %{
1025     BasicType bt = Matcher::vector_element_basic_type(this);
1026     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1027     __ sve_eor(as_FloatRegister($dst_src1$$reg), size,
1028           as_PRegister($pg$$reg),
1029           as_FloatRegister($src2$$reg));
1030   %}
1031   ins_pipe(pipe_slow);
1032 %}
1033 
1034 // vector not
1035 
1036 instruct vnotI(vReg dst, vReg src, immI_M1 m1) %{
1037   predicate(UseSVE > 0);
1038   match(Set dst (XorV src (ReplicateB m1)));
1039   match(Set dst (XorV src (ReplicateS m1)));
1040   match(Set dst (XorV src (ReplicateI m1)));
1041   ins_cost(SVE_COST);
1042   format %{ "sve_not $dst, $src\t# vector (sve) B/H/S" %}
1043   ins_encode %{
1044     __ sve_not(as_FloatRegister($dst$$reg), __ D,
1045                ptrue, as_FloatRegister($src$$reg));
1046   %}
1047   ins_pipe(pipe_slow);
1048 %}
1049 
1050 instruct vnotL(vReg dst, vReg src, immL_M1 m1) %{
1051   predicate(UseSVE > 0);
1052   match(Set dst (XorV src (ReplicateL m1)));
1053   ins_cost(SVE_COST);
1054   format %{ "sve_not $dst, $src\t# vector (sve) D" %}
1055   ins_encode %{
1056     __ sve_not(as_FloatRegister($dst$$reg), __ D,
1057                ptrue, as_FloatRegister($src$$reg));
1058   %}
1059   ins_pipe(pipe_slow);
1060 %}
1061 
1062 // vector and_not
1063 
1064 instruct vand_notI(vReg dst, vReg src1, vReg src2, immI_M1 m1) %{
1065   predicate(UseSVE > 0);
1066   match(Set dst (AndV src1 (XorV src2 (ReplicateB m1))));
1067   match(Set dst (AndV src1 (XorV src2 (ReplicateS m1))));
1068   match(Set dst (AndV src1 (XorV src2 (ReplicateI m1))));
1069   ins_cost(SVE_COST);
1070   format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) B/H/S" %}
1071   ins_encode %{
1072     __ sve_bic(as_FloatRegister($dst$$reg),
1073                as_FloatRegister($src1$$reg),
1074                as_FloatRegister($src2$$reg));
1075   %}
1076   ins_pipe(pipe_slow);
1077 %}
1078 
1079 instruct vand_notL(vReg dst, vReg src1, vReg src2, immL_M1 m1) %{
1080   predicate(UseSVE > 0);
1081   match(Set dst (AndV src1 (XorV src2 (ReplicateL m1))));
1082   ins_cost(SVE_COST);
1083   format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) D" %}
1084   ins_encode %{
1085     __ sve_bic(as_FloatRegister($dst$$reg),
1086                as_FloatRegister($src1$$reg),
1087                as_FloatRegister($src2$$reg));
1088   %}
1089   ins_pipe(pipe_slow);
1090 %}
1091 
1092 // vector float div
1093 
1094 instruct vdivF(vReg dst_src1, vReg src2) %{
1095   predicate(UseSVE > 0);
1096   match(Set dst_src1 (DivVF dst_src1 src2));
1097   ins_cost(SVE_COST);
1098   format %{ "sve_fdiv  $dst_src1, $dst_src1, $src2\t# vector (sve) (S)" %}
1099   ins_encode %{
1100     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ S,
1101          ptrue, as_FloatRegister($src2$$reg));
1102   %}
1103   ins_pipe(pipe_slow);
1104 %}
1105 
1106 instruct vdivD(vReg dst_src1, vReg src2) %{
1107   predicate(UseSVE > 0);
1108   match(Set dst_src1 (DivVD dst_src1 src2));
1109   ins_cost(SVE_COST);
1110   format %{ "sve_fdiv  $dst_src1, $dst_src1, $src2\t# vector (sve) (D)" %}
1111   ins_encode %{
1112     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ D,
1113          ptrue, as_FloatRegister($src2$$reg));
1114   %}
1115   ins_pipe(pipe_slow);
1116 %}
1117 
1118 // vector float div - predicated
1119 
1120 instruct vfdivF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1121   predicate(UseSVE > 0);
1122   match(Set dst_src1 (DivVF (Binary dst_src1 src2) pg));
1123   ins_cost(SVE_COST);
1124   format %{ "sve_fdiv $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
1125   ins_encode %{
1126     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ S,
1127             as_PRegister($pg$$reg),
1128             as_FloatRegister($src2$$reg));
1129   %}
1130   ins_pipe(pipe_slow);
1131 %}
1132 
1133 instruct vfdivD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1134   predicate(UseSVE > 0);
1135   match(Set dst_src1 (DivVD (Binary dst_src1 src2) pg));
1136   ins_cost(SVE_COST);
1137   format %{ "sve_fdiv $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
1138   ins_encode %{
1139     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ D,
1140             as_PRegister($pg$$reg),
1141             as_FloatRegister($src2$$reg));
1142   %}
1143   ins_pipe(pipe_slow);
1144 %}
1145 
1146 // vector min/max
1147 
1148 instruct vmin(vReg dst_src1, vReg src2) %{
1149   predicate(UseSVE > 0);
1150   match(Set dst_src1 (MinV dst_src1 src2));
1151   ins_cost(SVE_COST);
1152   format %{ "sve_min $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
1153   ins_encode %{
1154     BasicType bt = Matcher::vector_element_basic_type(this);
1155     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1156     if (is_floating_point_type(bt)) {
1157       __ sve_fmin(as_FloatRegister($dst_src1$$reg), size,
1158                   ptrue, as_FloatRegister($src2$$reg));
1159     } else {
1160       assert(is_integral_type(bt), "unsupported type");
1161       __ sve_smin(as_FloatRegister($dst_src1$$reg), size,
1162                   ptrue, as_FloatRegister($src2$$reg));
1163     }
1164   %}
1165   ins_pipe(pipe_slow);
1166 %}
1167 
1168 instruct vmax(vReg dst_src1, vReg src2) %{
1169   predicate(UseSVE > 0);
1170   match(Set dst_src1 (MaxV dst_src1 src2));
1171   ins_cost(SVE_COST);
1172   format %{ "sve_max $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
1173   ins_encode %{
1174     BasicType bt = Matcher::vector_element_basic_type(this);
1175     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1176     if (is_floating_point_type(bt)) {
1177       __ sve_fmax(as_FloatRegister($dst_src1$$reg), size,
1178                   ptrue, as_FloatRegister($src2$$reg));
1179     } else {
1180       assert(is_integral_type(bt), "unsupported type");
1181       __ sve_smax(as_FloatRegister($dst_src1$$reg), size,
1182                   ptrue, as_FloatRegister($src2$$reg));
1183     }
1184   %}
1185   ins_pipe(pipe_slow);
1186 %}
1187 
1188 // vector min/max - predicated
1189 
1190 instruct vmin_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1191   predicate(UseSVE > 0);
1192   match(Set dst_src1 (MinV (Binary dst_src1 src2) pg));
1193   ins_cost(SVE_COST);
1194   format %{ "sve_min $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %}
1195   ins_encode %{
1196     BasicType bt = Matcher::vector_element_basic_type(this);
1197     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1198     if (is_floating_point_type(bt)) {
1199       __ sve_fmin(as_FloatRegister($dst_src1$$reg), size,
1200                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1201     } else {
1202       assert(is_integral_type(bt), "unsupported type");
1203       __ sve_smin(as_FloatRegister($dst_src1$$reg), size,
1204                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1205     }
1206   %}
1207   ins_pipe(pipe_slow);
1208 %}
1209 
1210 instruct vmax_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1211   predicate(UseSVE > 0);
1212   match(Set dst_src1 (MaxV (Binary dst_src1 src2) pg));
1213   ins_cost(SVE_COST);
1214   format %{ "sve_max $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %}
1215   ins_encode %{
1216     BasicType bt = Matcher::vector_element_basic_type(this);
1217     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1218     if (is_floating_point_type(bt)) {
1219       __ sve_fmax(as_FloatRegister($dst_src1$$reg), size,
1220                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1221     } else {
1222       assert(is_integral_type(bt), "unsupported type");
1223       __ sve_smax(as_FloatRegister($dst_src1$$reg), size,
1224                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1225     }
1226   %}
1227   ins_pipe(pipe_slow);
1228 %}
1229 
1230 // vector fmla
1231 
1232 // dst_src1 = dst_src1 + src2 * src3
1233 instruct vfmlaF(vReg dst_src1, vReg src2, vReg src3) %{
1234   predicate(UseFMA && UseSVE > 0);
1235   match(Set dst_src1 (FmaVF dst_src1 (Binary src2 src3)));
1236   ins_cost(SVE_COST);
1237   format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1238   ins_encode %{
1239     __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ S,
1240          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1241   %}
1242   ins_pipe(pipe_slow);
1243 %}
1244 
1245 // dst_src1 = dst_src1 + src2 * src3
1246 instruct vfmlaD(vReg dst_src1, vReg src2, vReg src3) %{
1247   predicate(UseFMA && UseSVE > 0);
1248   match(Set dst_src1 (FmaVD dst_src1 (Binary src2 src3)));
1249   ins_cost(SVE_COST);
1250   format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1251   ins_encode %{
1252     __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ D,
1253          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1254   %}
1255   ins_pipe(pipe_slow);
1256 %}
1257 
1258 // vector fmla - predicated
1259 
1260 // dst_src1 = dst_src1 * src2 + src3
1261 instruct vfmlaF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1262   predicate(UseFMA && UseSVE > 0);
1263   match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary src3 pg)));
1264   ins_cost(SVE_COST);
1265   format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) (S)" %}
1266   ins_encode %{
1267     __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
1268          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1269   %}
1270   ins_pipe(pipe_slow);
1271 %}
1272 
1273 // dst_src1 = dst_src1 * src2 + src3
1274 instruct vfmlaD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1275   predicate(UseFMA && UseSVE > 0);
1276   match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary src3 pg)));
1277   ins_cost(SVE_COST);
1278   format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) (D)" %}
1279   ins_encode %{
1280     __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
1281          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1282   %}
1283   ins_pipe(pipe_slow);
1284 %}
1285 
1286 // vector fmls
1287 
1288 // dst_src1 = dst_src1 + -src2 * src3
1289 // dst_src1 = dst_src1 + src2 * -src3
1290 instruct vfmlsF(vReg dst_src1, vReg src2, vReg src3) %{
1291   predicate(UseFMA && UseSVE > 0);
1292   match(Set dst_src1 (FmaVF dst_src1 (Binary (NegVF src2) src3)));
1293   match(Set dst_src1 (FmaVF dst_src1 (Binary src2 (NegVF src3))));
1294   ins_cost(SVE_COST);
1295   format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1296   ins_encode %{
1297     __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ S,
1298          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1299   %}
1300   ins_pipe(pipe_slow);
1301 %}
1302 
1303 // dst_src1 = dst_src1 + -src2 * src3
1304 // dst_src1 = dst_src1 + src2 * -src3
1305 instruct vfmlsD(vReg dst_src1, vReg src2, vReg src3) %{
1306   predicate(UseFMA && UseSVE > 0);
1307   match(Set dst_src1 (FmaVD dst_src1 (Binary (NegVD src2) src3)));
1308   match(Set dst_src1 (FmaVD dst_src1 (Binary src2 (NegVD src3))));
1309   ins_cost(SVE_COST);
1310   format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1311   ins_encode %{
1312     __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ D,
1313          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1314   %}
1315   ins_pipe(pipe_slow);
1316 %}
1317 
1318 // vector fnmla
1319 
1320 // dst_src1 = -dst_src1 + -src2 * src3
1321 // dst_src1 = -dst_src1 + src2 * -src3
1322 instruct vfnmlaF(vReg dst_src1, vReg src2, vReg src3) %{
1323   predicate(UseFMA && UseSVE > 0);
1324   match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary (NegVF src2) src3)));
1325   match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 (NegVF src3))));
1326   ins_cost(SVE_COST);
1327   format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1328   ins_encode %{
1329     __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ S,
1330          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1331   %}
1332   ins_pipe(pipe_slow);
1333 %}
1334 
1335 // dst_src1 = -dst_src1 + -src2 * src3
1336 // dst_src1 = -dst_src1 + src2 * -src3
1337 instruct vfnmlaD(vReg dst_src1, vReg src2, vReg src3) %{
1338   predicate(UseFMA && UseSVE > 0);
1339   match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary (NegVD src2) src3)));
1340   match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 (NegVD src3))));
1341   ins_cost(SVE_COST);
1342   format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1343   ins_encode %{
1344     __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ D,
1345          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1346   %}
1347   ins_pipe(pipe_slow);
1348 %}
1349 
1350 // vector fnmls
1351 
1352 // dst_src1 = -dst_src1 + src2 * src3
1353 instruct vfnmlsF(vReg dst_src1, vReg src2, vReg src3) %{
1354   predicate(UseFMA && UseSVE > 0);
1355   match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 src3)));
1356   ins_cost(SVE_COST);
1357   format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1358   ins_encode %{
1359     __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ S,
1360          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1361   %}
1362   ins_pipe(pipe_slow);
1363 %}
1364 
1365 // dst_src1 = -dst_src1 + src2 * src3
1366 instruct vfnmlsD(vReg dst_src1, vReg src2, vReg src3) %{
1367   predicate(UseFMA && UseSVE > 0);
1368   match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 src3)));
1369   ins_cost(SVE_COST);
1370   format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1371   ins_encode %{
1372     __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ D,
1373          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1374   %}
1375   ins_pipe(pipe_slow);
1376 %}
1377 
1378 // vector mla
1379 
1380 // dst_src1 = dst_src1 + src2 * src3
1381 instruct vmlaB(vReg dst_src1, vReg src2, vReg src3)
1382 %{
1383   predicate(UseSVE > 0);
1384   match(Set dst_src1 (AddVB dst_src1 (MulVB src2 src3)));
1385   ins_cost(SVE_COST);
1386   format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (B)" %}
1387   ins_encode %{
1388     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ B,
1389       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1390   %}
1391   ins_pipe(pipe_slow);
1392 %}
1393 
1394 // dst_src1 = dst_src1 + src2 * src3
1395 instruct vmlaS(vReg dst_src1, vReg src2, vReg src3)
1396 %{
1397   predicate(UseSVE > 0);
1398   match(Set dst_src1 (AddVS dst_src1 (MulVS src2 src3)));
1399   ins_cost(SVE_COST);
1400   format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (H)" %}
1401   ins_encode %{
1402     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ H,
1403       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1404   %}
1405   ins_pipe(pipe_slow);
1406 %}
1407 
1408 // dst_src1 = dst_src1 + src2 * src3
1409 instruct vmlaI(vReg dst_src1, vReg src2, vReg src3)
1410 %{
1411   predicate(UseSVE > 0);
1412   match(Set dst_src1 (AddVI dst_src1 (MulVI src2 src3)));
1413   ins_cost(SVE_COST);
1414   format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (S)" %}
1415   ins_encode %{
1416     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ S,
1417       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1418   %}
1419   ins_pipe(pipe_slow);
1420 %}
1421 
1422 // dst_src1 = dst_src1 + src2 * src3
1423 instruct vmlaL(vReg dst_src1, vReg src2, vReg src3)
1424 %{
1425   predicate(UseSVE > 0);
1426   match(Set dst_src1 (AddVL dst_src1 (MulVL src2 src3)));
1427   ins_cost(SVE_COST);
1428   format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (D)" %}
1429   ins_encode %{
1430     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ D,
1431       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1432   %}
1433   ins_pipe(pipe_slow);
1434 %}
1435 
1436 // vector mls
1437 
1438 // dst_src1 = dst_src1 - src2 * src3
1439 instruct vmlsB(vReg dst_src1, vReg src2, vReg src3)
1440 %{
1441   predicate(UseSVE > 0);
1442   match(Set dst_src1 (SubVB dst_src1 (MulVB src2 src3)));
1443   ins_cost(SVE_COST);
1444   format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (B)" %}
1445   ins_encode %{
1446     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ B,
1447       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1448   %}
1449   ins_pipe(pipe_slow);
1450 %}
1451 
1452 // dst_src1 = dst_src1 - src2 * src3
1453 instruct vmlsS(vReg dst_src1, vReg src2, vReg src3)
1454 %{
1455   predicate(UseSVE > 0);
1456   match(Set dst_src1 (SubVS dst_src1 (MulVS src2 src3)));
1457   ins_cost(SVE_COST);
1458   format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (H)" %}
1459   ins_encode %{
1460     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ H,
1461       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1462   %}
1463   ins_pipe(pipe_slow);
1464 %}
1465 
1466 // dst_src1 = dst_src1 - src2 * src3
1467 instruct vmlsI(vReg dst_src1, vReg src2, vReg src3)
1468 %{
1469   predicate(UseSVE > 0);
1470   match(Set dst_src1 (SubVI dst_src1 (MulVI src2 src3)));
1471   ins_cost(SVE_COST);
1472   format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (S)" %}
1473   ins_encode %{
1474     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ S,
1475       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1476   %}
1477   ins_pipe(pipe_slow);
1478 %}
1479 
1480 // dst_src1 = dst_src1 - src2 * src3
1481 instruct vmlsL(vReg dst_src1, vReg src2, vReg src3)
1482 %{
1483   predicate(UseSVE > 0);
1484   match(Set dst_src1 (SubVL dst_src1 (MulVL src2 src3)));
1485   ins_cost(SVE_COST);
1486   format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (D)" %}
1487   ins_encode %{
1488     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ D,
1489       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1490   %}
1491   ins_pipe(pipe_slow);
1492 %}
1493 
1494 // vector mul
1495 
1496 instruct vmulB(vReg dst_src1, vReg src2) %{
1497   predicate(UseSVE > 0);
1498   match(Set dst_src1 (MulVB dst_src1 src2));
1499   ins_cost(SVE_COST);
1500   format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (B)" %}
1501   ins_encode %{
1502     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ B,
1503          ptrue, as_FloatRegister($src2$$reg));
1504   %}
1505   ins_pipe(pipe_slow);
1506 %}
1507 
1508 instruct vmulS(vReg dst_src1, vReg src2) %{
1509   predicate(UseSVE > 0);
1510   match(Set dst_src1 (MulVS dst_src1 src2));
1511   ins_cost(SVE_COST);
1512   format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (H)" %}
1513   ins_encode %{
1514     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ H,
1515          ptrue, as_FloatRegister($src2$$reg));
1516   %}
1517   ins_pipe(pipe_slow);
1518 %}
1519 
1520 instruct vmulI(vReg dst_src1, vReg src2) %{
1521   predicate(UseSVE > 0);
1522   match(Set dst_src1 (MulVI dst_src1 src2));
1523   ins_cost(SVE_COST);
1524   format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (S)" %}
1525   ins_encode %{
1526     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ S,
1527          ptrue, as_FloatRegister($src2$$reg));
1528   %}
1529   ins_pipe(pipe_slow);
1530 %}
1531 
1532 instruct vmulL(vReg dst_src1, vReg src2) %{
1533   predicate(UseSVE > 0);
1534   match(Set dst_src1 (MulVL dst_src1 src2));
1535   ins_cost(SVE_COST);
1536   format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (D)" %}
1537   ins_encode %{
1538     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ D,
1539          ptrue, as_FloatRegister($src2$$reg));
1540   %}
1541   ins_pipe(pipe_slow);
1542 %}
1543 
1544 instruct vmulF(vReg dst, vReg src1, vReg src2) %{
1545   predicate(UseSVE > 0);
1546   match(Set dst (MulVF src1 src2));
1547   ins_cost(SVE_COST);
1548   format %{ "sve_fmul $dst, $src1, $src2\t # vector (sve) (S)" %}
1549   ins_encode %{
1550     __ sve_fmul(as_FloatRegister($dst$$reg), __ S,
1551          as_FloatRegister($src1$$reg),
1552          as_FloatRegister($src2$$reg));
1553   %}
1554   ins_pipe(pipe_slow);
1555 %}
1556 
1557 instruct vmulD(vReg dst, vReg src1, vReg src2) %{
1558   predicate(UseSVE > 0);
1559   match(Set dst (MulVD src1 src2));
1560   ins_cost(SVE_COST);
1561   format %{ "sve_fmul $dst, $src1, $src2\t # vector (sve) (D)" %}
1562   ins_encode %{
1563     __ sve_fmul(as_FloatRegister($dst$$reg), __ D,
1564          as_FloatRegister($src1$$reg),
1565          as_FloatRegister($src2$$reg));
1566   %}
1567   ins_pipe(pipe_slow);
1568 %}
1569 
1570 // vector mul - predicated
1571 
1572 instruct vmulB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1573   predicate(UseSVE > 0);
1574   match(Set dst_src1 (MulVB (Binary dst_src1 src2) pg));
1575   ins_cost(SVE_COST);
1576   format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
1577   ins_encode %{
1578     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ B,
1579             as_PRegister($pg$$reg),
1580             as_FloatRegister($src2$$reg));
1581   %}
1582   ins_pipe(pipe_slow);
1583 %}
1584 
1585 instruct vmulS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1586   predicate(UseSVE > 0);
1587   match(Set dst_src1 (MulVS (Binary dst_src1 src2) pg));
1588   ins_cost(SVE_COST);
1589   format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
1590   ins_encode %{
1591     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ H,
1592             as_PRegister($pg$$reg),
1593             as_FloatRegister($src2$$reg));
1594   %}
1595   ins_pipe(pipe_slow);
1596 %}
1597 
1598 instruct vmulI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1599   predicate(UseSVE > 0);
1600   match(Set dst_src1 (MulVI (Binary dst_src1 src2) pg));
1601   ins_cost(SVE_COST);
1602   format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
1603   ins_encode %{
1604     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ S,
1605             as_PRegister($pg$$reg),
1606             as_FloatRegister($src2$$reg));
1607   %}
1608   ins_pipe(pipe_slow);
1609 %}
1610 
1611 instruct vmulL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1612   predicate(UseSVE > 0);
1613   match(Set dst_src1 (MulVL (Binary dst_src1 src2) pg));
1614   ins_cost(SVE_COST);
1615   format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
1616   ins_encode %{
1617     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ D,
1618             as_PRegister($pg$$reg),
1619             as_FloatRegister($src2$$reg));
1620   %}
1621   ins_pipe(pipe_slow);
1622 %}
1623 
1624 instruct vmulF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1625   predicate(UseSVE > 0);
1626   match(Set dst_src1 (MulVF (Binary dst_src1 src2) pg));
1627   ins_cost(SVE_COST);
1628   format %{ "sve_fmul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
1629   ins_encode %{
1630     __ sve_fmul(as_FloatRegister($dst_src1$$reg), __ S,
1631             as_PRegister($pg$$reg),
1632             as_FloatRegister($src2$$reg));
1633   %}
1634   ins_pipe(pipe_slow);
1635 %}
1636 
1637 instruct vmulD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1638   predicate(UseSVE > 0);
1639   match(Set dst_src1 (MulVD (Binary dst_src1 src2) pg));
1640   ins_cost(SVE_COST);
1641   format %{ "sve_fmul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
1642   ins_encode %{
1643     __ sve_fmul(as_FloatRegister($dst_src1$$reg), __ D,
1644             as_PRegister($pg$$reg),
1645             as_FloatRegister($src2$$reg));
1646   %}
1647   ins_pipe(pipe_slow);
1648 %}
1649 
1650 // vector fneg
1651 
1652 instruct vnegF(vReg dst, vReg src) %{
1653   predicate(UseSVE > 0 &&
1654             !n->as_Vector()->is_predicated_vector());
1655   match(Set dst (NegVF src));
1656   ins_cost(SVE_COST);
1657   format %{ "sve_fneg $dst, $src\t# vector (sve) (S)" %}
1658   ins_encode %{
1659     __ sve_fneg(as_FloatRegister($dst$$reg), __ S,
1660          ptrue, as_FloatRegister($src$$reg));
1661   %}
1662   ins_pipe(pipe_slow);
1663 %}
1664 
1665 instruct vnegD(vReg dst, vReg src) %{
1666   predicate(UseSVE > 0 &&
1667             !n->as_Vector()->is_predicated_vector());
1668   match(Set dst (NegVD src));
1669   ins_cost(SVE_COST);
1670   format %{ "sve_fneg $dst, $src\t# vector (sve) (D)" %}
1671   ins_encode %{
1672     __ sve_fneg(as_FloatRegister($dst$$reg), __ D,
1673          ptrue, as_FloatRegister($src$$reg));
1674   %}
1675   ins_pipe(pipe_slow);
1676 %}
1677 
1678 // vector fneg - predicated
1679 
1680 instruct vnegF_masked(vReg dst_src, pRegGov pg) %{
1681   predicate(UseSVE > 0);
1682   match(Set dst_src (NegVF dst_src pg));
1683   ins_cost(SVE_COST);
1684   format %{ "sve_fneg $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
1685   ins_encode %{
1686     __ sve_fneg(as_FloatRegister($dst_src$$reg), __ S,
1687             as_PRegister($pg$$reg),
1688             as_FloatRegister($dst_src$$reg));
1689   %}
1690   ins_pipe(pipe_slow);
1691 %}
1692 
1693 instruct vnegD_masked(vReg dst_src, pRegGov pg) %{
1694   predicate(UseSVE > 0);
1695   match(Set dst_src (NegVD dst_src pg));
1696   ins_cost(SVE_COST);
1697   format %{ "sve_fneg $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
1698   ins_encode %{
1699     __ sve_fneg(as_FloatRegister($dst_src$$reg), __ D,
1700             as_PRegister($pg$$reg),
1701             as_FloatRegister($dst_src$$reg));
1702   %}
1703   ins_pipe(pipe_slow);
1704 %}
1705 
1706 // popcount vector
1707 
1708 instruct vpopcountI(vReg dst, vReg src) %{
1709   predicate(UseSVE > 0);
1710   match(Set dst (PopCountVI src));
1711   format %{ "sve_cnt $dst, $src\t# vector (sve) (S)\n\t" %}
1712   ins_encode %{
1713      __ sve_cnt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg));
1714   %}
1715   ins_pipe(pipe_slow);
1716 %}
1717 
1718 // vector mask compare
1719 
1720 instruct vmaskcmp(pRegGov dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %{
1721   predicate(UseSVE > 0);
1722   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
1723   effect(KILL cr);
1724   ins_cost(SVE_COST);
1725   format %{ "sve_cmp $dst, $src1, $src2\t# vector mask cmp (sve)" %}
1726   ins_encode %{
1727     BasicType bt = Matcher::vector_element_basic_type(this);
1728     __ sve_compare(as_PRegister($dst$$reg), bt, ptrue, as_FloatRegister($src1$$reg),
1729                    as_FloatRegister($src2$$reg), (int)$cond$$constant);
1730   %}
1731   ins_pipe(pipe_slow);
1732 %}
1733 
1734 instruct vmaskcmp_masked(pRegGov dst, vReg src1, vReg src2, immI cond, pRegGov pg, rFlagsReg cr) %{
1735   predicate(UseSVE > 0);
1736   match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond pg)));
1737   effect(KILL cr);
1738   ins_cost(SVE_COST);
1739   format %{ "sve_cmp $dst, $pg, $src1, $src2\t# vector mask cmp (sve)" %}
1740   ins_encode %{
1741     BasicType bt = Matcher::vector_element_basic_type(this);
1742     __ sve_compare(as_PRegister($dst$$reg), bt, as_PRegister($pg$$reg), as_FloatRegister($src1$$reg),
1743                    as_FloatRegister($src2$$reg), (int)$cond$$constant);
1744   %}
1745   ins_pipe(pipe_slow);
1746 %}
1747 
1748 // vector blend
1749 
1750 instruct vblend(vReg dst, vReg src1, vReg src2, pRegGov pg) %{
1751   predicate(UseSVE > 0);
1752   match(Set dst (VectorBlend (Binary src1 src2) pg));
1753   ins_cost(SVE_COST);
1754   format %{ "sve_sel $dst, $pg, $src2, $src1\t# vector blend (sve)" %}
1755   ins_encode %{
1756     Assembler::SIMD_RegVariant size =
1757                __ elemType_to_regVariant(Matcher::vector_element_basic_type(this));
1758     __ sve_sel(as_FloatRegister($dst$$reg), size, as_PRegister($pg$$reg),
1759                as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
1760   %}
1761   ins_pipe(pipe_slow);
1762 %}
1763 
1764 // vector load mask
1765 
1766 instruct vloadmaskB(pRegGov dst, vReg src, rFlagsReg cr) %{
1767   predicate(UseSVE > 0 &&
1768             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
1769   match(Set dst (VectorLoadMask src));
1770   effect(KILL cr);
1771   ins_cost(SVE_COST);
1772   format %{ "vloadmaskB $dst, $src\t# vector load mask (sve) (B)" %}
1773   ins_encode %{
1774     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ B,
1775                ptrue, as_FloatRegister($src$$reg), 0);
1776   %}
1777   ins_pipe(pipe_slow);
1778 %}
1779 
1780 instruct vloadmask_extend(pRegGov dst, vReg src, vReg tmp, rFlagsReg cr) %{
1781   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() != T_BYTE);
1782   match(Set dst (VectorLoadMask src));
1783   effect(TEMP tmp, KILL cr);
1784   ins_cost(3 * SVE_COST);
1785   format %{ "vloadmask $dst, $src\t# vector load mask (sve) (H/S/D)" %}
1786   ins_encode %{
1787     BasicType bt = Matcher::vector_element_basic_type(this);
1788     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1789     __ sve_vector_extend(as_FloatRegister($tmp$$reg), size, as_FloatRegister($src$$reg), __ B);
1790     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
1791   %}
1792   ins_pipe(pipe_slow);
1793 %}
1794 
1795 // vector store mask
1796 
1797 instruct vstoremaskB(vReg dst, pRegGov src, immI_1 size) %{
1798   predicate(UseSVE > 0);
1799   match(Set dst (VectorStoreMask src size));
1800   ins_cost(SVE_COST);
1801   format %{ "vstoremask $dst, $src\t# vector store mask (sve) (B)" %}
1802   ins_encode %{
1803     __ sve_cpy(as_FloatRegister($dst$$reg), __ B, as_PRegister($src$$reg), 1, false);
1804   %}
1805   ins_pipe(pipe_slow);
1806 %}
1807 
1808 instruct vstoremask_narrow(vReg dst, pRegGov src, vReg tmp, immI_gt_1 size) %{
1809   predicate(UseSVE > 0);
1810   match(Set dst (VectorStoreMask src size));
1811   effect(TEMP_DEF dst, TEMP tmp);
1812   ins_cost(3 * SVE_COST);
1813   format %{ "vstoremask $dst, $src\t# vector store mask (sve) (H/S/D)" %}
1814   ins_encode %{
1815     Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant((int)$size$$constant);
1816     __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($src$$reg), 1, false);
1817     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
1818                          as_FloatRegister($dst$$reg), size, as_FloatRegister($tmp$$reg));
1819   %}
1820   ins_pipe(pipe_slow);
1821 %}
1822 
1823 // Combine LoadVector+VectorLoadMask when the vector element type is not T_BYTE
1824 
1825 instruct vloadmask_loadV(pRegGov dst, indirect mem, vReg tmp, rFlagsReg cr) %{
1826   predicate(UseSVE > 0 &&
1827             n->as_Vector()->length_in_bytes() == MaxVectorSize &&
1828             type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
1829   match(Set dst (VectorLoadMask (LoadVector mem)));
1830   effect(TEMP tmp, KILL cr);
1831   ins_cost(3 * SVE_COST);
1832   format %{ "sve_ld1b $tmp, $mem\n\t"
1833             "sve_cmpne $dst, $tmp, 0\t# load vector mask (sve) (H/S/D)" %}
1834   ins_encode %{
1835     // Load mask values which are boolean type, and extend them to the
1836     // expected vector element type. Convert the vector to predicate.
1837     BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
1838     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($tmp$$reg),
1839                           ptrue, T_BOOLEAN, to_vect_bt, $mem->opcode(),
1840                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1841     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ elemType_to_regVariant(to_vect_bt),
1842                ptrue, as_FloatRegister($tmp$$reg), 0);
1843   %}
1844   ins_pipe(pipe_slow);
1845 %}
1846 
1847 instruct vloadmask_loadV_partial(pRegGov dst, indirect mem, vReg vtmp, pRegGov ptmp, rFlagsReg cr) %{
1848   predicate(UseSVE > 0 &&
1849             n->as_Vector()->length_in_bytes() > 16 &&
1850             n->as_Vector()->length_in_bytes() < MaxVectorSize &&
1851             type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
1852   match(Set dst (VectorLoadMask (LoadVector mem)));
1853   effect(TEMP vtmp, TEMP ptmp, KILL cr);
1854   ins_cost(6 * SVE_COST);
1855   format %{ "vloadmask_loadV $dst, $mem\t# load vector mask partial (sve) (H/S/D)" %}
1856   ins_encode %{
1857     // Load valid mask values which are boolean type, and extend them to the
1858     // expected vector element type. Convert the vector to predicate.
1859     BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
1860     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(to_vect_bt);
1861     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this));
1862     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($vtmp$$reg),
1863                           as_PRegister($ptmp$$reg), T_BOOLEAN, to_vect_bt, $mem->opcode(),
1864                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1865     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($vtmp$$reg), 0);
1866   %}
1867   ins_pipe(pipe_slow);
1868 %}
1869 
1870 // Combine VectorStoreMask+StoreVector when the vector element type is not T_BYTE
1871 
1872 instruct storeV_vstoremask(indirect mem, pRegGov src, vReg tmp, immI_gt_1 esize) %{
1873   predicate(UseSVE > 0 &&
1874             Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
1875   match(Set mem (StoreVector mem (VectorStoreMask src esize)));
1876   effect(TEMP tmp);
1877   ins_cost(3 * SVE_COST);
1878   format %{ "sve_cpy $tmp, $src, 1\n\t"
1879             "sve_st1b $tmp, $mem\t# store vector mask (sve) (H/S/D)" %}
1880   ins_encode %{
1881     BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
1882     assert(type2aelembytes(from_vect_bt) == (int)$esize$$constant, "unsupported type.");
1883     Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
1884     __ sve_cpy(as_FloatRegister($tmp$$reg), size, as_PRegister($src$$reg), 1, false);
1885     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($tmp$$reg),
1886                           ptrue, T_BOOLEAN, from_vect_bt, $mem->opcode(),
1887                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1888   %}
1889   ins_pipe(pipe_slow);
1890 %}
1891 
1892 instruct storeV_vstoremask_partial(indirect mem, pRegGov src, vReg vtmp,
1893                                    immI_gt_1 esize, pRegGov ptmp, rFlagsReg cr) %{
1894   predicate(UseSVE > 0 &&
1895             n->as_StoreVector()->memory_size() > 16 &&
1896             type2aelembytes(n->as_StoreVector()->vect_type()->element_basic_type()) > 1 &&
1897             Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
1898   match(Set mem (StoreVector mem (VectorStoreMask src esize)));
1899   effect(TEMP vtmp, TEMP ptmp, KILL cr);
1900   format %{ "storeV_vstoremask $src, $mem\t# store vector mask partial (sve) (H/S/D)" %}
1901   ins_cost(6 * SVE_COST);
1902   ins_encode %{
1903     // Convert the valid src predicate to vector, and store the vector
1904     // elements as boolean values.
1905     BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
1906     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(from_vect_bt);
1907     __ sve_cpy(as_FloatRegister($vtmp$$reg), size, as_PRegister($src$$reg), 1, false);
1908     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
1909     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($vtmp$$reg),
1910                           as_PRegister($ptmp$$reg), T_BOOLEAN, from_vect_bt, $mem->opcode(),
1911                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1912   %}
1913   ins_pipe(pipe_slow);
1914 %}
1915 
1916 // vector add reduction
1917 
1918 instruct reduce_addI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
1919   predicate(UseSVE > 0 &&
1920             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1921   match(Set dst (AddReductionVI src1 src2));
1922   effect(TEMP_DEF dst, TEMP tmp);
1923   ins_cost(SVE_COST);
1924   format %{ "sve_reduce_addI $dst, $src1, $src2\t# addI reduction (sve) (may extend)" %}
1925   ins_encode %{
1926     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1927     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1928                            $src1$$Register, as_FloatRegister($src2$$reg),
1929                            ptrue, as_FloatRegister($tmp$$reg));
1930   %}
1931   ins_pipe(pipe_slow);
1932 %}
1933 
1934 instruct reduce_addL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
1935   predicate(UseSVE > 0 &&
1936             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1937   match(Set dst (AddReductionVL src1 src2));
1938   effect(TEMP_DEF dst, TEMP tmp);
1939   ins_cost(SVE_COST);
1940   format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction (sve)" %}
1941   ins_encode %{
1942     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1943                            $src1$$Register, as_FloatRegister($src2$$reg),
1944                            ptrue, as_FloatRegister($tmp$$reg));
1945   %}
1946   ins_pipe(pipe_slow);
1947 %}
1948 
1949 instruct reduce_addF(vRegF src1_dst, vReg src2) %{
1950   predicate(UseSVE > 0 &&
1951             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1952   match(Set src1_dst (AddReductionVF src1_dst src2));
1953   ins_cost(SVE_COST);
1954   format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (S)" %}
1955   ins_encode %{
1956     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
1957          ptrue, as_FloatRegister($src2$$reg));
1958   %}
1959   ins_pipe(pipe_slow);
1960 %}
1961 
1962 instruct reduce_addD(vRegD src1_dst, vReg src2) %{
1963   predicate(UseSVE > 0 &&
1964             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1965   match(Set src1_dst (AddReductionVD src1_dst src2));
1966   ins_cost(SVE_COST);
1967   format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (D)" %}
1968   ins_encode %{
1969     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
1970          ptrue, as_FloatRegister($src2$$reg));
1971   %}
1972   ins_pipe(pipe_slow);
1973 %}
1974 
1975 instruct reduce_addI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1976                              pRegGov ptmp, rFlagsReg cr) %{
1977   predicate(UseSVE > 0 &&
1978             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1979   match(Set dst (AddReductionVI src1 src2));
1980   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1981   ins_cost(2 * SVE_COST);
1982   format %{ "sve_reduce_addI $dst, $src1, $src2\t# addI reduction partial (sve) (may extend)" %}
1983   ins_encode %{
1984     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1985     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1986     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1987                           Matcher::vector_length(this, $src2));
1988     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1989                            $src1$$Register, as_FloatRegister($src2$$reg),
1990                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1991   %}
1992   ins_pipe(pipe_slow);
1993 %}
1994 
1995 instruct reduce_addL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1996                              pRegGov ptmp, rFlagsReg cr) %{
1997   predicate(UseSVE > 0 &&
1998             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1999   match(Set dst (AddReductionVL src1 src2));
2000   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2001   ins_cost(2 * SVE_COST);
2002   format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction partial (sve)" %}
2003   ins_encode %{
2004     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2005                           Matcher::vector_length(this, $src2));
2006     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2007                            $src1$$Register, as_FloatRegister($src2$$reg),
2008                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2009   %}
2010   ins_pipe(pipe_slow);
2011 %}
2012 
2013 instruct reduce_addF_partial(vRegF src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
2014   predicate(UseSVE > 0 &&
2015             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2016   match(Set src1_dst (AddReductionVF src1_dst src2));
2017   ins_cost(SVE_COST);
2018   effect(TEMP ptmp, KILL cr);
2019   format %{ "sve_reduce_addF $src1_dst, $src1_dst, $src2\t# addF reduction partial (sve) (S)" %}
2020   ins_encode %{
2021     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2022                           Matcher::vector_length(this, $src2));
2023     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
2024                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2025   %}
2026   ins_pipe(pipe_slow);
2027 %}
2028 
2029 instruct reduce_addD_partial(vRegD src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
2030   predicate(UseSVE > 0 &&
2031             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2032   match(Set src1_dst (AddReductionVD src1_dst src2));
2033   ins_cost(SVE_COST);
2034   effect(TEMP ptmp, KILL cr);
2035   format %{ "sve_reduce_addD $src1_dst, $src1_dst, $src2\t# addD reduction partial (sve) (D)" %}
2036   ins_encode %{
2037     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2038                           Matcher::vector_length(this, $src2));
2039     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
2040                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2041   %}
2042   ins_pipe(pipe_slow);
2043 %}
2044 
2045 // vector add reduction - predicated
2046 
2047 instruct reduce_addI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2048   predicate(UseSVE > 0 &&
2049             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2050   match(Set dst (AddReductionVI (Binary src1 src2) pg));
2051   effect(TEMP_DEF dst, TEMP tmp);
2052   ins_cost(SVE_COST);
2053   format %{ "sve_reduce_addI $dst, $src1, $pg, $src2\t# addI reduction predicated (sve) (may extend)" %}
2054   ins_encode %{
2055     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2056     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2057                            $src1$$Register, as_FloatRegister($src2$$reg),
2058                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2059   %}
2060   ins_pipe(pipe_slow);
2061 %}
2062 
2063 instruct reduce_addL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2064   predicate(UseSVE > 0 &&
2065             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2066   match(Set dst (AddReductionVL (Binary src1 src2) pg));
2067   effect(TEMP_DEF dst, TEMP tmp);
2068   ins_cost(SVE_COST);
2069   format %{ "sve_reduce_addL $dst, $src1, $pg, $src2\t# addL reduction predicated (sve)" %}
2070   ins_encode %{
2071     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2072                            $src1$$Register, as_FloatRegister($src2$$reg),
2073                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2074   %}
2075   ins_pipe(pipe_slow);
2076 %}
2077 
2078 instruct reduce_addF_masked(vRegF src1_dst, vReg src2, pRegGov pg) %{
2079   predicate(UseSVE > 0 &&
2080             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2081   match(Set src1_dst (AddReductionVF (Binary src1_dst src2) pg));
2082   ins_cost(SVE_COST);
2083   format %{ "sve_reduce_addF $src1_dst, $pg, $src2\t# addF reduction predicated (sve)" %}
2084   ins_encode %{
2085     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
2086                  as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
2087   %}
2088   ins_pipe(pipe_slow);
2089 %}
2090 
2091 instruct reduce_addD_masked(vRegD src1_dst, vReg src2, pRegGov pg) %{
2092   predicate(UseSVE > 0 &&
2093             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2094   match(Set src1_dst (AddReductionVD (Binary src1_dst src2) pg));
2095   ins_cost(SVE_COST);
2096   format %{ "sve_reduce_addD $src1_dst, $pg, $src2\t# addD reduction predicated (sve)" %}
2097   ins_encode %{
2098     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
2099                  as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
2100   %}
2101   ins_pipe(pipe_slow);
2102 %}
2103 
2104 instruct reduce_addI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2105                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2106   predicate(UseSVE > 0 &&
2107             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2108   match(Set dst (AddReductionVI (Binary src1 src2) pg));
2109   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2110   ins_cost(3 * SVE_COST);
2111   format %{ "sve_reduce_addI $dst, $src1, $pg, $src2\t# addI reduction predicated partial (sve) (may extend)" %}
2112   ins_encode %{
2113     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2114     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2115     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2116                           Matcher::vector_length(this, $src2));
2117     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2118                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2119     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2120                            $src1$$Register, as_FloatRegister($src2$$reg),
2121                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2122   %}
2123   ins_pipe(pipe_slow);
2124 %}
2125 
2126 instruct reduce_addL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2127                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2128   predicate(UseSVE > 0 &&
2129             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2130   match(Set dst (AddReductionVL (Binary src1 src2) pg));
2131   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2132   ins_cost(3 * SVE_COST);
2133   format %{ "sve_reduce_addL $dst, $src1, $pg, $src2\t# addL reduction predicated partial (sve)" %}
2134   ins_encode %{
2135     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2136                           Matcher::vector_length(this, $src2));
2137     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2138                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2139     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2140                            $src1$$Register, as_FloatRegister($src2$$reg),
2141                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2142   %}
2143   ins_pipe(pipe_slow);
2144 %}
2145 
2146 instruct reduce_addF_masked_partial(vRegF src1_dst, vReg src2, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2147   predicate(UseSVE > 0 &&
2148             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2149   match(Set src1_dst (AddReductionVF (Binary src1_dst src2) pg));
2150   effect(TEMP ptmp, KILL cr);
2151   ins_cost(SVE_COST);
2152   format %{ "sve_reduce_addF $src1_dst, $pg, $src2\t# addF reduction predicated partial (sve)" %}
2153   ins_encode %{
2154     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2155                           Matcher::vector_length(this, $src2));
2156     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2157                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2158     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
2159                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2160   %}
2161   ins_pipe(pipe_slow);
2162 %}
2163 
2164 instruct reduce_addD_masked_partial(vRegD src1_dst, vReg src2, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2165   predicate(UseSVE > 0 &&
2166             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2167   match(Set src1_dst (AddReductionVD (Binary src1_dst src2) pg));
2168   effect(TEMP ptmp, KILL cr);
2169   ins_cost(SVE_COST);
2170   format %{ "sve_reduce_addD $src1_dst, $pg, $src2\t# addD reduction predicated partial (sve)" %}
2171   ins_encode %{
2172     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2173                           Matcher::vector_length(this, $src2));
2174     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2175                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2176     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
2177                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2178   %}
2179   ins_pipe(pipe_slow);
2180 %}
2181 
2182 // vector and reduction
2183 
2184 instruct reduce_andI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
2185   predicate(UseSVE > 0 &&
2186             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2187             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2188   match(Set dst (AndReductionV src1 src2));
2189   effect(TEMP_DEF dst, TEMP tmp);
2190   ins_cost(SVE_COST);
2191   format %{ "sve_reduce_andI $dst, $src1, $src2\t# andI reduction (sve) (may extend)" %}
2192   ins_encode %{
2193     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2194     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2195                            $src1$$Register, as_FloatRegister($src2$$reg),
2196                            ptrue, as_FloatRegister($tmp$$reg));
2197   %}
2198   ins_pipe(pipe_slow);
2199 %}
2200 
2201 instruct reduce_andL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
2202   predicate(UseSVE > 0 &&
2203             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2204             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2205   match(Set dst (AndReductionV src1 src2));
2206   effect(TEMP_DEF dst, TEMP tmp);
2207   ins_cost(SVE_COST);
2208   format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction (sve)" %}
2209   ins_encode %{
2210     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2211                            $src1$$Register, as_FloatRegister($src2$$reg),
2212                            ptrue, as_FloatRegister($tmp$$reg));
2213   %}
2214   ins_pipe(pipe_slow);
2215 %}
2216 
2217 instruct reduce_andI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2218                              pRegGov ptmp, rFlagsReg cr) %{
2219   predicate(UseSVE > 0 &&
2220             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2221             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2222   match(Set dst (AndReductionV src1 src2));
2223   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2224   ins_cost(2 * SVE_COST);
2225   format %{ "sve_reduce_andI $dst, $src1, $src2\t# andI reduction partial (sve) (may extend)" %}
2226   ins_encode %{
2227     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2228     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2229     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2230                           Matcher::vector_length(this, $src2));
2231     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2232                            $src1$$Register, as_FloatRegister($src2$$reg),
2233                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2234   %}
2235   ins_pipe(pipe_slow);
2236 %}
2237 
2238 instruct reduce_andL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2239                              pRegGov ptmp, rFlagsReg cr) %{
2240   predicate(UseSVE > 0 &&
2241             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2242             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2243   match(Set dst (AndReductionV src1 src2));
2244   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2245   ins_cost(2 * SVE_COST);
2246   format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction partial (sve)" %}
2247   ins_encode %{
2248     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2249                           Matcher::vector_length(this, $src2));
2250     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2251                            $src1$$Register, as_FloatRegister($src2$$reg),
2252                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2253   %}
2254   ins_pipe(pipe_slow);
2255 %}
2256 
2257 // vector and reduction - predicated
2258 
2259 instruct reduce_andI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2260   predicate(UseSVE > 0 &&
2261             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2262             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2263   match(Set dst (AndReductionV (Binary src1 src2) pg));
2264   effect(TEMP_DEF dst, TEMP tmp);
2265   ins_cost(SVE_COST);
2266   format %{ "sve_reduce_andI $dst, $src1, $pg, $src2\t# andI reduction predicated (sve) (may extend)" %}
2267   ins_encode %{
2268     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2269     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2270                            $src1$$Register, as_FloatRegister($src2$$reg),
2271                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2272   %}
2273   ins_pipe(pipe_slow);
2274 %}
2275 
2276 instruct reduce_andL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2277   predicate(UseSVE > 0 &&
2278             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2279             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2280   match(Set dst (AndReductionV (Binary src1 src2) pg));
2281   effect(TEMP_DEF dst, TEMP tmp);
2282   ins_cost(SVE_COST);
2283   format %{ "sve_reduce_andL $dst, $src1, $pg, $src2\t# andL reduction predicated (sve)" %}
2284   ins_encode %{
2285     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2286                            $src1$$Register, as_FloatRegister($src2$$reg),
2287                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2288   %}
2289   ins_pipe(pipe_slow);
2290 %}
2291 
2292 instruct reduce_andI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2293                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2294   predicate(UseSVE > 0 &&
2295             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2296             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2297   match(Set dst (AndReductionV (Binary src1 src2) pg));
2298   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2299   ins_cost(3 * SVE_COST);
2300   format %{ "sve_reduce_andI $dst, $src1, $pg, $src2\t# andI reduction predicated partial (sve) (may extend)" %}
2301   ins_encode %{
2302     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2303     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2304     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2305                           Matcher::vector_length(this, $src2));
2306     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2307                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2308     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2309                            $src1$$Register, as_FloatRegister($src2$$reg),
2310                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2311   %}
2312   ins_pipe(pipe_slow);
2313 %}
2314 
2315 instruct reduce_andL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2316                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2317   predicate(UseSVE > 0 &&
2318             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2319             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2320   match(Set dst (AndReductionV (Binary src1 src2) pg));
2321   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2322   ins_cost(3 * SVE_COST);
2323   format %{ "sve_reduce_andL $dst, $src1, $pg, $src2\t# andL reduction predicated partial (sve)" %}
2324   ins_encode %{
2325     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2326                           Matcher::vector_length(this, $src2));
2327     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2328                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2329     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2330                            $src1$$Register, as_FloatRegister($src2$$reg),
2331                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2332   %}
2333   ins_pipe(pipe_slow);
2334 %}
2335 
2336 // vector or reduction
2337 
2338 instruct reduce_orI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
2339   predicate(UseSVE > 0 &&
2340             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2341             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2342   match(Set dst (OrReductionV src1 src2));
2343   effect(TEMP_DEF dst, TEMP tmp);
2344   ins_cost(SVE_COST);
2345   format %{ "sve_reduce_orI $dst, $src1, $src2\t# orI reduction (sve) (may extend)" %}
2346   ins_encode %{
2347     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2348     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2349                            $src1$$Register, as_FloatRegister($src2$$reg),
2350                            ptrue, as_FloatRegister($tmp$$reg));
2351   %}
2352   ins_pipe(pipe_slow);
2353 %}
2354 
2355 instruct reduce_orL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
2356   predicate(UseSVE > 0 &&
2357             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2358             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2359   match(Set dst (OrReductionV src1 src2));
2360   effect(TEMP_DEF dst, TEMP tmp);
2361   ins_cost(SVE_COST);
2362   format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction (sve)" %}
2363   ins_encode %{
2364     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2365                            $src1$$Register, as_FloatRegister($src2$$reg),
2366                            ptrue, as_FloatRegister($tmp$$reg));
2367   %}
2368   ins_pipe(pipe_slow);
2369 %}
2370 
2371 instruct reduce_orI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2372                              pRegGov ptmp, rFlagsReg cr) %{
2373   predicate(UseSVE > 0 &&
2374             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2375             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2376   match(Set dst (OrReductionV src1 src2));
2377   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2378   ins_cost(2 * SVE_COST);
2379   format %{ "sve_reduce_orI $dst, $src1, $src2\t# orI reduction partial (sve) (may extend)" %}
2380   ins_encode %{
2381     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2382     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2383     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2384                           Matcher::vector_length(this, $src2));
2385     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2386                            $src1$$Register, as_FloatRegister($src2$$reg),
2387                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2388   %}
2389   ins_pipe(pipe_slow);
2390 %}
2391 
2392 instruct reduce_orL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2393                              pRegGov ptmp, rFlagsReg cr) %{
2394   predicate(UseSVE > 0 &&
2395             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2396             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2397   match(Set dst (OrReductionV src1 src2));
2398   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2399   ins_cost(2 * SVE_COST);
2400   format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction partial (sve)" %}
2401   ins_encode %{
2402     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2403                           Matcher::vector_length(this, $src2));
2404     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2405                            $src1$$Register, as_FloatRegister($src2$$reg),
2406                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2407   %}
2408   ins_pipe(pipe_slow);
2409 %}
2410 
2411 // vector or reduction - predicated
2412 
2413 instruct reduce_orI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2414   predicate(UseSVE > 0 &&
2415             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2416             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2417   match(Set dst (OrReductionV (Binary src1 src2) pg));
2418   effect(TEMP_DEF dst, TEMP tmp);
2419   ins_cost(SVE_COST);
2420   format %{ "sve_reduce_orI $dst, $src1, $pg, $src2\t# orI reduction predicated (sve) (may extend)" %}
2421   ins_encode %{
2422     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2423     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2424                            $src1$$Register, as_FloatRegister($src2$$reg),
2425                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2426   %}
2427   ins_pipe(pipe_slow);
2428 %}
2429 
2430 instruct reduce_orL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2431   predicate(UseSVE > 0 &&
2432             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2433             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2434   match(Set dst (OrReductionV (Binary src1 src2) pg));
2435   effect(TEMP_DEF dst, TEMP tmp);
2436   ins_cost(SVE_COST);
2437   format %{ "sve_reduce_orL $dst, $src1, $pg, $src2\t# orL reduction predicated (sve)" %}
2438   ins_encode %{
2439     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2440                            $src1$$Register, as_FloatRegister($src2$$reg),
2441                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2442   %}
2443   ins_pipe(pipe_slow);
2444 %}
2445 
2446 instruct reduce_orI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2447                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2448   predicate(UseSVE > 0 &&
2449             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2450             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2451   match(Set dst (OrReductionV (Binary src1 src2) pg));
2452   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2453   ins_cost(3 * SVE_COST);
2454   format %{ "sve_reduce_orI $dst, $src1, $pg, $src2\t# orI reduction predicated partial (sve) (may extend)" %}
2455   ins_encode %{
2456     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2457     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2458     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2459                           Matcher::vector_length(this, $src2));
2460     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2461                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2462     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2463                            $src1$$Register, as_FloatRegister($src2$$reg),
2464                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2465   %}
2466   ins_pipe(pipe_slow);
2467 %}
2468 
2469 instruct reduce_orL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2470                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2471   predicate(UseSVE > 0 &&
2472             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2473             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2474   match(Set dst (OrReductionV (Binary src1 src2) pg));
2475   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2476   ins_cost(3 * SVE_COST);
2477   format %{ "sve_reduce_orL $dst, $src1, $pg, $src2\t# orL reduction predicated partial (sve)" %}
2478   ins_encode %{
2479     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2480                           Matcher::vector_length(this, $src2));
2481     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2482                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2483     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2484                            $src1$$Register, as_FloatRegister($src2$$reg),
2485                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2486   %}
2487   ins_pipe(pipe_slow);
2488 %}
2489 
2490 // vector xor reduction
2491 
2492 instruct reduce_eorI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
2493   predicate(UseSVE > 0 &&
2494             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2495             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2496   match(Set dst (XorReductionV src1 src2));
2497   effect(TEMP_DEF dst, TEMP tmp);
2498   ins_cost(SVE_COST);
2499   format %{ "sve_reduce_eorI $dst, $src1, $src2\t# eorI reduction (sve) (may extend)" %}
2500   ins_encode %{
2501     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2502     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2503                            $src1$$Register, as_FloatRegister($src2$$reg),
2504                            ptrue, as_FloatRegister($tmp$$reg));
2505   %}
2506   ins_pipe(pipe_slow);
2507 %}
2508 
2509 instruct reduce_eorL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
2510   predicate(UseSVE > 0 &&
2511             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2512             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2513   match(Set dst (XorReductionV src1 src2));
2514   effect(TEMP_DEF dst, TEMP tmp);
2515   ins_cost(SVE_COST);
2516   format %{ "sve_reduce_eorL $dst, $src1, $src2\t# eorL reduction (sve)" %}
2517   ins_encode %{
2518     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2519                            $src1$$Register, as_FloatRegister($src2$$reg),
2520                            ptrue, as_FloatRegister($tmp$$reg));
2521   %}
2522   ins_pipe(pipe_slow);
2523 %}
2524 
2525 instruct reduce_eorI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2526                              pRegGov ptmp, rFlagsReg cr) %{
2527   predicate(UseSVE > 0 &&
2528             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2529             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2530   match(Set dst (XorReductionV src1 src2));
2531   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2532   ins_cost(2 * SVE_COST);
2533   format %{ "sve_reduce_eorI $dst, $src1, $src2\t# eorI reduction partial (sve) (may extend)" %}
2534   ins_encode %{
2535     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2536     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2537     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2538                           Matcher::vector_length(this, $src2));
2539     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2540                            $src1$$Register, as_FloatRegister($src2$$reg),
2541                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2542   %}
2543   ins_pipe(pipe_slow);
2544 %}
2545 
2546 instruct reduce_eorL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2547                              pRegGov ptmp, rFlagsReg cr) %{
2548   predicate(UseSVE > 0 &&
2549             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2550             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2551   match(Set dst (XorReductionV src1 src2));
2552   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2553   ins_cost(2 * SVE_COST);
2554   format %{ "sve_reduce_eorL $dst, $src1, $src2\t# eorL reduction partial (sve)" %}
2555   ins_encode %{
2556     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2557                           Matcher::vector_length(this, $src2));
2558     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2559                            $src1$$Register, as_FloatRegister($src2$$reg),
2560                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2561   %}
2562   ins_pipe(pipe_slow);
2563 %}
2564 
2565 // vector xor reduction - predicated
2566 
2567 instruct reduce_eorI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2568   predicate(UseSVE > 0 &&
2569             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2570             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2571   match(Set dst (XorReductionV (Binary src1 src2) pg));
2572   effect(TEMP_DEF dst, TEMP tmp);
2573   ins_cost(SVE_COST);
2574   format %{ "sve_reduce_eorI $dst, $src1, $pg, $src2\t# eorI reduction predicated (sve) (may extend)" %}
2575   ins_encode %{
2576     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2577     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2578                            $src1$$Register, as_FloatRegister($src2$$reg),
2579                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2580   %}
2581   ins_pipe(pipe_slow);
2582 %}
2583 
2584 instruct reduce_eorL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2585   predicate(UseSVE > 0 &&
2586             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2587             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2588   match(Set dst (XorReductionV (Binary src1 src2) pg));
2589   effect(TEMP_DEF dst, TEMP tmp);
2590   ins_cost(SVE_COST);
2591   format %{ "sve_reduce_eorL $dst, $src1, $pg, $src2\t# eorL reduction predicated (sve)" %}
2592   ins_encode %{
2593     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2594                            $src1$$Register, as_FloatRegister($src2$$reg),
2595                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2596   %}
2597   ins_pipe(pipe_slow);
2598 %}
2599 
2600 instruct reduce_eorI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2601                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2602   predicate(UseSVE > 0 &&
2603             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2604             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2605   match(Set dst (XorReductionV (Binary src1 src2) pg));
2606   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2607   ins_cost(3 * SVE_COST);
2608   format %{ "sve_reduce_eorI $dst, $src1, $pg, $src2\t# eorI reduction predicated partial (sve) (may extend)" %}
2609   ins_encode %{
2610     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2611     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2612     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2613                           Matcher::vector_length(this, $src2));
2614     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2615                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2616     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2617                            $src1$$Register, as_FloatRegister($src2$$reg),
2618                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2619   %}
2620   ins_pipe(pipe_slow);
2621 %}
2622 
2623 instruct reduce_eorL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2624                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2625   predicate(UseSVE > 0 &&
2626             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2627             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2628   match(Set dst (XorReductionV (Binary src1 src2) pg));
2629   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2630   ins_cost(3 * SVE_COST);
2631   format %{ "sve_reduce_eorL $dst, $src1, $pg, $src2\t# eorL reduction predicated partial (sve)" %}
2632   ins_encode %{
2633     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2634                           Matcher::vector_length(this, $src2));
2635     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2636                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2637     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2638                            $src1$$Register, as_FloatRegister($src2$$reg),
2639                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2640   %}
2641   ins_pipe(pipe_slow);
2642 %}
2643 
2644 // vector max reduction
2645 
2646 instruct reduce_maxI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
2647   predicate(UseSVE > 0 &&
2648             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2649             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2650             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
2651   match(Set dst (MaxReductionV src1 src2));
2652   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2653   ins_cost(SVE_COST);
2654   format %{ "sve_reduce_maxI $dst, $src1, $src2\t# maxI reduction (sve)" %}
2655   ins_encode %{
2656     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2657     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2658                            $src1$$Register, as_FloatRegister($src2$$reg),
2659                            ptrue, as_FloatRegister($tmp$$reg));
2660   %}
2661   ins_pipe(pipe_slow);
2662 %}
2663 
2664 instruct reduce_maxL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
2665   predicate(UseSVE > 0 &&
2666             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2667             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2668   match(Set dst (MaxReductionV src1 src2));
2669   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2670   ins_cost(SVE_COST);
2671   format %{ "sve_reduce_maxL $dst, $src1, $src2\t# maxL reduction (sve)" %}
2672   ins_encode %{
2673     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2674                            $src1$$Register, as_FloatRegister($src2$$reg),
2675                            ptrue, as_FloatRegister($tmp$$reg));
2676   %}
2677   ins_pipe(pipe_slow);
2678 %}
2679 
2680 instruct reduce_maxI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2681                              pRegGov ptmp, rFlagsReg cr) %{
2682   predicate(UseSVE > 0 &&
2683             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2684             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2685             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
2686   match(Set dst (MaxReductionV src1 src2));
2687   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2688   ins_cost(2 * SVE_COST);
2689   format %{ "sve_reduce_maxI $dst, $src1, $src2\t# maxI reduction partial (sve)" %}
2690   ins_encode %{
2691     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2692     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2693     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2694                           Matcher::vector_length(this, $src2));
2695     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2696                            $src1$$Register, as_FloatRegister($src2$$reg),
2697                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2698   %}
2699   ins_pipe(pipe_slow);
2700 %}
2701 
2702 instruct reduce_maxL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2703                              pRegGov ptmp, rFlagsReg cr) %{
2704   predicate(UseSVE > 0 &&
2705             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2706             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2707   match(Set dst (MaxReductionV src1 src2));
2708   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2709   ins_cost(2 * SVE_COST);
2710   format %{ "sve_reduce_maxL $dst, $src1, $src2\t# maxL reduction  partial (sve)" %}
2711   ins_encode %{
2712     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2713                           Matcher::vector_length(this, $src2));
2714     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2715                            $src1$$Register, as_FloatRegister($src2$$reg),
2716                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2717   %}
2718   ins_pipe(pipe_slow);
2719 %}
2720 
2721 instruct reduce_maxF(vRegF dst, vRegF src1, vReg src2) %{
2722   predicate(UseSVE > 0 &&
2723             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
2724             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2725   match(Set dst (MaxReductionV src1 src2));
2726   ins_cost(INSN_COST);
2727   effect(TEMP_DEF dst);
2728   format %{ "sve_reduce_maxF $dst, $src1, $src2\t# maxF reduction (sve)" %}
2729   ins_encode %{
2730     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src2$$reg));
2731     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2732   %}
2733   ins_pipe(pipe_slow);
2734 %}
2735 
2736 instruct reduce_maxF_partial(vRegF dst, vRegF src1, vReg src2,
2737                              pRegGov ptmp, rFlagsReg cr) %{
2738   predicate(UseSVE > 0 &&
2739             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
2740             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2741   match(Set dst (MaxReductionV src1 src2));
2742   ins_cost(INSN_COST);
2743   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
2744   format %{ "sve_reduce_maxF $dst, $src1, $src2\t# maxF reduction partial (sve)" %}
2745   ins_encode %{
2746     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2747                           Matcher::vector_length(this, $src2));
2748     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2749     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2750   %}
2751   ins_pipe(pipe_slow);
2752 %}
2753 
2754 instruct reduce_maxD(vRegD dst, vRegD src1, vReg src2) %{
2755   predicate(UseSVE > 0 &&
2756             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
2757             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2758   match(Set dst (MaxReductionV src1 src2));
2759   ins_cost(INSN_COST);
2760   effect(TEMP_DEF dst);
2761   format %{ "sve_reduce_maxD $dst, $src1, $src2\t# maxD reduction (sve)" %}
2762   ins_encode %{
2763     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
2764     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2765   %}
2766   ins_pipe(pipe_slow);
2767 %}
2768 
2769 instruct reduce_maxD_partial(vRegD dst, vRegD src1, vReg src2,
2770                              pRegGov ptmp, rFlagsReg cr) %{
2771   predicate(UseSVE > 0 &&
2772             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
2773             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2774   match(Set dst (MaxReductionV src1 src2));
2775   ins_cost(INSN_COST);
2776   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
2777   format %{ "sve_reduce_maxD $dst, $src1, $src2\t# maxD reduction partial (sve)" %}
2778   ins_encode %{
2779     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2780                           Matcher::vector_length(this, $src2));
2781     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2782     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2783   %}
2784   ins_pipe(pipe_slow);
2785 %}
2786 
2787 // vector max reduction - predicated
2788 
2789 instruct reduce_maxI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp,
2790                            pRegGov pg, rFlagsReg cr) %{
2791   predicate(UseSVE > 0 &&
2792             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2793             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2794             is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
2795   match(Set dst (MaxReductionV (Binary src1 src2) pg));
2796   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2797   ins_cost(SVE_COST);
2798   format %{ "sve_reduce_maxI $dst, $src1, $pg, $src2\t# maxI reduction predicated (sve)" %}
2799   ins_encode %{
2800     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2801     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2802                            $src1$$Register, as_FloatRegister($src2$$reg),
2803                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2804   %}
2805   ins_pipe(pipe_slow);
2806 %}
2807 
2808 instruct reduce_maxL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp,
2809                           pRegGov pg, rFlagsReg cr) %{
2810   predicate(UseSVE > 0 &&
2811             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2812             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2813   match(Set dst (MaxReductionV (Binary src1 src2) pg));
2814   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2815   ins_cost(SVE_COST);
2816   format %{ "sve_reduce_maxL $dst, $src1, $pg, $src2\t# maxL reduction predicated (sve)" %}
2817   ins_encode %{
2818     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2819                            $src1$$Register, as_FloatRegister($src2$$reg),
2820                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2821   %}
2822   ins_pipe(pipe_slow);
2823 %}
2824 
2825 instruct reduce_maxI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2826                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2827   predicate(UseSVE > 0 &&
2828             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2829             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2830             is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
2831   match(Set dst (MaxReductionV (Binary src1 src2) pg));
2832   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2833   ins_cost(3 * SVE_COST);
2834   format %{ "sve_reduce_maxI $dst, $src1, $pg, $src2\t# maxI reduction predicated partial (sve)" %}
2835   ins_encode %{
2836     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2837     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2838     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2839                           Matcher::vector_length(this, $src2));
2840     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2841                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2842     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2843                            $src1$$Register, as_FloatRegister($src2$$reg),
2844                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2845   %}
2846   ins_pipe(pipe_slow);
2847 %}
2848 
2849 instruct reduce_maxL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2850                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2851   predicate(UseSVE > 0 &&
2852             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2853             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2854   match(Set dst (MaxReductionV (Binary src1 src2) pg));
2855   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2856   ins_cost(3 * SVE_COST);
2857   format %{ "sve_reduce_maxL $dst, $src1, $pg, $src2\t# maxL reduction predicated partial (sve)" %}
2858   ins_encode %{
2859     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2860                           Matcher::vector_length(this, $src2));
2861     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2862                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2863     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2864                            $src1$$Register, as_FloatRegister($src2$$reg),
2865                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2866   %}
2867   ins_pipe(pipe_slow);
2868 %}
2869 
2870 instruct reduce_maxF_masked(vRegF dst, vRegF src1, vReg src2, pRegGov pg) %{
2871   predicate(UseSVE > 0 &&
2872             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
2873             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2874   match(Set dst (MaxReductionV (Binary src1 src2) pg));
2875   ins_cost(SVE_COST);
2876   format %{ "sve_reduce_maxF $dst, $src1, $pg, $src2\t# maxF reduction predicated (sve)" %}
2877   ins_encode %{
2878     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
2879     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2880   %}
2881   ins_pipe(pipe_slow);
2882 %}
2883 
2884 instruct reduce_maxD_masked(vRegD dst, vRegD src1, vReg src2, pRegGov pg) %{
2885   predicate(UseSVE > 0 &&
2886             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
2887             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2888   match(Set dst (MaxReductionV (Binary src1 src2) pg));
2889   ins_cost(SVE_COST);
2890   format %{ "sve_reduce_maxD $dst, $src1, $pg, $src2\t# maxD reduction predicated (sve)" %}
2891   ins_encode %{
2892     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
2893     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2894   %}
2895   ins_pipe(pipe_slow);
2896 %}
2897 
2898 instruct reduce_maxF_masked_partial(vRegF dst, vRegF src1, vReg src2, pRegGov pg,
2899                                     pRegGov ptmp, rFlagsReg cr) %{
2900   predicate(UseSVE > 0 &&
2901             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
2902             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2903   match(Set dst (MaxReductionV (Binary src1 src2) pg));
2904   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
2905   ins_cost(3 * SVE_COST);
2906   format %{ "sve_reduce_maxF $dst, $src1, $pg, $src2\t# maxF reduction predicated partial (sve)" %}
2907   ins_encode %{
2908     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2909                           Matcher::vector_length(this, $src2));
2910     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2911                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2912     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S,
2913                as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2914     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2915   %}
2916   ins_pipe(pipe_slow);
2917 %}
2918 
2919 instruct reduce_maxD_masked_partial(vRegD dst, vRegD src1, vReg src2, pRegGov pg,
2920                                     pRegGov ptmp, rFlagsReg cr) %{
2921   predicate(UseSVE > 0 &&
2922             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
2923             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2924   match(Set dst (MaxReductionV (Binary src1 src2) pg));
2925   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
2926   ins_cost(3 * SVE_COST);
2927   format %{ "sve_reduce_maxD $dst, $src1, $pg, $src2\t# maxD reduction predicated partial (sve)" %}
2928   ins_encode %{
2929     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2930                           Matcher::vector_length(this, $src2));
2931     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2932                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2933     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D,
2934                as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2935     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2936   %}
2937   ins_pipe(pipe_slow);
2938 %}
2939 
2940 // vector min reduction
2941 
2942 instruct reduce_minI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
2943   predicate(UseSVE > 0 &&
2944             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2945             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2946             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
2947   match(Set dst (MinReductionV src1 src2));
2948   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2949   ins_cost(SVE_COST);
2950   format %{ "sve_reduce_minI $dst, $src1, $src2\t# minI reduction (sve)" %}
2951   ins_encode %{
2952     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2953     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2954                            $src1$$Register, as_FloatRegister($src2$$reg),
2955                            ptrue, as_FloatRegister($tmp$$reg));
2956   %}
2957   ins_pipe(pipe_slow);
2958 %}
2959 
2960 instruct reduce_minL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
2961   predicate(UseSVE > 0 &&
2962             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2963             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2964   match(Set dst (MinReductionV src1 src2));
2965   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2966   ins_cost(SVE_COST);
2967   format %{ "sve_reduce_minL $dst, $src1, $src2\t# minL reduction (sve)" %}
2968   ins_encode %{
2969     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2970                            $src1$$Register, as_FloatRegister($src2$$reg),
2971                            ptrue, as_FloatRegister($tmp$$reg));
2972   %}
2973   ins_pipe(pipe_slow);
2974 %}
2975 
2976 instruct reduce_minI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2977                              pRegGov ptmp, rFlagsReg cr) %{
2978   predicate(UseSVE > 0 &&
2979             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2980             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2981             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
2982   match(Set dst (MinReductionV src1 src2));
2983   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2984   ins_cost(2 * SVE_COST);
2985   format %{ "sve_reduce_minI $dst, $src1, $src2\t# minI reduction partial (sve)" %}
2986   ins_encode %{
2987     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2988     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2989     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2990                           Matcher::vector_length(this, $src2));
2991     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2992                            $src1$$Register, as_FloatRegister($src2$$reg),
2993                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2994   %}
2995   ins_pipe(pipe_slow);
2996 %}
2997 
2998 instruct reduce_minL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2999                              pRegGov ptmp, rFlagsReg cr) %{
3000   predicate(UseSVE > 0 &&
3001             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
3002             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3003   match(Set dst (MinReductionV src1 src2));
3004   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
3005   ins_cost(2 * SVE_COST);
3006   format %{ "sve_reduce_minL $dst, $src1, $src2\t# minL reduction  partial (sve)" %}
3007   ins_encode %{
3008     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
3009                           Matcher::vector_length(this, $src2));
3010     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
3011                            $src1$$Register, as_FloatRegister($src2$$reg),
3012                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
3013   %}
3014   ins_pipe(pipe_slow);
3015 %}
3016 
3017 instruct reduce_minF(vRegF dst, vRegF src1, vReg src2) %{
3018   predicate(UseSVE > 0 &&
3019             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3020             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3021   match(Set dst (MinReductionV src1 src2));
3022   ins_cost(INSN_COST);
3023   effect(TEMP_DEF dst);
3024   format %{ "sve_reduce_minF $dst, $src1, $src2\t# minF reduction (sve)" %}
3025   ins_encode %{
3026     __ sve_fminv(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src2$$reg));
3027     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3028   %}
3029   ins_pipe(pipe_slow);
3030 %}
3031 
3032 instruct reduce_minF_partial(vRegF dst, vRegF src1, vReg src2,
3033                              pRegGov ptmp, rFlagsReg cr) %{
3034   predicate(UseSVE > 0 &&
3035             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3036             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3037   match(Set dst (MinReductionV src1 src2));
3038   ins_cost(INSN_COST);
3039   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3040   format %{ "sve_reduce_minF $dst, $src1, $src2\t# minF reduction partial (sve)" %}
3041   ins_encode %{
3042     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
3043                           Matcher::vector_length(this, $src2));
3044     __ sve_fminv(as_FloatRegister($dst$$reg), __ S, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
3045     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3046   %}
3047   ins_pipe(pipe_slow);
3048 %}
3049 
3050 instruct reduce_minD(vRegD dst, vRegD src1, vReg src2) %{
3051   predicate(UseSVE > 0 &&
3052             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3053             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3054   match(Set dst (MinReductionV src1 src2));
3055   ins_cost(INSN_COST);
3056   effect(TEMP_DEF dst);
3057   format %{ "sve_reduce_minD $dst, $src1, $src2\t# minD reduction (sve)" %}
3058   ins_encode %{
3059     __ sve_fminv(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
3060     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3061   %}
3062   ins_pipe(pipe_slow);
3063 %}
3064 
3065 instruct reduce_minD_partial(vRegD dst, vRegD src1, vReg src2,
3066                              pRegGov ptmp, rFlagsReg cr) %{
3067   predicate(UseSVE > 0 &&
3068             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3069             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3070   match(Set dst (MinReductionV src1 src2));
3071   ins_cost(INSN_COST);
3072   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3073   format %{ "sve_reduce_minD $dst, $src1, $src2\t# minD reduction partial (sve)" %}
3074   ins_encode %{
3075     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
3076                           Matcher::vector_length(this, $src2));
3077     __ sve_fminv(as_FloatRegister($dst$$reg), __ D, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
3078     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3079   %}
3080   ins_pipe(pipe_slow);
3081 %}
3082 
3083 // vector min reduction - predicated
3084 
3085 instruct reduce_minI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp,
3086                            pRegGov pg, rFlagsReg cr) %{
3087   predicate(UseSVE > 0 &&
3088             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
3089             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
3090             is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
3091   match(Set dst (MinReductionV (Binary src1 src2) pg));
3092   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3093   ins_cost(SVE_COST);
3094   format %{ "sve_reduce_minI $dst, $src1, $pg, $src2\t# minI reduction predicated (sve)" %}
3095   ins_encode %{
3096     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
3097     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
3098                            $src1$$Register, as_FloatRegister($src2$$reg),
3099                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
3100   %}
3101   ins_pipe(pipe_slow);
3102 %}
3103 
3104 instruct reduce_minL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp,
3105                           pRegGov pg, rFlagsReg cr) %{
3106   predicate(UseSVE > 0 &&
3107             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
3108             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3109   match(Set dst (MinReductionV (Binary src1 src2) pg));
3110   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3111   ins_cost(SVE_COST);
3112   format %{ "sve_reduce_minL $dst, $src1, $pg, $src2\t# minL reduction predicated (sve)" %}
3113   ins_encode %{
3114     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
3115                            $src1$$Register, as_FloatRegister($src2$$reg),
3116                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
3117   %}
3118   ins_pipe(pipe_slow);
3119 %}
3120 
3121 instruct reduce_minI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
3122                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
3123   predicate(UseSVE > 0 &&
3124             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
3125             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
3126             is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
3127   match(Set dst (MinReductionV (Binary src1 src2) pg));
3128   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
3129   ins_cost(3 * SVE_COST);
3130   format %{ "sve_reduce_minI $dst, $src1, $pg, $src2\t# minI reduction predicated partial (sve)" %}
3131   ins_encode %{
3132     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
3133     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
3134     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
3135                           Matcher::vector_length(this, $src2));
3136     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
3137                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
3138     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
3139                            $src1$$Register, as_FloatRegister($src2$$reg),
3140                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
3141   %}
3142   ins_pipe(pipe_slow);
3143 %}
3144 
3145 instruct reduce_minL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
3146                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
3147   predicate(UseSVE > 0 &&
3148             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
3149             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3150   match(Set dst (MinReductionV (Binary src1 src2) pg));
3151   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
3152   ins_cost(3 * SVE_COST);
3153   format %{ "sve_reduce_minL $dst, $src1, $pg, $src2\t# minL reduction predicated partial (sve)" %}
3154   ins_encode %{
3155     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
3156                           Matcher::vector_length(this, $src2));
3157     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
3158                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
3159     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
3160                            $src1$$Register, as_FloatRegister($src2$$reg),
3161                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
3162   %}
3163   ins_pipe(pipe_slow);
3164 %}
3165 
3166 instruct reduce_minF_masked(vRegF dst, vRegF src1, vReg src2, pRegGov pg) %{
3167   predicate(UseSVE > 0 &&
3168             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3169             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3170   match(Set dst (MinReductionV (Binary src1 src2) pg));
3171   ins_cost(SVE_COST);
3172   format %{ "sve_reduce_minF $dst, $src1, $pg, $src2\t# minF reduction predicated (sve)" %}
3173   ins_encode %{
3174     __ sve_fminv(as_FloatRegister($dst$$reg), __ S, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
3175     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3176   %}
3177   ins_pipe(pipe_slow);
3178 %}
3179 
3180 instruct reduce_minD_masked(vRegD dst, vRegD src1, vReg src2, pRegGov pg) %{
3181   predicate(UseSVE > 0 &&
3182             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3183             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3184   match(Set dst (MinReductionV (Binary src1 src2) pg));
3185   ins_cost(SVE_COST);
3186   format %{ "sve_reduce_minD $dst, $src1, $pg, $src2\t# minD reduction predicated (sve)" %}
3187   ins_encode %{
3188     __ sve_fminv(as_FloatRegister($dst$$reg), __ D, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
3189     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3190   %}
3191   ins_pipe(pipe_slow);
3192 %}
3193 
3194 instruct reduce_minF_masked_partial(vRegF dst, vRegF src1, vReg src2, pRegGov pg,
3195                                     pRegGov ptmp, rFlagsReg cr) %{
3196   predicate(UseSVE > 0 &&
3197             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3198             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3199   match(Set dst (MinReductionV (Binary src1 src2) pg));
3200   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3201   ins_cost(3 * SVE_COST);
3202   format %{ "sve_reduce_minF $dst, $src1, $pg, $src2\t# minF reduction predicated partial (sve)" %}
3203   ins_encode %{
3204     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
3205                           Matcher::vector_length(this, $src2));
3206     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
3207                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
3208     __ sve_fminv(as_FloatRegister($dst$$reg), __ S,
3209                as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
3210     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3211   %}
3212   ins_pipe(pipe_slow);
3213 %}
3214 
3215 instruct reduce_minD_masked_partial(vRegD dst, vRegD src1, vReg src2, pRegGov pg,
3216                                     pRegGov ptmp, rFlagsReg cr) %{
3217   predicate(UseSVE > 0 &&
3218             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3219             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3220   match(Set dst (MinReductionV (Binary src1 src2) pg));
3221   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3222   ins_cost(3 * SVE_COST);
3223   format %{ "sve_reduce_minD $dst, $src1, $pg, $src2\t# minD reduction predicated partial (sve)" %}
3224   ins_encode %{
3225     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
3226                           Matcher::vector_length(this, $src2));
3227     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
3228                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
3229     __ sve_fminv(as_FloatRegister($dst$$reg), __ D,
3230                as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
3231     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3232   %}
3233   ins_pipe(pipe_slow);
3234 %}
3235 
3236 // vector Math.rint, floor, ceil
3237 
3238 instruct vroundD(vReg dst, vReg src, immI rmode) %{
3239   predicate(UseSVE > 0 &&
3240             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
3241   match(Set dst (RoundDoubleModeV src rmode));
3242   format %{ "sve_frint $dst, $src, $rmode\t# vector (sve) (D)" %}
3243   ins_encode %{
3244     switch ($rmode$$constant) {
3245       case RoundDoubleModeNode::rmode_rint:
3246         __ sve_frintn(as_FloatRegister($dst$$reg), __ D,
3247              ptrue, as_FloatRegister($src$$reg));
3248         break;
3249       case RoundDoubleModeNode::rmode_floor:
3250         __ sve_frintm(as_FloatRegister($dst$$reg), __ D,
3251              ptrue, as_FloatRegister($src$$reg));
3252         break;
3253       case RoundDoubleModeNode::rmode_ceil:
3254         __ sve_frintp(as_FloatRegister($dst$$reg), __ D,
3255              ptrue, as_FloatRegister($src$$reg));
3256         break;
3257     }
3258   %}
3259   ins_pipe(pipe_slow);
3260 %}
3261 
3262 // vector replicate
3263 
3264 instruct replicateB(vReg dst, iRegIorL2I src) %{
3265   predicate(UseSVE > 0);
3266   match(Set dst (ReplicateB src));
3267   ins_cost(SVE_COST);
3268   format %{ "sve_dup  $dst, $src\t# vector (sve) (B)" %}
3269   ins_encode %{
3270     __ sve_dup(as_FloatRegister($dst$$reg), __ B, as_Register($src$$reg));
3271   %}
3272   ins_pipe(pipe_slow);
3273 %}
3274 
3275 instruct replicateS(vReg dst, iRegIorL2I src) %{
3276   predicate(UseSVE > 0);
3277   match(Set dst (ReplicateS src));
3278   ins_cost(SVE_COST);
3279   format %{ "sve_dup  $dst, $src\t# vector (sve) (H)" %}
3280   ins_encode %{
3281     __ sve_dup(as_FloatRegister($dst$$reg), __ H, as_Register($src$$reg));
3282   %}
3283   ins_pipe(pipe_slow);
3284 %}
3285 
3286 instruct replicateI(vReg dst, iRegIorL2I src) %{
3287   predicate(UseSVE > 0);
3288   match(Set dst (ReplicateI src));
3289   ins_cost(SVE_COST);
3290   format %{ "sve_dup  $dst, $src\t# vector (sve) (S)" %}
3291   ins_encode %{
3292     __ sve_dup(as_FloatRegister($dst$$reg), __ S, as_Register($src$$reg));
3293   %}
3294   ins_pipe(pipe_slow);
3295 %}
3296 
3297 instruct replicateL(vReg dst, iRegL src) %{
3298   predicate(UseSVE > 0);
3299   match(Set dst (ReplicateL src));
3300   ins_cost(SVE_COST);
3301   format %{ "sve_dup  $dst, $src\t# vector (sve) (D)" %}
3302   ins_encode %{
3303     __ sve_dup(as_FloatRegister($dst$$reg), __ D, as_Register($src$$reg));
3304   %}
3305   ins_pipe(pipe_slow);
3306 %}
3307 
3308 instruct replicateB_imm8(vReg dst, immI8 con) %{
3309   predicate(UseSVE > 0);
3310   match(Set dst (ReplicateB con));
3311   ins_cost(SVE_COST);
3312   format %{ "sve_dup  $dst, $con\t# vector (sve) (B)" %}
3313   ins_encode %{
3314     __ sve_dup(as_FloatRegister($dst$$reg), __ B, $con$$constant);
3315   %}
3316   ins_pipe(pipe_slow);
3317 %}
3318 
3319 instruct replicateS_imm8(vReg dst, immI8_shift8 con) %{
3320   predicate(UseSVE > 0);
3321   match(Set dst (ReplicateS con));
3322   ins_cost(SVE_COST);
3323   format %{ "sve_dup  $dst, $con\t# vector (sve) (H)" %}
3324   ins_encode %{
3325     __ sve_dup(as_FloatRegister($dst$$reg), __ H, $con$$constant);
3326   %}
3327   ins_pipe(pipe_slow);
3328 %}
3329 
3330 instruct replicateI_imm8(vReg dst, immI8_shift8 con) %{
3331   predicate(UseSVE > 0);
3332   match(Set dst (ReplicateI con));
3333   ins_cost(SVE_COST);
3334   format %{ "sve_dup  $dst, $con\t# vector (sve) (S)" %}
3335   ins_encode %{
3336     __ sve_dup(as_FloatRegister($dst$$reg), __ S, $con$$constant);
3337   %}
3338   ins_pipe(pipe_slow);
3339 %}
3340 
3341 instruct replicateL_imm8(vReg dst, immL8_shift8 con) %{
3342   predicate(UseSVE > 0);
3343   match(Set dst (ReplicateL con));
3344   ins_cost(SVE_COST);
3345   format %{ "sve_dup  $dst, $con\t# vector (sve) (D)" %}
3346   ins_encode %{
3347     __ sve_dup(as_FloatRegister($dst$$reg), __ D, $con$$constant);
3348   %}
3349   ins_pipe(pipe_slow);
3350 %}
3351 
3352 instruct replicateF(vReg dst, vRegF src) %{
3353   predicate(UseSVE > 0);
3354   match(Set dst (ReplicateF src));
3355   ins_cost(SVE_COST);
3356   format %{ "sve_cpy  $dst, $src\t# vector (sve) (S)" %}
3357   ins_encode %{
3358     __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
3359          ptrue, as_FloatRegister($src$$reg));
3360   %}
3361   ins_pipe(pipe_slow);
3362 %}
3363 
3364 instruct replicateD(vReg dst, vRegD src) %{
3365   predicate(UseSVE > 0);
3366   match(Set dst (ReplicateD src));
3367   ins_cost(SVE_COST);
3368   format %{ "sve_cpy  $dst, $src\t# vector (sve) (D)" %}
3369   ins_encode %{
3370     __ sve_cpy(as_FloatRegister($dst$$reg), __ D,
3371          ptrue, as_FloatRegister($src$$reg));
3372   %}
3373   ins_pipe(pipe_slow);
3374 %}
3375 
3376 // vector shift
3377 
3378 instruct vasrB(vReg dst, vReg shift) %{
3379   predicate(UseSVE > 0);
3380   match(Set dst (RShiftVB dst shift));
3381   ins_cost(SVE_COST);
3382   format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (B)" %}
3383   ins_encode %{
3384     __ sve_asr(as_FloatRegister($dst$$reg), __ B,
3385          ptrue, as_FloatRegister($shift$$reg));
3386   %}
3387   ins_pipe(pipe_slow);
3388 %}
3389 
3390 instruct vasrS(vReg dst, vReg shift) %{
3391   predicate(UseSVE > 0);
3392   match(Set dst (RShiftVS dst shift));
3393   ins_cost(SVE_COST);
3394   format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (H)" %}
3395   ins_encode %{
3396     __ sve_asr(as_FloatRegister($dst$$reg), __ H,
3397          ptrue, as_FloatRegister($shift$$reg));
3398   %}
3399   ins_pipe(pipe_slow);
3400 %}
3401 
3402 instruct vasrI(vReg dst, vReg shift) %{
3403   predicate(UseSVE > 0);
3404   match(Set dst (RShiftVI dst shift));
3405   ins_cost(SVE_COST);
3406   format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (S)" %}
3407   ins_encode %{
3408     __ sve_asr(as_FloatRegister($dst$$reg), __ S,
3409          ptrue, as_FloatRegister($shift$$reg));
3410   %}
3411   ins_pipe(pipe_slow);
3412 %}
3413 
3414 instruct vasrL(vReg dst, vReg shift) %{
3415   predicate(UseSVE > 0);
3416   match(Set dst (RShiftVL dst shift));
3417   ins_cost(SVE_COST);
3418   format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (D)" %}
3419   ins_encode %{
3420     __ sve_asr(as_FloatRegister($dst$$reg), __ D,
3421          ptrue, as_FloatRegister($shift$$reg));
3422   %}
3423   ins_pipe(pipe_slow);
3424 %}
3425 
3426 instruct vlslB(vReg dst, vReg shift) %{
3427   predicate(UseSVE > 0);
3428   match(Set dst (LShiftVB dst shift));
3429   ins_cost(SVE_COST);
3430   format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (B)" %}
3431   ins_encode %{
3432     __ sve_lsl(as_FloatRegister($dst$$reg), __ B,
3433          ptrue, as_FloatRegister($shift$$reg));
3434   %}
3435   ins_pipe(pipe_slow);
3436 %}
3437 
3438 instruct vlslS(vReg dst, vReg shift) %{
3439   predicate(UseSVE > 0);
3440   match(Set dst (LShiftVS dst shift));
3441   ins_cost(SVE_COST);
3442   format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (H)" %}
3443   ins_encode %{
3444     __ sve_lsl(as_FloatRegister($dst$$reg), __ H,
3445          ptrue, as_FloatRegister($shift$$reg));
3446   %}
3447   ins_pipe(pipe_slow);
3448 %}
3449 
3450 instruct vlslI(vReg dst, vReg shift) %{
3451   predicate(UseSVE > 0);
3452   match(Set dst (LShiftVI dst shift));
3453   ins_cost(SVE_COST);
3454   format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (S)" %}
3455   ins_encode %{
3456     __ sve_lsl(as_FloatRegister($dst$$reg), __ S,
3457          ptrue, as_FloatRegister($shift$$reg));
3458   %}
3459   ins_pipe(pipe_slow);
3460 %}
3461 
3462 instruct vlslL(vReg dst, vReg shift) %{
3463   predicate(UseSVE > 0);
3464   match(Set dst (LShiftVL dst shift));
3465   ins_cost(SVE_COST);
3466   format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (D)" %}
3467   ins_encode %{
3468     __ sve_lsl(as_FloatRegister($dst$$reg), __ D,
3469          ptrue, as_FloatRegister($shift$$reg));
3470   %}
3471   ins_pipe(pipe_slow);
3472 %}
3473 
3474 instruct vlsrB(vReg dst, vReg shift) %{
3475   predicate(UseSVE > 0);
3476   match(Set dst (URShiftVB dst shift));
3477   ins_cost(SVE_COST);
3478   format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (B)" %}
3479   ins_encode %{
3480     __ sve_lsr(as_FloatRegister($dst$$reg), __ B,
3481          ptrue, as_FloatRegister($shift$$reg));
3482   %}
3483   ins_pipe(pipe_slow);
3484 %}
3485 
3486 instruct vlsrS(vReg dst, vReg shift) %{
3487   predicate(UseSVE > 0);
3488   match(Set dst (URShiftVS dst shift));
3489   ins_cost(SVE_COST);
3490   format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (H)" %}
3491   ins_encode %{
3492     __ sve_lsr(as_FloatRegister($dst$$reg), __ H,
3493          ptrue, as_FloatRegister($shift$$reg));
3494   %}
3495   ins_pipe(pipe_slow);
3496 %}
3497 
3498 instruct vlsrI(vReg dst, vReg shift) %{
3499   predicate(UseSVE > 0);
3500   match(Set dst (URShiftVI dst shift));
3501   ins_cost(SVE_COST);
3502   format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (S)" %}
3503   ins_encode %{
3504     __ sve_lsr(as_FloatRegister($dst$$reg), __ S,
3505          ptrue, as_FloatRegister($shift$$reg));
3506   %}
3507   ins_pipe(pipe_slow);
3508 %}
3509 
3510 instruct vlsrL(vReg dst, vReg shift) %{
3511   predicate(UseSVE > 0);
3512   match(Set dst (URShiftVL dst shift));
3513   ins_cost(SVE_COST);
3514   format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (D)" %}
3515   ins_encode %{
3516     __ sve_lsr(as_FloatRegister($dst$$reg), __ D,
3517          ptrue, as_FloatRegister($shift$$reg));
3518   %}
3519   ins_pipe(pipe_slow);
3520 %}
3521 
3522 instruct vasrB_imm(vReg dst, vReg src, immI shift) %{
3523   predicate(UseSVE > 0);
3524   match(Set dst (RShiftVB src (RShiftCntV shift)));
3525   ins_cost(SVE_COST);
3526   format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (B)" %}
3527   ins_encode %{
3528     int con = (int)$shift$$constant;
3529     if (con == 0) {
3530       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3531            as_FloatRegister($src$$reg));
3532       return;
3533     }
3534     if (con >= 8) con = 7;
3535     __ sve_asr(as_FloatRegister($dst$$reg), __ B,
3536          as_FloatRegister($src$$reg), con);
3537   %}
3538   ins_pipe(pipe_slow);
3539 %}
3540 
3541 instruct vasrS_imm(vReg dst, vReg src, immI shift) %{
3542   predicate(UseSVE > 0);
3543   match(Set dst (RShiftVS src (RShiftCntV shift)));
3544   ins_cost(SVE_COST);
3545   format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (H)" %}
3546   ins_encode %{
3547     int con = (int)$shift$$constant;
3548     if (con == 0) {
3549       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3550            as_FloatRegister($src$$reg));
3551       return;
3552     }
3553     if (con >= 16) con = 15;
3554     __ sve_asr(as_FloatRegister($dst$$reg), __ H,
3555          as_FloatRegister($src$$reg), con);
3556   %}
3557   ins_pipe(pipe_slow);
3558 %}
3559 
3560 instruct vasrI_imm(vReg dst, vReg src, immI shift) %{
3561   predicate(UseSVE > 0);
3562   match(Set dst (RShiftVI src (RShiftCntV shift)));
3563   ins_cost(SVE_COST);
3564   format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (S)" %}
3565   ins_encode %{
3566     int con = (int)$shift$$constant;
3567     if (con == 0) {
3568       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3569            as_FloatRegister($src$$reg));
3570       return;
3571     }
3572     __ sve_asr(as_FloatRegister($dst$$reg), __ S,
3573          as_FloatRegister($src$$reg), con);
3574   %}
3575   ins_pipe(pipe_slow);
3576 %}
3577 
3578 instruct vasrL_imm(vReg dst, vReg src, immI shift) %{
3579   predicate(UseSVE > 0);
3580   match(Set dst (RShiftVL src (RShiftCntV shift)));
3581   ins_cost(SVE_COST);
3582   format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (D)" %}
3583   ins_encode %{
3584     int con = (int)$shift$$constant;
3585     if (con == 0) {
3586       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3587            as_FloatRegister($src$$reg));
3588       return;
3589     }
3590     __ sve_asr(as_FloatRegister($dst$$reg), __ D,
3591          as_FloatRegister($src$$reg), con);
3592   %}
3593   ins_pipe(pipe_slow);
3594 %}
3595 
3596 instruct vlsrB_imm(vReg dst, vReg src, immI shift) %{
3597   predicate(UseSVE > 0);
3598   match(Set dst (URShiftVB src (RShiftCntV shift)));
3599   ins_cost(SVE_COST);
3600   format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (B)" %}
3601   ins_encode %{
3602     int con = (int)$shift$$constant;
3603     if (con == 0) {
3604       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3605            as_FloatRegister($src$$reg));
3606       return;
3607     }
3608     if (con >= 8) {
3609       __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3610            as_FloatRegister($src$$reg));
3611       return;
3612     }
3613     __ sve_lsr(as_FloatRegister($dst$$reg), __ B,
3614          as_FloatRegister($src$$reg), con);
3615   %}
3616   ins_pipe(pipe_slow);
3617 %}
3618 
3619 instruct vlsrS_imm(vReg dst, vReg src, immI shift) %{
3620   predicate(UseSVE > 0);
3621   match(Set dst (URShiftVS src (RShiftCntV shift)));
3622   ins_cost(SVE_COST);
3623   format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (H)" %}
3624   ins_encode %{
3625     int con = (int)$shift$$constant;
3626     if (con == 0) {
3627       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3628            as_FloatRegister($src$$reg));
3629       return;
3630     }
3631     if (con >= 16) {
3632       __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3633            as_FloatRegister($src$$reg));
3634       return;
3635     }
3636     __ sve_lsr(as_FloatRegister($dst$$reg), __ H,
3637          as_FloatRegister($src$$reg), con);
3638   %}
3639   ins_pipe(pipe_slow);
3640 %}
3641 
3642 instruct vlsrI_imm(vReg dst, vReg src, immI shift) %{
3643   predicate(UseSVE > 0);
3644   match(Set dst (URShiftVI src (RShiftCntV shift)));
3645   ins_cost(SVE_COST);
3646   format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (S)" %}
3647   ins_encode %{
3648     int con = (int)$shift$$constant;
3649     if (con == 0) {
3650       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3651            as_FloatRegister($src$$reg));
3652       return;
3653     }
3654     __ sve_lsr(as_FloatRegister($dst$$reg), __ S,
3655          as_FloatRegister($src$$reg), con);
3656   %}
3657   ins_pipe(pipe_slow);
3658 %}
3659 
3660 instruct vlsrL_imm(vReg dst, vReg src, immI shift) %{
3661   predicate(UseSVE > 0);
3662   match(Set dst (URShiftVL src (RShiftCntV shift)));
3663   ins_cost(SVE_COST);
3664   format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (D)" %}
3665   ins_encode %{
3666     int con = (int)$shift$$constant;
3667     if (con == 0) {
3668       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3669            as_FloatRegister($src$$reg));
3670       return;
3671     }
3672     __ sve_lsr(as_FloatRegister($dst$$reg), __ D,
3673          as_FloatRegister($src$$reg), con);
3674   %}
3675   ins_pipe(pipe_slow);
3676 %}
3677 
3678 instruct vlslB_imm(vReg dst, vReg src, immI shift) %{
3679   predicate(UseSVE > 0);
3680   match(Set dst (LShiftVB src (LShiftCntV shift)));
3681   ins_cost(SVE_COST);
3682   format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (B)" %}
3683   ins_encode %{
3684     int con = (int)$shift$$constant;
3685     if (con >= 8) {
3686       __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3687            as_FloatRegister($src$$reg));
3688       return;
3689     }
3690     __ sve_lsl(as_FloatRegister($dst$$reg), __ B,
3691          as_FloatRegister($src$$reg), con);
3692   %}
3693   ins_pipe(pipe_slow);
3694 %}
3695 
3696 instruct vlslS_imm(vReg dst, vReg src, immI shift) %{
3697   predicate(UseSVE > 0);
3698   match(Set dst (LShiftVS src (LShiftCntV shift)));
3699   ins_cost(SVE_COST);
3700   format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (H)" %}
3701   ins_encode %{
3702     int con = (int)$shift$$constant;
3703     if (con >= 16) {
3704       __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3705            as_FloatRegister($src$$reg));
3706       return;
3707     }
3708     __ sve_lsl(as_FloatRegister($dst$$reg), __ H,
3709          as_FloatRegister($src$$reg), con);
3710   %}
3711   ins_pipe(pipe_slow);
3712 %}
3713 
3714 instruct vlslI_imm(vReg dst, vReg src, immI shift) %{
3715   predicate(UseSVE > 0);
3716   match(Set dst (LShiftVI src (LShiftCntV shift)));
3717   ins_cost(SVE_COST);
3718   format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (S)" %}
3719   ins_encode %{
3720     int con = (int)$shift$$constant;
3721     __ sve_lsl(as_FloatRegister($dst$$reg), __ S,
3722          as_FloatRegister($src$$reg), con);
3723   %}
3724   ins_pipe(pipe_slow);
3725 %}
3726 
3727 instruct vlslL_imm(vReg dst, vReg src, immI shift) %{
3728   predicate(UseSVE > 0);
3729   match(Set dst (LShiftVL src (LShiftCntV shift)));
3730   ins_cost(SVE_COST);
3731   format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (D)" %}
3732   ins_encode %{
3733     int con = (int)$shift$$constant;
3734     __ sve_lsl(as_FloatRegister($dst$$reg), __ D,
3735          as_FloatRegister($src$$reg), con);
3736   %}
3737   ins_pipe(pipe_slow);
3738 %}
3739 
3740 instruct vshiftcntB(vReg dst, iRegIorL2I cnt) %{
3741   predicate(UseSVE > 0 &&
3742             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE));
3743   match(Set dst (LShiftCntV cnt));
3744   match(Set dst (RShiftCntV cnt));
3745   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (B)" %}
3746   ins_encode %{
3747     __ sve_dup(as_FloatRegister($dst$$reg), __ B, as_Register($cnt$$reg));
3748   %}
3749   ins_pipe(pipe_slow);
3750 %}
3751 
3752 instruct vshiftcntS(vReg dst, iRegIorL2I cnt) %{
3753   predicate(UseSVE > 0 &&
3754             (n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
3755             (n->bottom_type()->is_vect()->element_basic_type() == T_CHAR)));
3756   match(Set dst (LShiftCntV cnt));
3757   match(Set dst (RShiftCntV cnt));
3758   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (H)" %}
3759   ins_encode %{
3760     __ sve_dup(as_FloatRegister($dst$$reg), __ H, as_Register($cnt$$reg));
3761   %}
3762   ins_pipe(pipe_slow);
3763 %}
3764 
3765 instruct vshiftcntI(vReg dst, iRegIorL2I cnt) %{
3766   predicate(UseSVE > 0 &&
3767             (n->bottom_type()->is_vect()->element_basic_type() == T_INT));
3768   match(Set dst (LShiftCntV cnt));
3769   match(Set dst (RShiftCntV cnt));
3770   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (S)" %}
3771   ins_encode %{
3772     __ sve_dup(as_FloatRegister($dst$$reg), __ S, as_Register($cnt$$reg));
3773   %}
3774   ins_pipe(pipe_slow);
3775 %}
3776 
3777 instruct vshiftcntL(vReg dst, iRegIorL2I cnt) %{
3778   predicate(UseSVE > 0 &&
3779             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
3780   match(Set dst (LShiftCntV cnt));
3781   match(Set dst (RShiftCntV cnt));
3782   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (D)" %}
3783   ins_encode %{
3784     __ sve_dup(as_FloatRegister($dst$$reg), __ D, as_Register($cnt$$reg));
3785   %}
3786   ins_pipe(pipe_slow);
3787 %}
3788 
3789 // vector shift - predicated
3790 
3791 instruct vasrB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3792   predicate(UseSVE > 0);
3793   match(Set dst_src1 (RShiftVB (Binary dst_src1 src2) pg));
3794   ins_cost(SVE_COST);
3795   format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
3796   ins_encode %{
3797     __ sve_asr(as_FloatRegister($dst_src1$$reg), __ B,
3798             as_PRegister($pg$$reg),
3799             as_FloatRegister($src2$$reg));
3800   %}
3801   ins_pipe(pipe_slow);
3802 %}
3803 
3804 instruct vasrS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3805   predicate(UseSVE > 0);
3806   match(Set dst_src1 (RShiftVS (Binary dst_src1 src2) pg));
3807   ins_cost(SVE_COST);
3808   format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
3809   ins_encode %{
3810     __ sve_asr(as_FloatRegister($dst_src1$$reg), __ H,
3811             as_PRegister($pg$$reg),
3812             as_FloatRegister($src2$$reg));
3813   %}
3814   ins_pipe(pipe_slow);
3815 %}
3816 
3817 instruct vasrI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3818   predicate(UseSVE > 0);
3819   match(Set dst_src1 (RShiftVI (Binary dst_src1 src2) pg));
3820   ins_cost(SVE_COST);
3821   format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
3822   ins_encode %{
3823     __ sve_asr(as_FloatRegister($dst_src1$$reg), __ S,
3824             as_PRegister($pg$$reg),
3825             as_FloatRegister($src2$$reg));
3826   %}
3827   ins_pipe(pipe_slow);
3828 %}
3829 
3830 instruct vasrL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3831   predicate(UseSVE > 0);
3832   match(Set dst_src1 (RShiftVL (Binary dst_src1 src2) pg));
3833   ins_cost(SVE_COST);
3834   format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
3835   ins_encode %{
3836     __ sve_asr(as_FloatRegister($dst_src1$$reg), __ D,
3837             as_PRegister($pg$$reg),
3838             as_FloatRegister($src2$$reg));
3839   %}
3840   ins_pipe(pipe_slow);
3841 %}
3842 
3843 instruct vlslB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3844   predicate(UseSVE > 0);
3845   match(Set dst_src1 (LShiftVB (Binary dst_src1 src2) pg));
3846   ins_cost(SVE_COST);
3847   format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
3848   ins_encode %{
3849     __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ B,
3850             as_PRegister($pg$$reg),
3851             as_FloatRegister($src2$$reg));
3852   %}
3853   ins_pipe(pipe_slow);
3854 %}
3855 
3856 instruct vlslS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3857   predicate(UseSVE > 0);
3858   match(Set dst_src1 (LShiftVS (Binary dst_src1 src2) pg));
3859   ins_cost(SVE_COST);
3860   format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
3861   ins_encode %{
3862     __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ H,
3863             as_PRegister($pg$$reg),
3864             as_FloatRegister($src2$$reg));
3865   %}
3866   ins_pipe(pipe_slow);
3867 %}
3868 
3869 instruct vlslI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3870   predicate(UseSVE > 0);
3871   match(Set dst_src1 (LShiftVI (Binary dst_src1 src2) pg));
3872   ins_cost(SVE_COST);
3873   format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
3874   ins_encode %{
3875     __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ S,
3876             as_PRegister($pg$$reg),
3877             as_FloatRegister($src2$$reg));
3878   %}
3879   ins_pipe(pipe_slow);
3880 %}
3881 
3882 instruct vlslL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3883   predicate(UseSVE > 0);
3884   match(Set dst_src1 (LShiftVL (Binary dst_src1 src2) pg));
3885   ins_cost(SVE_COST);
3886   format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
3887   ins_encode %{
3888     __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ D,
3889             as_PRegister($pg$$reg),
3890             as_FloatRegister($src2$$reg));
3891   %}
3892   ins_pipe(pipe_slow);
3893 %}
3894 
3895 instruct vlsrB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3896   predicate(UseSVE > 0);
3897   match(Set dst_src1 (URShiftVB (Binary dst_src1 src2) pg));
3898   ins_cost(SVE_COST);
3899   format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
3900   ins_encode %{
3901     __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ B,
3902             as_PRegister($pg$$reg),
3903             as_FloatRegister($src2$$reg));
3904   %}
3905   ins_pipe(pipe_slow);
3906 %}
3907 
3908 instruct vlsrS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3909   predicate(UseSVE > 0);
3910   match(Set dst_src1 (URShiftVS (Binary dst_src1 src2) pg));
3911   ins_cost(SVE_COST);
3912   format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
3913   ins_encode %{
3914     __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ H,
3915             as_PRegister($pg$$reg),
3916             as_FloatRegister($src2$$reg));
3917   %}
3918   ins_pipe(pipe_slow);
3919 %}
3920 
3921 instruct vlsrI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3922   predicate(UseSVE > 0);
3923   match(Set dst_src1 (URShiftVI (Binary dst_src1 src2) pg));
3924   ins_cost(SVE_COST);
3925   format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
3926   ins_encode %{
3927     __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ S,
3928             as_PRegister($pg$$reg),
3929             as_FloatRegister($src2$$reg));
3930   %}
3931   ins_pipe(pipe_slow);
3932 %}
3933 
3934 instruct vlsrL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3935   predicate(UseSVE > 0);
3936   match(Set dst_src1 (URShiftVL (Binary dst_src1 src2) pg));
3937   ins_cost(SVE_COST);
3938   format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
3939   ins_encode %{
3940     __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ D,
3941             as_PRegister($pg$$reg),
3942             as_FloatRegister($src2$$reg));
3943   %}
3944   ins_pipe(pipe_slow);
3945 %}
3946 
3947 instruct vasrB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
3948   predicate(UseSVE > 0);
3949   match(Set dst_src (RShiftVB (Binary dst_src (RShiftCntV shift)) pg));
3950   ins_cost(SVE_COST);
3951   format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %}
3952   ins_encode %{
3953     int con = (int)$shift$$constant;
3954     assert(con > 0 && con < 8, "invalid shift immediate");
3955     __ sve_asr(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con);
3956   %}
3957   ins_pipe(pipe_slow);
3958 %}
3959 
3960 instruct vasrS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
3961   predicate(UseSVE > 0);
3962   match(Set dst_src (RShiftVS (Binary dst_src (RShiftCntV shift)) pg));
3963   ins_cost(SVE_COST);
3964   format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %}
3965   ins_encode %{
3966     int con = (int)$shift$$constant;
3967     assert(con > 0 && con < 16, "invalid shift immediate");
3968     __ sve_asr(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con);
3969   %}
3970   ins_pipe(pipe_slow);
3971 %}
3972 
3973 instruct vasrI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
3974   predicate(UseSVE > 0);
3975   match(Set dst_src (RShiftVI (Binary dst_src (RShiftCntV shift)) pg));
3976   ins_cost(SVE_COST);
3977   format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %}
3978   ins_encode %{
3979     int con = (int)$shift$$constant;
3980     assert(con > 0 && con < 32, "invalid shift immediate");
3981     __ sve_asr(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con);
3982   %}
3983   ins_pipe(pipe_slow);
3984 %}
3985 
3986 instruct vasrL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
3987   predicate(UseSVE > 0);
3988   match(Set dst_src (RShiftVL (Binary dst_src (RShiftCntV shift)) pg));
3989   ins_cost(SVE_COST);
3990   format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %}
3991   ins_encode %{
3992     int con = (int)$shift$$constant;
3993     assert(con > 0 && con < 64, "invalid shift immediate");
3994     __ sve_asr(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con);
3995   %}
3996   ins_pipe(pipe_slow);
3997 %}
3998 
3999 instruct vlsrB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4000   predicate(UseSVE > 0);
4001   match(Set dst_src (URShiftVB (Binary dst_src (RShiftCntV shift)) pg));
4002   ins_cost(SVE_COST);
4003   format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %}
4004   ins_encode %{
4005     int con = (int)$shift$$constant;
4006     assert(con > 0 && con < 8, "invalid shift immediate");
4007     __ sve_lsr(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con);
4008   %}
4009   ins_pipe(pipe_slow);
4010 %}
4011 
4012 instruct vlsrS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4013   predicate(UseSVE > 0);
4014   match(Set dst_src (URShiftVS (Binary dst_src (RShiftCntV shift)) pg));
4015   ins_cost(SVE_COST);
4016   format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %}
4017   ins_encode %{
4018     int con = (int)$shift$$constant;
4019     assert(con > 0 && con < 16, "invalid shift immediate");
4020     __ sve_lsr(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con);
4021   %}
4022   ins_pipe(pipe_slow);
4023 %}
4024 
4025 instruct vlsrI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4026   predicate(UseSVE > 0);
4027   match(Set dst_src (URShiftVI (Binary dst_src (RShiftCntV shift)) pg));
4028   ins_cost(SVE_COST);
4029   format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %}
4030   ins_encode %{
4031     int con = (int)$shift$$constant;
4032     assert(con > 0 && con < 32, "invalid shift immediate");
4033     __ sve_lsr(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con);
4034   %}
4035   ins_pipe(pipe_slow);
4036 %}
4037 
4038 instruct vlsrL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4039   predicate(UseSVE > 0);
4040   match(Set dst_src (URShiftVL (Binary dst_src (RShiftCntV shift)) pg));
4041   ins_cost(SVE_COST);
4042   format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %}
4043   ins_encode %{
4044     int con = (int)$shift$$constant;
4045     assert(con > 0 && con < 64, "invalid shift immediate");
4046     __ sve_lsr(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con);
4047   %}
4048   ins_pipe(pipe_slow);
4049 %}
4050 
4051 instruct vlslB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4052   predicate(UseSVE > 0);
4053   match(Set dst_src (LShiftVB (Binary dst_src (LShiftCntV shift)) pg));
4054   ins_cost(SVE_COST);
4055   format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %}
4056   ins_encode %{
4057     int con = (int)$shift$$constant;
4058     assert(con >= 0 && con < 8, "invalid shift immediate");
4059     __ sve_lsl(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con);
4060   %}
4061   ins_pipe(pipe_slow);
4062 %}
4063 
4064 instruct vlslS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4065   predicate(UseSVE > 0);
4066   match(Set dst_src (LShiftVS (Binary dst_src (LShiftCntV shift)) pg));
4067   ins_cost(SVE_COST);
4068   format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %}
4069   ins_encode %{
4070     int con = (int)$shift$$constant;
4071     assert(con >= 0 && con < 16, "invalid shift immediate");
4072     __ sve_lsl(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con);
4073   %}
4074   ins_pipe(pipe_slow);
4075 %}
4076 
4077 instruct vlslI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4078   predicate(UseSVE > 0);
4079   match(Set dst_src (LShiftVI (Binary dst_src (LShiftCntV shift)) pg));
4080   ins_cost(SVE_COST);
4081   format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %}
4082   ins_encode %{
4083     int con = (int)$shift$$constant;
4084     assert(con >= 0 && con < 32, "invalid shift immediate");
4085     __ sve_lsl(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con);
4086   %}
4087   ins_pipe(pipe_slow);
4088 %}
4089 
4090 instruct vlslL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4091   predicate(UseSVE > 0);
4092   match(Set dst_src (LShiftVL (Binary dst_src (LShiftCntV shift)) pg));
4093   ins_cost(SVE_COST);
4094   format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %}
4095   ins_encode %{
4096     int con = (int)$shift$$constant;
4097     assert(con >= 0 && con < 64, "invalid shift immediate");
4098     __ sve_lsl(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con);
4099   %}
4100   ins_pipe(pipe_slow);
4101 %}
4102 
4103 // vector sqrt
4104 
4105 instruct vsqrtF(vReg dst, vReg src) %{
4106   predicate(UseSVE > 0 &&
4107             !n->as_Vector()->is_predicated_vector());
4108   match(Set dst (SqrtVF src));
4109   ins_cost(SVE_COST);
4110   format %{ "sve_fsqrt $dst, $src\t# vector (sve) (S)" %}
4111   ins_encode %{
4112     __ sve_fsqrt(as_FloatRegister($dst$$reg), __ S,
4113          ptrue, as_FloatRegister($src$$reg));
4114   %}
4115   ins_pipe(pipe_slow);
4116 %}
4117 
4118 instruct vsqrtD(vReg dst, vReg src) %{
4119   predicate(UseSVE > 0 &&
4120             !n->as_Vector()->is_predicated_vector());
4121   match(Set dst (SqrtVD src));
4122   ins_cost(SVE_COST);
4123   format %{ "sve_fsqrt $dst, $src\t# vector (sve) (D)" %}
4124   ins_encode %{
4125     __ sve_fsqrt(as_FloatRegister($dst$$reg), __ D,
4126          ptrue, as_FloatRegister($src$$reg));
4127   %}
4128   ins_pipe(pipe_slow);
4129 %}
4130 
4131 // vector sqrt - predicated
4132 
4133 instruct vsqrtF_masked(vReg dst_src, pRegGov pg) %{
4134   predicate(UseSVE > 0);
4135   match(Set dst_src (SqrtVF dst_src pg));
4136   ins_cost(SVE_COST);
4137   format %{ "sve_fsqrt $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
4138   ins_encode %{
4139     __ sve_fsqrt(as_FloatRegister($dst_src$$reg), __ S,
4140             as_PRegister($pg$$reg),
4141             as_FloatRegister($dst_src$$reg));
4142   %}
4143   ins_pipe(pipe_slow);
4144 %}
4145 
4146 instruct vsqrtD_masked(vReg dst_src, pRegGov pg) %{
4147   predicate(UseSVE > 0);
4148   match(Set dst_src (SqrtVD dst_src pg));
4149   ins_cost(SVE_COST);
4150   format %{ "sve_fsqrt $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
4151   ins_encode %{
4152     __ sve_fsqrt(as_FloatRegister($dst_src$$reg), __ D,
4153             as_PRegister($pg$$reg),
4154             as_FloatRegister($dst_src$$reg));
4155   %}
4156   ins_pipe(pipe_slow);
4157 %}
4158 
4159 // vector sub
4160 
4161 instruct vsubB(vReg dst, vReg src1, vReg src2) %{
4162   predicate(UseSVE > 0);
4163   match(Set dst (SubVB src1 src2));
4164   ins_cost(SVE_COST);
4165   format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (B)" %}
4166   ins_encode %{
4167     __ sve_sub(as_FloatRegister($dst$$reg), __ B,
4168          as_FloatRegister($src1$$reg),
4169          as_FloatRegister($src2$$reg));
4170   %}
4171   ins_pipe(pipe_slow);
4172 %}
4173 
4174 instruct vsubS(vReg dst, vReg src1, vReg src2) %{
4175   predicate(UseSVE > 0);
4176   match(Set dst (SubVS src1 src2));
4177   ins_cost(SVE_COST);
4178   format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (H)" %}
4179   ins_encode %{
4180     __ sve_sub(as_FloatRegister($dst$$reg), __ H,
4181          as_FloatRegister($src1$$reg),
4182          as_FloatRegister($src2$$reg));
4183   %}
4184   ins_pipe(pipe_slow);
4185 %}
4186 
4187 instruct vsubI(vReg dst, vReg src1, vReg src2) %{
4188   predicate(UseSVE > 0);
4189   match(Set dst (SubVI src1 src2));
4190   ins_cost(SVE_COST);
4191   format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (S)" %}
4192   ins_encode %{
4193     __ sve_sub(as_FloatRegister($dst$$reg), __ S,
4194          as_FloatRegister($src1$$reg),
4195          as_FloatRegister($src2$$reg));
4196   %}
4197   ins_pipe(pipe_slow);
4198 %}
4199 
4200 instruct vsubL(vReg dst, vReg src1, vReg src2) %{
4201   predicate(UseSVE > 0);
4202   match(Set dst (SubVL src1 src2));
4203   ins_cost(SVE_COST);
4204   format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (D)" %}
4205   ins_encode %{
4206     __ sve_sub(as_FloatRegister($dst$$reg), __ D,
4207          as_FloatRegister($src1$$reg),
4208          as_FloatRegister($src2$$reg));
4209   %}
4210   ins_pipe(pipe_slow);
4211 %}
4212 
4213 instruct vsubF(vReg dst, vReg src1, vReg src2) %{
4214   predicate(UseSVE > 0);
4215   match(Set dst (SubVF src1 src2));
4216   ins_cost(SVE_COST);
4217   format %{ "sve_fsub $dst, $src1, $src2\t # vector (sve) (S)" %}
4218   ins_encode %{
4219     __ sve_fsub(as_FloatRegister($dst$$reg), __ S,
4220          as_FloatRegister($src1$$reg),
4221          as_FloatRegister($src2$$reg));
4222   %}
4223   ins_pipe(pipe_slow);
4224 %}
4225 
4226 instruct vsubD(vReg dst, vReg src1, vReg src2) %{
4227   predicate(UseSVE > 0);
4228   match(Set dst (SubVD src1 src2));
4229   ins_cost(SVE_COST);
4230   format %{ "sve_fsub $dst, $src1, $src2\t # vector (sve) (D)" %}
4231   ins_encode %{
4232     __ sve_fsub(as_FloatRegister($dst$$reg), __ D,
4233          as_FloatRegister($src1$$reg),
4234          as_FloatRegister($src2$$reg));
4235   %}
4236   ins_pipe(pipe_slow);
4237 %}
4238 
4239 // vector sub - predicated
4240 
4241 instruct vsubB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4242   predicate(UseSVE > 0);
4243   match(Set dst_src1 (SubVB (Binary dst_src1 src2) pg));
4244   ins_cost(SVE_COST);
4245   format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
4246   ins_encode %{
4247     __ sve_sub(as_FloatRegister($dst_src1$$reg), __ B,
4248             as_PRegister($pg$$reg),
4249             as_FloatRegister($src2$$reg));
4250   %}
4251   ins_pipe(pipe_slow);
4252 %}
4253 
4254 instruct vsubS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4255   predicate(UseSVE > 0);
4256   match(Set dst_src1 (SubVS (Binary dst_src1 src2) pg));
4257   ins_cost(SVE_COST);
4258   format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
4259   ins_encode %{
4260     __ sve_sub(as_FloatRegister($dst_src1$$reg), __ H,
4261             as_PRegister($pg$$reg),
4262             as_FloatRegister($src2$$reg));
4263   %}
4264   ins_pipe(pipe_slow);
4265 %}
4266 
4267 instruct vsubI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4268   predicate(UseSVE > 0);
4269   match(Set dst_src1 (SubVI (Binary dst_src1 src2) pg));
4270   ins_cost(SVE_COST);
4271   format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
4272   ins_encode %{
4273     __ sve_sub(as_FloatRegister($dst_src1$$reg), __ S,
4274             as_PRegister($pg$$reg),
4275             as_FloatRegister($src2$$reg));
4276   %}
4277   ins_pipe(pipe_slow);
4278 %}
4279 
4280 instruct vsubL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4281   predicate(UseSVE > 0);
4282   match(Set dst_src1 (SubVL (Binary dst_src1 src2) pg));
4283   ins_cost(SVE_COST);
4284   format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
4285   ins_encode %{
4286     __ sve_sub(as_FloatRegister($dst_src1$$reg), __ D,
4287             as_PRegister($pg$$reg),
4288             as_FloatRegister($src2$$reg));
4289   %}
4290   ins_pipe(pipe_slow);
4291 %}
4292 
4293 instruct vsubF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4294   predicate(UseSVE > 0);
4295   match(Set dst_src1 (SubVF (Binary dst_src1 src2) pg));
4296   ins_cost(SVE_COST);
4297   format %{ "sve_fsub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
4298   ins_encode %{
4299     __ sve_fsub(as_FloatRegister($dst_src1$$reg), __ S,
4300             as_PRegister($pg$$reg),
4301             as_FloatRegister($src2$$reg));
4302   %}
4303   ins_pipe(pipe_slow);
4304 %}
4305 
4306 instruct vsubD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4307   predicate(UseSVE > 0);
4308   match(Set dst_src1 (SubVD (Binary dst_src1 src2) pg));
4309   ins_cost(SVE_COST);
4310   format %{ "sve_fsub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
4311   ins_encode %{
4312     __ sve_fsub(as_FloatRegister($dst_src1$$reg), __ D,
4313             as_PRegister($pg$$reg),
4314             as_FloatRegister($src2$$reg));
4315   %}
4316   ins_pipe(pipe_slow);
4317 %}
4318 
4319 // ------------------------------ Vector mask cast --------------------------
4320 
4321 instruct vmaskcast(pRegGov dst_src) %{
4322   predicate(UseSVE > 0 &&
4323             n->bottom_type()->is_vect()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
4324             n->bottom_type()->is_vect()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
4325   match(Set dst_src (VectorMaskCast dst_src));
4326   ins_cost(0);
4327   format %{ "vmaskcast $dst_src\t# empty (sve)" %}
4328   ins_encode %{
4329     // empty
4330   %}
4331   ins_pipe(pipe_class_empty);
4332 %}
4333 
4334 instruct vmaskcast_extend(pRegGov dst, pReg src)
4335 %{
4336   predicate(UseSVE > 0 &&
4337             (Matcher::vector_length_in_bytes(n) == 2 * Matcher::vector_length_in_bytes(n->in(1)) ||
4338              Matcher::vector_length_in_bytes(n) == 4 * Matcher::vector_length_in_bytes(n->in(1)) ||
4339              Matcher::vector_length_in_bytes(n) == 8 * Matcher::vector_length_in_bytes(n->in(1))));
4340   match(Set dst (VectorMaskCast src));
4341   ins_cost(SVE_COST * 3);
4342   format %{ "sve_vmaskcast_extend  $dst, $src\t# extend predicate $src" %}
4343   ins_encode %{
4344     __ sve_vmaskcast_extend(as_PRegister($dst$$reg), as_PRegister($src$$reg),
4345                             Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
4346   %}
4347   ins_pipe(pipe_slow);
4348 %}
4349 
4350 instruct vmaskcast_narrow(pRegGov dst, pReg src)
4351 %{
4352   predicate(UseSVE > 0 &&
4353             (Matcher::vector_length_in_bytes(n) * 2 == Matcher::vector_length_in_bytes(n->in(1)) ||
4354              Matcher::vector_length_in_bytes(n) * 4 == Matcher::vector_length_in_bytes(n->in(1)) ||
4355              Matcher::vector_length_in_bytes(n) * 8 == Matcher::vector_length_in_bytes(n->in(1))));
4356   match(Set dst (VectorMaskCast src));
4357   ins_cost(SVE_COST * 3);
4358   format %{ "sve_vmaskcast_narrow  $dst, $src\t# narrow predicate $src" %}
4359   ins_encode %{
4360     __ sve_vmaskcast_narrow(as_PRegister($dst$$reg), as_PRegister($src$$reg),
4361                             Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
4362   %}
4363   ins_pipe(pipe_slow);
4364 %}
4365 
4366 // ------------------------------ Vector cast -------------------------------
4367 
4368 instruct vcvtBtoX_extend(vReg dst, vReg src)
4369 %{
4370   predicate(UseSVE > 0);
4371   match(Set dst (VectorCastB2X src));
4372   ins_cost(2 * SVE_COST);
4373   format %{ "sve_vectorcast_b2x  $dst, $src\t# convert B to X vector (extend)" %}
4374   ins_encode %{
4375     BasicType to_bt = Matcher::vector_element_basic_type(this);
4376     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4377     __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ B);
4378     if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
4379       __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
4380     }
4381   %}
4382   ins_pipe(pipe_slow);
4383 %}
4384 
4385 instruct vcvtStoB(vReg dst, vReg src, vReg tmp)
4386 %{
4387   predicate(UseSVE > 0 &&
4388             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
4389   match(Set dst (VectorCastS2X src));
4390   effect(TEMP tmp);
4391   ins_cost(2 * SVE_COST);
4392   format %{ "sve_vectorcast_s2b  $dst, $src\t# convert H to B vector" %}
4393   ins_encode %{
4394     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
4395                          as_FloatRegister($src$$reg), __ H, as_FloatRegister($tmp$$reg));
4396   %}
4397   ins_pipe(pipe_slow);
4398 %}
4399 
4400 instruct vcvtStoX_extend(vReg dst, vReg src)
4401 %{
4402   predicate(UseSVE > 0 &&
4403             type2aelembytes(Matcher::vector_element_basic_type(n)) > 2);
4404   match(Set dst (VectorCastS2X src));
4405   ins_cost(2 * SVE_COST);
4406   format %{ "sve_vectorcast_s2x  $dst, $src\t# convert H to X vector (extend)" %}
4407   ins_encode %{
4408     BasicType to_bt = Matcher::vector_element_basic_type(this);
4409     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4410     __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ H);
4411     if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
4412       __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
4413     }
4414   %}
4415   ins_pipe(pipe_slow);
4416 %}
4417 
4418 instruct vcvtItoB(vReg dst, vReg src, vReg tmp)
4419 %{
4420   predicate(UseSVE > 0 &&
4421             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
4422   match(Set dst (VectorCastI2X src));
4423   effect(TEMP_DEF dst, TEMP tmp);
4424   ins_cost(3 * SVE_COST);
4425   format %{ "sve_vectorcast_i2b  $dst, $src\t# convert I to B vector" %}
4426   ins_encode %{
4427     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
4428                          as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
4429   %}
4430   ins_pipe(pipe_slow);
4431 %}
4432 
4433 instruct vcvtItoS(vReg dst, vReg src, vReg tmp)
4434 %{
4435   predicate(UseSVE > 0 &&
4436             n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
4437   match(Set dst (VectorCastI2X src));
4438   effect(TEMP tmp);
4439   ins_cost(2 * SVE_COST);
4440   format %{ "sve_vectorcast_i2s $dst, $src\t# convert I to H vector" %}
4441   ins_encode %{
4442     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ H,
4443                          as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
4444   %}
4445   ins_pipe(pipe_slow);
4446 %}
4447 
4448 instruct vcvtItoL(vReg dst, vReg src)
4449 %{
4450   predicate(UseSVE > 0 &&
4451             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
4452   match(Set dst (VectorCastI2X src));
4453   ins_cost(SVE_COST);
4454   format %{ "sve_vectorcast_i2l  $dst, $src\t# convert I to L vector" %}
4455   ins_encode %{
4456     __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
4457   %}
4458   ins_pipe(pipe_slow);
4459 %}
4460 
4461 instruct vcvtItoF(vReg dst, vReg src)
4462 %{
4463   predicate(UseSVE > 0 &&
4464             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4465   match(Set dst (VectorCastI2X src));
4466   ins_cost(SVE_COST);
4467   format %{ "sve_vectorcast_i2f  $dst, $src\t# convert I to F vector" %}
4468   ins_encode %{
4469     __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
4470   %}
4471   ins_pipe(pipe_slow);
4472 %}
4473 
4474 instruct vcvtItoD(vReg dst, vReg src)
4475 %{
4476   predicate(UseSVE > 0 &&
4477             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
4478   match(Set dst (VectorCastI2X src));
4479   ins_cost(2 * SVE_COST);
4480   format %{ "sve_vectorcast_i2d  $dst, $src\t# convert I to D vector" %}
4481   ins_encode %{
4482     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
4483     __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ D);
4484   %}
4485   ins_pipe(pipe_slow);
4486 %}
4487 
4488 instruct vcvtLtoX_narrow(vReg dst, vReg src, vReg tmp)
4489 %{
4490   predicate(UseSVE > 0 && is_integral_type(Matcher::vector_element_basic_type(n)));
4491   match(Set dst (VectorCastL2X src));
4492   effect(TEMP_DEF dst, TEMP tmp);
4493   ins_cost(2 * SVE_COST);
4494   format %{ "sve_vectorcast_l2x  $dst, $src\t# convert L to B/H/S vector (narrow)" %}
4495   ins_encode %{
4496     BasicType to_bt = Matcher::vector_element_basic_type(this);
4497     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4498     __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
4499                          as_FloatRegister($src$$reg), __ D, as_FloatRegister($tmp$$reg));
4500   %}
4501   ins_pipe(pipe_slow);
4502 %}
4503 
4504 instruct vcvtLtoF(vReg dst, vReg src, vReg tmp)
4505 %{
4506   predicate(UseSVE > 0 &&
4507             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4508   match(Set dst (VectorCastL2X src));
4509   effect(TEMP_DEF dst, TEMP tmp);
4510   ins_cost(3 * SVE_COST);
4511   format %{ "sve_vectorcast_l2f  $dst, $src\t# convert L to F vector" %}
4512   ins_encode %{
4513     __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
4514     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
4515                          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
4516 
4517   %}
4518   ins_pipe(pipe_slow);
4519 %}
4520 
4521 instruct vcvtLtoD(vReg dst, vReg src)
4522 %{
4523   predicate(UseSVE > 0 &&
4524             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
4525   match(Set dst (VectorCastL2X src));
4526   ins_cost(SVE_COST);
4527   format %{ "sve_vectorcast_l2d  $dst, $src\t# convert L to D vector" %}
4528   ins_encode %{
4529     __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
4530   %}
4531   ins_pipe(pipe_slow);
4532 %}
4533 
4534 instruct vcvtFtoX_narrow(vReg dst, vReg src, vReg tmp)
4535 %{
4536   predicate(UseSVE > 0 &&
4537             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4538              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT));
4539   match(Set dst (VectorCastF2X src));
4540   effect(TEMP_DEF dst, TEMP tmp);
4541   ins_cost(3 * SVE_COST);
4542   format %{ "sve_vectorcast_f2x  $dst, $src\t# convert F to B/H vector" %}
4543   ins_encode %{
4544     BasicType to_bt = Matcher::vector_element_basic_type(this);
4545     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4546     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
4547     __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
4548                          as_FloatRegister($dst$$reg), __ S, as_FloatRegister($tmp$$reg));
4549   %}
4550   ins_pipe(pipe_slow);
4551 %}
4552 
4553 instruct vcvtFtoX_extend(vReg dst, vReg src)
4554 %{
4555   predicate(UseSVE > 0 &&
4556             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
4557              n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
4558   match(Set dst (VectorCastF2X src));
4559   ins_cost(SVE_COST);
4560   format %{ "sve_vectorcast_f2x  $dst, $src\t# convert F to I/L vector" %}
4561   ins_encode %{
4562     BasicType to_bt = Matcher::vector_element_basic_type(this);
4563     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
4564     if (to_bt == T_LONG) {
4565       __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($dst$$reg), __ S);
4566     }
4567   %}
4568   ins_pipe(pipe_slow);
4569 %}
4570 
4571 instruct vcvtFtoD(vReg dst, vReg src)
4572 %{
4573   predicate(UseSVE > 0 &&
4574             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
4575   match(Set dst (VectorCastF2X src));
4576   ins_cost(2 * SVE_COST);
4577   format %{ "sve_vectorcast_f2d  $dst, $dst\t# convert F to D vector" %}
4578   ins_encode %{
4579     __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
4580     __ sve_fcvt(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);
4581   %}
4582   ins_pipe(pipe_slow);
4583 %}
4584 
4585 instruct vcvtDtoX_narrow(vReg dst, vReg src, vReg tmp)
4586 %{
4587   predicate(UseSVE > 0 &&
4588             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4589              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
4590              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4591   match(Set dst (VectorCastD2X src));
4592   effect(TEMP_DEF dst, TEMP tmp);
4593   ins_cost(3 * SVE_COST);
4594   format %{ "sve_vectorcast_d2x  $dst, $src\t# convert D to X vector (narrow)" %}
4595   ins_encode %{
4596     BasicType to_bt = Matcher::vector_element_basic_type(this);
4597     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4598     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
4599     __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
4600                          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
4601   %}
4602   ins_pipe(pipe_slow);
4603 %}
4604 
4605 instruct vcvtDtoL(vReg dst, vReg src)
4606 %{
4607   predicate(UseSVE > 0 &&
4608             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
4609   match(Set dst (VectorCastD2X src));
4610   ins_cost(SVE_COST);
4611   format %{ "sve_vectorcast_d2l  $dst, $src\t# convert D to L vector" %}
4612   ins_encode %{
4613     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
4614   %}
4615   ins_pipe(pipe_slow);
4616 %}
4617 
4618 instruct vcvtDtoF(vReg dst, vReg src, vReg tmp)
4619 %{
4620   predicate(UseSVE > 0 &&
4621             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4622   match(Set dst (VectorCastD2X src));
4623   effect(TEMP_DEF dst, TEMP tmp);
4624   ins_cost(3 * SVE_COST);
4625   format %{ "sve_vectorcast_d2f  $dst, S, $dst\t# convert D to F vector" %}
4626   ins_encode %{
4627     __ sve_fcvt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
4628     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
4629                          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
4630   %}
4631   ins_pipe(pipe_slow);
4632 %}
4633 
4634 // ------------------------------ Vector extract ---------------------------------
4635 
4636 instruct extractB(iRegINoSp dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
4637 %{
4638   predicate(UseSVE > 0);
4639   match(Set dst (ExtractB src idx));
4640   effect(TEMP pgtmp, KILL cr);
4641   ins_cost(2 * SVE_COST);
4642   format %{ "sve_extract $dst, B, $pgtmp, $src, $idx\n\t"
4643             "sbfmw $dst, $dst, 0U, 7U\t# extract from vector(B)" %}
4644   ins_encode %{
4645     __ sve_extract(as_Register($dst$$reg), __ B, as_PRegister($pgtmp$$reg),
4646                    as_FloatRegister($src$$reg), (int)($idx$$constant));
4647     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 7U);
4648   %}
4649   ins_pipe(pipe_slow);
4650 %}
4651 
4652 instruct extractS(iRegINoSp dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
4653 %{
4654   predicate(UseSVE > 0);
4655   match(Set dst (ExtractS src idx));
4656   effect(TEMP pgtmp, KILL cr);
4657   ins_cost(2 * SVE_COST);
4658   format %{ "sve_extract $dst, H, $pgtmp, $src, $idx\n\t"
4659             "sbfmw $dst, $dst, 0U, 15U\t# extract from vector(S)" %}
4660   ins_encode %{
4661     __ sve_extract(as_Register($dst$$reg), __ H, as_PRegister($pgtmp$$reg),
4662                    as_FloatRegister($src$$reg), (int)($idx$$constant));
4663     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
4664   %}
4665   ins_pipe(pipe_slow);
4666 %}
4667 
4668 
4669 instruct extractI(iRegINoSp dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
4670 %{
4671   predicate(UseSVE > 0);
4672   match(Set dst (ExtractI src idx));
4673   effect(TEMP pgtmp, KILL cr);
4674   ins_cost(2 * SVE_COST);
4675   format %{ "sve_extract $dst, S, $pgtmp, $src, $idx\t# extract from vector(I)" %}
4676   ins_encode %{
4677     __ sve_extract(as_Register($dst$$reg), __ S, as_PRegister($pgtmp$$reg),
4678                    as_FloatRegister($src$$reg), (int)($idx$$constant));
4679   %}
4680   ins_pipe(pipe_slow);
4681 %}
4682 
4683 instruct extractL(iRegLNoSp dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
4684 %{
4685   predicate(UseSVE > 0);
4686   match(Set dst (ExtractL src idx));
4687   effect(TEMP pgtmp, KILL cr);
4688   ins_cost(2 * SVE_COST);
4689   format %{ "sve_extract $dst, D, $pgtmp, $src, $idx\t# extract from vector(L)" %}
4690   ins_encode %{
4691     __ sve_extract(as_Register($dst$$reg), __ D, as_PRegister($pgtmp$$reg),
4692                    as_FloatRegister($src$$reg), (int)($idx$$constant));
4693   %}
4694   ins_pipe(pipe_slow);
4695 %}
4696 
4697 instruct extractF(vRegF dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
4698 %{
4699   predicate(UseSVE > 0);
4700   match(Set dst (ExtractF src idx));
4701   effect(TEMP pgtmp, KILL cr);
4702   ins_cost(2 * SVE_COST);
4703   format %{ "sve_extract $dst, S, $pgtmp, $src, $idx\t# extract from vector(F)" %}
4704   ins_encode %{
4705     __ sve_extract(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg),
4706                    as_FloatRegister($src$$reg), (int)($idx$$constant));
4707   %}
4708   ins_pipe(pipe_slow);
4709 %}
4710 
4711 instruct extractD(vRegD dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
4712 %{
4713   predicate(UseSVE > 0);
4714   match(Set dst (ExtractD src idx));
4715   effect(TEMP pgtmp, KILL cr);
4716   ins_cost(2 * SVE_COST);
4717   format %{ "sve_extract $dst, D, $pgtmp, $src, $idx\t# extract from vector(D)" %}
4718   ins_encode %{
4719     __ sve_extract(as_FloatRegister($dst$$reg), __ D, as_PRegister($pgtmp$$reg),
4720                    as_FloatRegister($src$$reg), (int)($idx$$constant));
4721   %}
4722   ins_pipe(pipe_slow);
4723 %}
4724 
4725 // ------------------------------- VectorTest ----------------------------------
4726 
4727 instruct vtest_alltrue(iRegINoSp dst, pRegGov src1, pRegGov src2, pReg ptmp, rFlagsReg cr)
4728 %{
4729   predicate(UseSVE > 0 &&
4730             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
4731             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
4732   match(Set dst (VectorTest src1 src2));
4733   effect(TEMP ptmp, KILL cr);
4734   ins_cost(SVE_COST);
4735   format %{ "sve_eors $ptmp, $src1, $src2\t# $src2 is all true mask\n"
4736             "csetw $dst, EQ\t# VectorTest (sve) - alltrue" %}
4737   ins_encode %{
4738     __ sve_eors(as_PRegister($ptmp$$reg), ptrue,
4739                 as_PRegister($src1$$reg), as_PRegister($src2$$reg));
4740     __ csetw(as_Register($dst$$reg), Assembler::EQ);
4741   %}
4742   ins_pipe(pipe_slow);
4743 %}
4744 
4745 instruct vtest_anytrue(iRegINoSp dst, pRegGov src1, pRegGov src2, rFlagsReg cr)
4746 %{
4747   predicate(UseSVE > 0 &&
4748             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
4749             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
4750   match(Set dst (VectorTest src1 src2));
4751   effect(KILL cr);
4752   ins_cost(SVE_COST);
4753   format %{ "sve_ptest $src1\n\t"
4754             "csetw $dst, NE\t# VectorTest (sve) - anytrue" %}
4755   ins_encode %{
4756     // "src2" is not used for sve.
4757     __ sve_ptest(ptrue, as_PRegister($src1$$reg));
4758     __ csetw(as_Register($dst$$reg), Assembler::NE);
4759   %}
4760   ins_pipe(pipe_slow);
4761 %}
4762 
4763 instruct vtest_alltrue_partial(iRegINoSp dst, pRegGov src1, pRegGov src2, pRegGov ptmp, rFlagsReg cr)
4764 %{
4765   predicate(UseSVE > 0 &&
4766             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
4767             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
4768   match(Set dst (VectorTest src1 src2));
4769   effect(TEMP ptmp, KILL cr);
4770   ins_cost(SVE_COST);
4771   format %{ "vtest_alltrue_partial $dst, $src1, $src2\t# VectorTest partial (sve) - alltrue" %}
4772   ins_encode %{
4773     BasicType bt = Matcher::vector_element_basic_type(this, $src1);
4774     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
4775     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size,
4776                           Matcher::vector_length(this, $src1));
4777     __ sve_eors(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
4778           as_PRegister($src1$$reg), as_PRegister($src2$$reg));
4779     __ csetw(as_Register($dst$$reg), Assembler::EQ);
4780   %}
4781   ins_pipe(pipe_slow);
4782 %}
4783 
4784 instruct vtest_anytrue_partial(iRegINoSp dst, pRegGov src1, pRegGov src2, pRegGov ptmp, rFlagsReg cr)
4785 %{
4786   predicate(UseSVE > 0 &&
4787             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
4788             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
4789   match(Set dst (VectorTest src1 src2));
4790   effect(TEMP ptmp, KILL cr);
4791   ins_cost(SVE_COST);
4792   format %{ "vtest_anytrue_partial $dst, $src1, $src2\t# VectorTest partial (sve) - anytrue" %}
4793   ins_encode %{
4794     BasicType bt = Matcher::vector_element_basic_type(this, $src1);
4795     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
4796     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size,
4797                           Matcher::vector_length(this, $src1));
4798     __ sve_ands(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
4799           as_PRegister($src1$$reg), as_PRegister($src2$$reg));
4800     __ csetw(as_Register($dst$$reg), Assembler::NE);
4801   %}
4802   ins_pipe(pipe_slow);
4803 %}
4804 
4805 // ------------------------------ Vector insert ---------------------------------
4806 
4807 instruct insertI_small(vReg dst, vReg src, iRegIorL2I val, immI idx, pRegGov pgtmp, rFlagsReg cr)
4808 %{
4809   predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
4810             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4811              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
4812              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4813   match(Set dst (VectorInsert (Binary src val) idx));
4814   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
4815   ins_cost(4 * SVE_COST);
4816   format %{ "sve_index $dst, -16, 1\t# (B/H/S)\n\t"
4817             "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
4818             "sve_orr $dst, $src, $src\n\t"
4819             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
4820   ins_encode %{
4821     BasicType bt = Matcher::vector_element_basic_type(this, $src);
4822     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
4823     __ sve_index(as_FloatRegister($dst$$reg), size, -16, 1);
4824     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
4825                as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
4826     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4827     __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
4828   %}
4829   ins_pipe(pipe_slow);
4830 %}
4831 
4832 instruct insertF_small(vReg dst, vReg src, vRegF val, immI idx, pRegGov pgtmp, rFlagsReg cr)
4833 %{
4834   predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
4835             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4836   match(Set dst (VectorInsert (Binary src val) idx));
4837   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
4838   ins_cost(4 * SVE_COST);
4839   format %{ "sve_index $dst, S, -16, 1\n\t"
4840             "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
4841             "sve_orr $dst, $src, $src\n\t"
4842             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
4843   ins_encode %{
4844     __ sve_index(as_FloatRegister($dst$$reg), __ S, -16, 1);
4845     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
4846                as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
4847     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4848     __ sve_cpy(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
4849   %}
4850   ins_pipe(pipe_slow);
4851 %}
4852 
4853 instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr)
4854 %{
4855   predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
4856             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4857              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
4858              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4859   match(Set dst (VectorInsert (Binary src val) idx));
4860   effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
4861   ins_cost(5 * SVE_COST);
4862   format %{ "sve_index $tmp1, 0, 1\t# (B/H/S)\n\t"
4863             "sve_dup $dst, $idx\t# (B/H/S)\n\t"
4864             "sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
4865             "sve_orr $dst, $src, $src\n\t"
4866             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
4867   ins_encode %{
4868     BasicType bt = Matcher::vector_element_basic_type(this, $src);
4869     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
4870     __ sve_index(as_FloatRegister($tmp1$$reg), size, 0, 1);
4871     __ sve_dup(as_FloatRegister($dst$$reg), size, (int)($idx$$constant));
4872     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
4873                as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
4874     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4875     __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
4876   %}
4877   ins_pipe(pipe_slow);
4878 %}
4879 
4880 instruct insertL(vReg dst, vReg src, iRegL val, immI idx, pRegGov pgtmp, rFlagsReg cr)
4881 %{
4882   predicate(UseSVE > 0 &&
4883             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
4884   match(Set dst (VectorInsert (Binary src val) idx));
4885   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
4886   ins_cost(4 * SVE_COST);
4887   format %{ "sve_index $dst, D, -16, 1\n\t"
4888             "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
4889             "sve_orr $dst, $src, $src\n\t"
4890             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (L)" %}
4891   ins_encode %{
4892     __ sve_index(as_FloatRegister($dst$$reg), __ D, -16, 1);
4893     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
4894                as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
4895     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4896     __ sve_cpy(as_FloatRegister($dst$$reg), __ D, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
4897   %}
4898   ins_pipe(pipe_slow);
4899 %}
4900 
4901 instruct insertD(vReg dst, vReg src, vRegD val, immI idx, pRegGov pgtmp, rFlagsReg cr)
4902 %{
4903   predicate(UseSVE > 0 &&
4904             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
4905   match(Set dst (VectorInsert (Binary src val) idx));
4906   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
4907   ins_cost(4 * SVE_COST);
4908   format %{ "sve_index $dst, D, -16, 1\n\t"
4909             "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
4910             "sve_orr $dst, $src, $src\n\t"
4911             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (D)" %}
4912   ins_encode %{
4913     __ sve_index(as_FloatRegister($dst$$reg), __ D, -16, 1);
4914     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
4915                as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
4916     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4917     __ sve_cpy(as_FloatRegister($dst$$reg), __ D, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
4918   %}
4919   ins_pipe(pipe_slow);
4920 %}
4921 
4922 instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr)
4923 %{
4924   predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
4925             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4926   match(Set dst (VectorInsert (Binary src val) idx));
4927   effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
4928   ins_cost(5 * SVE_COST);
4929   format %{ "sve_index $tmp1, S, 0, 1\n\t"
4930             "sve_dup $dst, S, $idx\n\t"
4931             "sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
4932             "sve_orr $dst, $src, $src\n\t"
4933             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
4934   ins_encode %{
4935     __ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
4936     __ sve_dup(as_FloatRegister($dst$$reg), __ S, (int)($idx$$constant));
4937     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
4938                as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
4939     __ sve_orr(as_FloatRegister($dst$$reg),
4940                as_FloatRegister($src$$reg),
4941                as_FloatRegister($src$$reg));
4942     __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
4943                as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
4944   %}
4945   ins_pipe(pipe_slow);
4946 %}
4947 
4948 // ------------------------------ Vector shuffle -------------------------------
4949 
4950 instruct loadshuffle(vReg dst, vReg src) %{
4951   predicate(UseSVE > 0);
4952   match(Set dst (VectorLoadShuffle src));
4953   ins_cost(SVE_COST);
4954   format %{ "sve_loadshuffle $dst, $src\t# vector load shuffle (B/H/S/D)" %}
4955   ins_encode %{
4956     BasicType bt = Matcher::vector_element_basic_type(this);
4957     if (bt == T_BYTE) {
4958       if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
4959         __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
4960                    as_FloatRegister($src$$reg));
4961       }
4962     } else {
4963       __ sve_vector_extend(as_FloatRegister($dst$$reg),  __ elemType_to_regVariant(bt),
4964                            as_FloatRegister($src$$reg), __ B);
4965     }
4966   %}
4967   ins_pipe(pipe_slow);
4968 %}
4969 
4970 // ------------------------------ Vector rearrange -------------------------------
4971 
4972 instruct rearrange(vReg dst, vReg src, vReg shuffle)
4973 %{
4974   predicate(UseSVE > 0);
4975   match(Set dst (VectorRearrange src shuffle));
4976   ins_cost(SVE_COST);
4977   format %{ "sve_tbl $dst, $src, $shuffle\t# vector rearrange" %}
4978   ins_encode %{
4979     BasicType bt = Matcher::vector_element_basic_type(this, $src);
4980     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
4981     __ sve_tbl(as_FloatRegister($dst$$reg), size,
4982                as_FloatRegister($src$$reg), as_FloatRegister($shuffle$$reg));
4983   %}
4984   ins_pipe(pipe_slow);
4985 %}
4986 
4987 // ------------------------------ Vector Load Gather ---------------------------------
4988 
4989 instruct gatherI(vReg dst, indirect mem, vReg idx) %{
4990   predicate(UseSVE > 0 &&
4991             n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
4992             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
4993              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
4994   match(Set dst (LoadVectorGather mem idx));
4995   ins_cost(SVE_COST);
4996   format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (S)" %}
4997   ins_encode %{
4998     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), ptrue,
4999                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5000   %}
5001   ins_pipe(pipe_slow);
5002 %}
5003 
5004 instruct gatherL(vReg dst, indirect mem, vReg idx) %{
5005   predicate(UseSVE > 0 &&
5006             n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
5007             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5008              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5009   match(Set dst (LoadVectorGather mem idx));
5010   ins_cost(2 * SVE_COST);
5011   format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (D)" %}
5012   ins_encode %{
5013     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5014     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), ptrue, as_Register($mem$$base),
5015                        as_FloatRegister($idx$$reg));
5016   %}
5017   ins_pipe(pipe_slow);
5018 %}
5019 
5020 // ------------------------------ Vector Load Gather Partial-------------------------------
5021 
5022 instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5023   predicate(UseSVE > 0 &&
5024             n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
5025             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5026              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5027   match(Set dst (LoadVectorGather mem idx));
5028   effect(TEMP ptmp, KILL cr);
5029   ins_cost(2 * SVE_COST + INSN_COST);
5030   format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (S)" %}
5031   ins_encode %{
5032     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this));
5033     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
5034                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5035   %}
5036   ins_pipe(pipe_slow);
5037 %}
5038 
5039 instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5040   predicate(UseSVE > 0 &&
5041             n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
5042             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5043              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5044   match(Set dst (LoadVectorGather mem idx));
5045   effect(TEMP ptmp, KILL cr);
5046   ins_cost(3 * SVE_COST + INSN_COST);
5047   format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (D)" %}
5048   ins_encode %{
5049     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
5050                           Matcher::vector_length(this));
5051     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5052     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
5053                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5054   %}
5055   ins_pipe(pipe_slow);
5056 %}
5057 
5058 // ------------------------------ Vector Load Gather Predicated -------------------------------
5059 
5060 instruct gatherI_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
5061   predicate(UseSVE > 0 &&
5062             n->as_LoadVector()->memory_size() == MaxVectorSize &&
5063             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5064              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5065   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
5066   ins_cost(SVE_COST);
5067   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (S)" %}
5068   ins_encode %{
5069     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
5070                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5071   %}
5072   ins_pipe(pipe_slow);
5073 %}
5074 
5075 instruct gatherL_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
5076   predicate(UseSVE > 0 &&
5077             n->as_LoadVector()->memory_size() == MaxVectorSize &&
5078             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5079              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5080   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
5081   ins_cost(2 * SVE_COST);
5082   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (D)" %}
5083   ins_encode %{
5084     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5085     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
5086                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5087   %}
5088   ins_pipe(pipe_slow);
5089 %}
5090 
5091 // ------------------------------ Vector Load Gather Predicated Partial -------------------------------
5092 
5093 instruct gatherI_masked_partial(vReg dst, indirect mem, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
5094   predicate(UseSVE > 0 &&
5095             n->as_LoadVector()->memory_size() < MaxVectorSize &&
5096             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5097              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5098   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
5099   effect(TEMP ptmp, KILL cr);
5100   ins_cost(3 * SVE_COST);
5101   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated partial (S)" %}
5102   ins_encode %{
5103     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
5104                           Matcher::vector_length(this));
5105     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
5106                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
5107     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
5108                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5109   %}
5110   ins_pipe(pipe_slow);
5111 %}
5112 
5113 instruct gatherL_masked_partial(vReg dst, indirect mem, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
5114   predicate(UseSVE > 0 &&
5115             n->as_LoadVector()->memory_size() < MaxVectorSize &&
5116             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5117              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5118   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
5119   effect(TEMP ptmp, KILL cr);
5120   ins_cost(4 * SVE_COST);
5121   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated partial (D)" %}
5122   ins_encode %{
5123     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this));
5124     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
5125                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
5126     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5127     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
5128                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5129   %}
5130   ins_pipe(pipe_slow);
5131 %}
5132 
5133 // ------------------------------ Vector Store Scatter -------------------------------
5134 
5135 instruct scatterI(indirect mem, vReg src, vReg idx) %{
5136   predicate(UseSVE > 0 &&
5137             n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
5138             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5139              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5140   match(Set mem (StoreVectorScatter mem (Binary src idx)));
5141   ins_cost(SVE_COST);
5142   format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (S)" %}
5143   ins_encode %{
5144     __ sve_st1w_scatter(as_FloatRegister($src$$reg), ptrue,
5145                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5146   %}
5147   ins_pipe(pipe_slow);
5148 %}
5149 
5150 instruct scatterL(indirect mem, vReg src, vReg idx) %{
5151   predicate(UseSVE > 0 &&
5152             n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
5153             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5154              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5155   match(Set mem (StoreVectorScatter mem (Binary src idx)));
5156   ins_cost(2 * SVE_COST);
5157   format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (D)" %}
5158   ins_encode %{
5159     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5160     __ sve_st1d_scatter(as_FloatRegister($src$$reg), ptrue,
5161                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5162   %}
5163   ins_pipe(pipe_slow);
5164 %}
5165 
5166 // ------------------------------ Vector Store Scatter Partial -------------------------------
5167 
5168 instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5169   predicate(UseSVE > 0 &&
5170             n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
5171             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5172              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5173   match(Set mem (StoreVectorScatter mem (Binary src idx)));
5174   effect(TEMP ptmp, KILL cr);
5175   ins_cost(2 * SVE_COST + INSN_COST);
5176   format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (S)" %}
5177   ins_encode %{
5178     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
5179                           Matcher::vector_length(this, $src));
5180     __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
5181                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5182   %}
5183   ins_pipe(pipe_slow);
5184 %}
5185 
5186 instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5187   predicate(UseSVE > 0 &&
5188             n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
5189             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5190              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5191   match(Set mem (StoreVectorScatter mem (Binary src idx)));
5192   effect(TEMP ptmp, KILL cr);
5193   ins_cost(3 * SVE_COST + INSN_COST);
5194   format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (D)" %}
5195   ins_encode %{
5196     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
5197                           Matcher::vector_length(this, $src));
5198     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5199     __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
5200                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5201   %}
5202   ins_pipe(pipe_slow);
5203 %}
5204 
5205 // ------------------------------ Vector Store Scatter Predicated -------------------------------
5206 
5207 instruct scatterI_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
5208   predicate(UseSVE > 0 &&
5209             n->as_StoreVector()->memory_size() == MaxVectorSize &&
5210             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5211              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5212   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
5213   ins_cost(SVE_COST);
5214   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicate (S)" %}
5215   ins_encode %{
5216     __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
5217                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5218   %}
5219   ins_pipe(pipe_slow);
5220 %}
5221 
5222 instruct scatterL_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
5223   predicate(UseSVE > 0 &&
5224             n->as_StoreVector()->memory_size() == MaxVectorSize &&
5225             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5226              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5227   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
5228   ins_cost(2 * SVE_COST);
5229   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated (D)" %}
5230   ins_encode %{
5231     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5232     __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
5233                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5234   %}
5235   ins_pipe(pipe_slow);
5236 %}
5237 
5238 // ------------------------------ Vector Store Scatter Predicated Partial -------------------------------
5239 
5240 instruct scatterI_masked_partial(indirect mem, vReg src, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
5241   predicate(UseSVE > 0 &&
5242             n->as_StoreVector()->memory_size() < MaxVectorSize &&
5243             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5244              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5245   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
5246   effect(TEMP ptmp, KILL cr);
5247   ins_cost(3 * SVE_COST);
5248   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated partial (S)" %}
5249   ins_encode %{
5250     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
5251                           Matcher::vector_length(this, $src));
5252     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
5253                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
5254     __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
5255                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5256   %}
5257   ins_pipe(pipe_slow);
5258 %}
5259 
5260 instruct scatterL_masked_partial(indirect mem, vReg src, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
5261   predicate(UseSVE > 0 &&
5262             n->as_StoreVector()->memory_size() < MaxVectorSize &&
5263             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5264              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5265   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
5266   effect(TEMP ptmp, KILL cr);
5267   ins_cost(4 * SVE_COST);
5268   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated partial (D)" %}
5269   ins_encode %{
5270     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
5271                           Matcher::vector_length(this, $src));
5272     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
5273                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
5274     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5275     __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
5276                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5277   %}
5278   ins_pipe(pipe_slow);
5279 %}
5280 
5281 // ------------------------------ Vector Load Const -------------------------------
5282 
5283 instruct loadconB(vReg dst, immI0 src) %{
5284   predicate(UseSVE > 0 &&
5285             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
5286   match(Set dst (VectorLoadConst src));
5287   ins_cost(SVE_COST);
5288   format %{ "sve_index $dst, 0, 1\t# generate iota indices" %}
5289   ins_encode %{
5290     __ sve_index(as_FloatRegister($dst$$reg), __ B, 0, 1);
5291   %}
5292   ins_pipe(pipe_slow);
5293 %}
5294 
5295 // Intrisics for String.indexOf(char)
5296 
5297 
5298 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
5299                                   iRegI_R0 result, vReg ztmp1, vReg ztmp2,
5300                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
5301 %{
5302   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
5303   predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
5304   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
5305 
5306   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
5307 
5308   ins_encode %{
5309     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
5310                                as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
5311                                as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), true /* isL */);
5312   %}
5313   ins_pipe(pipe_class_memory);
5314 %}
5315 
5316 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
5317                                   iRegI_R0 result, vReg ztmp1, vReg ztmp2,
5318                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
5319 %{
5320   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
5321   predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
5322   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
5323 
5324   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
5325 
5326   ins_encode %{
5327     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
5328                                as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
5329                                as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), false /* isL */);
5330   %}
5331   ins_pipe(pipe_class_memory);
5332 %}
5333 
5334 // ---------------------------- Vector mask reductions ---------------------------
5335 instruct vmask_truecount(iRegINoSp dst, pReg src) %{
5336   predicate(UseSVE > 0 &&
5337             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
5338   match(Set dst (VectorMaskTrueCount src));
5339   ins_cost(SVE_COST);
5340   format %{ "vmask_truecount $dst, $src\t# vector mask truecount (sve)" %}
5341   ins_encode %{
5342     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5343     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5344     __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($src$$reg));
5345   %}
5346   ins_pipe(pipe_slow);
5347 %}
5348 
5349 instruct vmask_firsttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
5350   predicate(UseSVE > 0 &&
5351             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
5352   match(Set dst (VectorMaskFirstTrue src));
5353   effect(TEMP ptmp);
5354   ins_cost(2 * SVE_COST);
5355   format %{ "vmask_firsttrue $dst, $src\t# vector mask firsttrue (sve)" %}
5356   ins_encode %{
5357     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5358     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5359     __ sve_brkb(as_PRegister($ptmp$$reg), ptrue, as_PRegister($src$$reg), false);
5360     __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($ptmp$$reg));
5361   %}
5362   ins_pipe(pipe_slow);
5363 %}
5364 
5365 instruct vmask_lasttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
5366   predicate(UseSVE > 0 &&
5367             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
5368   match(Set dst (VectorMaskLastTrue src));
5369   effect(TEMP ptmp);
5370   ins_cost(3 * SVE_COST);
5371   format %{ "vmask_lasttrue $dst, $src\t# vector mask lasttrue (sve)" %}
5372   ins_encode %{
5373     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5374     __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($src$$reg), as_PRegister($ptmp$$reg));
5375   %}
5376   ins_pipe(pipe_slow);
5377 %}
5378 
5379 instruct vmask_truecount_partial(iRegINoSp dst, pReg src, pRegGov pgtmp, rFlagsReg cr) %{
5380   predicate(UseSVE > 0 &&
5381             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
5382   match(Set dst (VectorMaskTrueCount src));
5383   effect(TEMP pgtmp, KILL cr);
5384   ins_cost(2 * SVE_COST);
5385   format %{ "vmask_truecount_partial $dst, $src\t# vector mask truecount partial (sve)" %}
5386   ins_encode %{
5387     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5388     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5389     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), size, Matcher::vector_length(this, $src));
5390     __ sve_cntp($dst$$Register, size, as_PRegister($pgtmp$$reg), as_PRegister($src$$reg));
5391   %}
5392   ins_pipe(pipe_slow);
5393 %}
5394 
5395 instruct vmask_firsttrue_partial(iRegINoSp dst, pReg src, pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
5396   predicate(UseSVE > 0 &&
5397             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
5398   match(Set dst (VectorMaskFirstTrue src));
5399   effect(TEMP pgtmp, TEMP ptmp, KILL cr);
5400   ins_cost(3 * SVE_COST);
5401   format %{ "vmask_firsttrue_partial $dst, $src\t# vector mask firsttrue partial (sve)" %}
5402   ins_encode %{
5403     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5404     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5405     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), size,
5406                           Matcher::vector_length(this, $src));
5407     __ sve_brkb(as_PRegister($ptmp$$reg), as_PRegister($pgtmp$$reg), as_PRegister($src$$reg), false);
5408     __ sve_cntp($dst$$Register, size, as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg));
5409   %}
5410   ins_pipe(pipe_slow);
5411 %}
5412 
5413 instruct vmask_lasttrue_partial(iRegINoSp dst, pReg src, pReg ptmp, rFlagsReg cr) %{
5414   predicate(UseSVE > 0 &&
5415             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
5416   match(Set dst (VectorMaskLastTrue src));
5417   effect(TEMP ptmp, KILL cr);
5418   ins_cost(5 * SVE_COST);
5419   format %{ "vmask_lasttrue_partial $dst, $src\t# vector mask lasttrue partial (sve)" %}
5420   ins_encode %{
5421     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5422     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5423     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
5424     __ sve_and(as_PRegister($ptmp$$reg), ptrue, as_PRegister($ptmp$$reg), as_PRegister($src$$reg));
5425     __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg));
5426   %}
5427   ins_pipe(pipe_slow);
5428 %}