1 //
   2 // Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
   3 // Copyright (c) 2020, 2022, Arm Limited. All rights reserved.
   4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5 //
   6 // This code is free software; you can redistribute it and/or modify it
   7 // under the terms of the GNU General Public License version 2 only, as
   8 // published by the Free Software Foundation.
   9 //
  10 // This code is distributed in the hope that it will be useful, but WITHOUT
  11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 // version 2 for more details (a copy is included in the LICENSE file that
  14 // accompanied this code).
  15 //
  16 // You should have received a copy of the GNU General Public License version
  17 // 2 along with this work; if not, write to the Free Software Foundation,
  18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21 // or visit www.oracle.com if you need additional information or have any
  22 // questions.
  23 //
  24 //
  25 
  26 // This file is automatically generated by running "m4 aarch64_sve_ad.m4". Do not edit ----
  27 
  28 // AArch64 SVE Architecture Description File
  29 
  30 
  31 // 4 bit signed offset -- for predicated load/store
  32 
  33 operand vmemA_immIOffset4()
  34 %{
  35   // (esize / msize) = 1
  36   predicate(Address::offset_ok_for_sve_immed(n->get_int(), 4,
  37             Matcher::scalable_vector_reg_size(T_BYTE)));
  38   match(ConI);
  39 
  40   op_cost(0);
  41   format %{ %}
  42   interface(CONST_INTER);
  43 %}
  44 
  45 operand vmemA_immLOffset4()
  46 %{
  47   // (esize / msize) = 1
  48   predicate(Address::offset_ok_for_sve_immed(n->get_long(), 4,
  49             Matcher::scalable_vector_reg_size(T_BYTE)));
  50   match(ConL);
  51 
  52   op_cost(0);
  53   format %{ %}
  54   interface(CONST_INTER);
  55 %}
  56 
  57 operand vmemA_indOffI4(iRegP reg, vmemA_immIOffset4 off)
  58 %{
  59   constraint(ALLOC_IN_RC(ptr_reg));
  60   match(AddP reg off);
  61   op_cost(0);
  62   format %{ "[$reg, $off]" %}
  63   interface(MEMORY_INTER) %{
  64     base($reg);
  65     index(0xffffffff);
  66     scale(0x0);
  67     disp($off);
  68   %}
  69 %}
  70 
  71 operand vmemA_indOffL4(iRegP reg, vmemA_immLOffset4 off)
  72 %{
  73   constraint(ALLOC_IN_RC(ptr_reg));
  74   match(AddP reg off);
  75   op_cost(0);
  76   format %{ "[$reg, $off]" %}
  77   interface(MEMORY_INTER) %{
  78     base($reg);
  79     index(0xffffffff);
  80     scale(0x0);
  81     disp($off);
  82   %}
  83 %}
  84 
  85 // The indOff of vmemA is valid only when the vector element (load to/store from)
  86 // size equals to memory element (load from/store to) size.
  87 opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);
  88 
  89 source_hpp %{
  90   bool op_sve_supported(int opcode, int vlen, BasicType bt);
  91   bool masked_op_sve_supported(int opcode, int vlen, BasicType bt);
  92 %}
  93 
  94 source %{
  95 
  96   typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
  97                                                              PRegister Pg, const Address &adr);
  98 
  99   // Predicated load/store, with optional ptrue to all elements of given predicate register.
 100   static void loadStoreA_predicated(C2_MacroAssembler masm, bool is_store, FloatRegister reg,
 101                                     PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
 102                                     int opcode, Register base, int index, int size, int disp) {
 103     sve_mem_insn_predicate insn;
 104     int mesize = type2aelembytes(mem_elem_bt);
 105     if (index == -1) {
 106       assert(size == 0, "unsupported address mode: scale size = %d", size);
 107       switch(mesize) {
 108       case 1:
 109         insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
 110         break;
 111       case 2:
 112         insn = is_store ? &C2_MacroAssembler::sve_st1h : &C2_MacroAssembler::sve_ld1h;
 113         break;
 114       case 4:
 115         insn = is_store ? &C2_MacroAssembler::sve_st1w : &C2_MacroAssembler::sve_ld1w;
 116         break;
 117       case 8:
 118         insn = is_store ? &C2_MacroAssembler::sve_st1d : &C2_MacroAssembler::sve_ld1d;
 119         break;
 120       default:
 121         assert(false, "unsupported");
 122         ShouldNotReachHere();
 123       }
 124       int imm4 = disp / mesize / Matcher::scalable_vector_reg_size(vector_elem_bt);
 125       (masm.*insn)(reg, Assembler::elemType_to_regVariant(vector_elem_bt), pg, Address(base, imm4));
 126     } else {
 127       assert(false, "unimplemented");
 128       ShouldNotReachHere();
 129     }
 130   }
 131 
 132   bool op_sve_supported(int opcode, int vlen, BasicType bt) {
 133     int length_in_bytes = vlen * type2aelembytes(bt);
 134     switch (opcode) {
 135       case Op_MulAddVS2VI:
 136       // No multiply reduction instructions
 137       case Op_MulReductionVD:
 138       case Op_MulReductionVF:
 139       case Op_MulReductionVI:
 140       case Op_MulReductionVL:
 141       // Others
 142       case Op_ExtractC:
 143       case Op_ExtractUB:
 144         return false;
 145       // Vector API specific
 146       case Op_VectorLoadShuffle:
 147       case Op_VectorRearrange:
 148         return vlen >= 4 && length_in_bytes <= MaxVectorSize;
 149       case Op_LoadVector:
 150       case Op_StoreVector:
 151         return Matcher::vector_size_supported(bt, vlen);
 152       default:
 153         break;
 154     }
 155     // By default, we only support vector operations with no less than 8 bytes and 2 elements.
 156     return 8 <= length_in_bytes && length_in_bytes <= MaxVectorSize && vlen >= 2;
 157   }
 158 
 159   bool masked_op_sve_supported(int opcode, int vlen, BasicType bt) {
 160     if (opcode == Op_VectorRearrange) {
 161       return false;
 162     }
 163     return op_sve_supported(opcode, vlen, bt);
 164   }
 165 %}
 166 
 167 definitions %{
 168   int_def SVE_COST             (200, 200);
 169 %}
 170 
 171 
 172 // All SVE instructions
 173 
 174 // vector load/store
 175 
 176 // Unpredicated vector load/store
 177 instruct loadV(vReg dst, vmemA mem) %{
 178   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() >= 16 &&
 179             n->as_LoadVector()->memory_size() == MaxVectorSize);
 180   match(Set dst (LoadVector mem));
 181   ins_cost(4 * SVE_COST);
 182   format %{ "sve_ldr $dst, $mem\t# vector (sve)" %}
 183   ins_encode %{
 184     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 185     BasicType bt = Matcher::vector_element_basic_type(this);
 186     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg, ptrue,
 187                           bt, bt, $mem->opcode(),
 188                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 189   %}
 190   ins_pipe(pipe_slow);
 191 %}
 192 
 193 instruct storeV(vReg src, vmemA mem) %{
 194   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() >= 16 &&
 195             n->as_StoreVector()->memory_size() == MaxVectorSize);
 196   match(Set mem (StoreVector mem src));
 197   ins_cost(4 * SVE_COST);
 198   format %{ "sve_str $mem, $src\t# vector (sve)" %}
 199   ins_encode %{
 200     FloatRegister src_reg = as_FloatRegister($src$$reg);
 201     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 202     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg, ptrue,
 203                           bt, bt, $mem->opcode(),
 204                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 205   %}
 206   ins_pipe(pipe_slow);
 207 %}
 208 
 209 // Load Vector (16 bits)
 210 instruct loadV2_vreg(vReg dst, vmem2 mem)
 211 %{
 212   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 2);
 213   match(Set dst (LoadVector mem));
 214   ins_cost(4 * INSN_COST);
 215   format %{ "ldrh   $dst,$mem\t# vector (16 bits)" %}
 216   ins_encode( aarch64_enc_ldrvH(dst, mem) );
 217   ins_pipe(vload_reg_mem64);
 218 %}
 219 
 220 // Store Vector (16 bits)
 221 instruct storeV2_vreg(vReg src, vmem2 mem)
 222 %{
 223   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 2);
 224   match(Set mem (StoreVector mem src));
 225   ins_cost(4 * INSN_COST);
 226   format %{ "strh   $mem,$src\t# vector (16 bits)" %}
 227   ins_encode( aarch64_enc_strvH(src, mem) );
 228   ins_pipe(vstore_reg_mem64);
 229 %}
 230 
 231 // Load Vector (32 bits)
 232 instruct loadV4_vreg(vReg dst, vmem4 mem)
 233 %{
 234   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 4);
 235   match(Set dst (LoadVector mem));
 236   ins_cost(4 * INSN_COST);
 237   format %{ "ldrs   $dst,$mem\t# vector (32 bits)" %}
 238   ins_encode( aarch64_enc_ldrvS(dst, mem) );
 239   ins_pipe(vload_reg_mem64);
 240 %}
 241 
 242 // Store Vector (32 bits)
 243 instruct storeV4_vreg(vReg src, vmem4 mem)
 244 %{
 245   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 4);
 246   match(Set mem (StoreVector mem src));
 247   ins_cost(4 * INSN_COST);
 248   format %{ "strs   $mem,$src\t# vector (32 bits)" %}
 249   ins_encode( aarch64_enc_strvS(src, mem) );
 250   ins_pipe(vstore_reg_mem64);
 251 %}
 252 
 253 // Load Vector (64 bits)
 254 instruct loadV8_vreg(vReg dst, vmem8 mem)
 255 %{
 256   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 8);
 257   match(Set dst (LoadVector mem));
 258   ins_cost(4 * INSN_COST);
 259   format %{ "ldrd   $dst,$mem\t# vector (64 bits)" %}
 260   ins_encode( aarch64_enc_ldrvD(dst, mem) );
 261   ins_pipe(vload_reg_mem64);
 262 %}
 263 
 264 // Store Vector (64 bits)
 265 instruct storeV8_vreg(vReg src, vmem8 mem)
 266 %{
 267   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 8);
 268   match(Set mem (StoreVector mem src));
 269   ins_cost(4 * INSN_COST);
 270   format %{ "strd   $mem,$src\t# vector (64 bits)" %}
 271   ins_encode( aarch64_enc_strvD(src, mem) );
 272   ins_pipe(vstore_reg_mem64);
 273 %}
 274 
 275 // Load Vector (128 bits)
 276 instruct loadV16_vreg(vReg dst, vmem16 mem)
 277 %{
 278   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 16);
 279   match(Set dst (LoadVector mem));
 280   ins_cost(4 * INSN_COST);
 281   format %{ "ldrq   $dst,$mem\t# vector (128 bits)" %}
 282   ins_encode( aarch64_enc_ldrvQ(dst, mem) );
 283   ins_pipe(vload_reg_mem128);
 284 %}
 285 
 286 // Store Vector (128 bits)
 287 instruct storeV16_vreg(vReg src, vmem16 mem)
 288 %{
 289   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 16);
 290   match(Set mem (StoreVector mem src));
 291   ins_cost(4 * INSN_COST);
 292   format %{ "strq   $mem,$src\t# vector (128 bits)" %}
 293   ins_encode( aarch64_enc_strvQ(src, mem) );
 294   ins_pipe(vstore_reg_mem128);
 295 %}
 296 
 297 // Predicated vector load/store, based on the vector length of the node.
 298 // Only load/store values in the range of the memory_size. This is needed
 299 // when the memory_size is lower than the hardware supported max vector size.
 300 // And this might happen for Vector API mask vector load/store.
 301 instruct loadV_partial(vReg dst, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
 302   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() > 16 &&
 303             n->as_LoadVector()->memory_size() < MaxVectorSize);
 304   match(Set dst (LoadVector mem));
 305   effect(TEMP pgtmp, KILL cr);
 306   ins_cost(6 * SVE_COST);
 307   format %{ "sve_ptrue $pgtmp, vector_length\n\t"
 308             "sve_ldr $dst, $pgtmp, $mem\t# load vector partial" %}
 309   ins_encode %{
 310     BasicType bt = Matcher::vector_element_basic_type(this);
 311     __ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 312                          Matcher::vector_length(this));
 313     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 314     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg,
 315                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 316                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 317   %}
 318   ins_pipe(pipe_slow);
 319 %}
 320 
 321 instruct storeV_partial(vReg src, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
 322   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() > 16 &&
 323             n->as_StoreVector()->memory_size() < MaxVectorSize);
 324   match(Set mem (StoreVector mem src));
 325   effect(TEMP pgtmp, KILL cr);
 326   ins_cost(5 * SVE_COST);
 327   format %{ "sve_ptrue $pgtmp, vector_length\n\t"
 328             "sve_str $src, $pgtmp, $mem\t# store vector partial" %}
 329   ins_encode %{
 330     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 331     __ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 332                          Matcher::vector_length(this, $src));
 333     FloatRegister src_reg = as_FloatRegister($src$$reg);
 334     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg,
 335                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 336                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 337   %}
 338   ins_pipe(pipe_slow);
 339 %}
 340 
 341 // vector load/store - predicated
 342 
 343 instruct loadV_masked(vReg dst, vmemA mem, pRegGov pg) %{
 344   predicate(UseSVE > 0);
 345   match(Set dst (LoadVectorMasked mem pg));
 346   ins_cost(4 * SVE_COST);
 347   format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated (sve)" %}
 348   ins_encode %{
 349     BasicType bt = Matcher::vector_element_basic_type(this);
 350     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
 351                           as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
 352                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 353   %}
 354   ins_pipe(pipe_slow);
 355 %}
 356 
 357 instruct storeV_masked(vReg src, vmemA mem, pRegGov pg) %{
 358   predicate(UseSVE > 0);
 359   match(Set mem (StoreVectorMasked mem (Binary src pg)));
 360   ins_cost(4 * SVE_COST);
 361   format %{ "sve_str $mem, $pg, $src\t# store vector predicated (sve)" %}
 362   ins_encode %{
 363     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 364     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
 365                           as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
 366                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 367   %}
 368   ins_pipe(pipe_slow);
 369 %}
 370 
 371 // mask logical and/or/xor
 372 
 373 instruct vmask_and(pRegGov pd, pRegGov pn, pRegGov pm) %{
 374   predicate(UseSVE > 0);
 375   match(Set pd (AndVMask pn pm));
 376   ins_cost(SVE_COST);
 377   format %{ "sve_and $pd, $pn, $pm\t# predicate (sve)" %}
 378   ins_encode %{
 379     __ sve_and(as_PRegister($pd$$reg), ptrue,
 380                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 381   %}
 382   ins_pipe(pipe_slow);
 383 %}
 384 
 385 instruct vmask_or(pRegGov pd, pRegGov pn, pRegGov pm) %{
 386   predicate(UseSVE > 0);
 387   match(Set pd (OrVMask pn pm));
 388   ins_cost(SVE_COST);
 389   format %{ "sve_orr $pd, $pn, $pm\t# predicate (sve)" %}
 390   ins_encode %{
 391     __ sve_orr(as_PRegister($pd$$reg), ptrue,
 392                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 393   %}
 394   ins_pipe(pipe_slow);
 395 %}
 396 
 397 instruct vmask_xor(pRegGov pd, pRegGov pn, pRegGov pm) %{
 398   predicate(UseSVE > 0);
 399   match(Set pd (XorVMask pn pm));
 400   ins_cost(SVE_COST);
 401   format %{ "sve_eor $pd, $pn, $pm\t# predicate (sve)" %}
 402   ins_encode %{
 403     __ sve_eor(as_PRegister($pd$$reg), ptrue,
 404                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 405   %}
 406   ins_pipe(pipe_slow);
 407 %}
 408 
 409 // mask logical and_not
 410 
 411 instruct vmask_and_notI(pRegGov pd, pRegGov pn, pRegGov pm, immI_M1 m1) %{
 412   predicate(UseSVE > 0);
 413   match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
 414   ins_cost(SVE_COST);
 415   format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) (B/H/S)" %}
 416   ins_encode %{
 417     __ sve_bic(as_PRegister($pd$$reg), ptrue,
 418                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 419   %}
 420   ins_pipe(pipe_slow);
 421 %}
 422 
 423 instruct vmask_and_notL(pRegGov pd, pRegGov pn, pRegGov pm, immL_M1 m1) %{
 424   predicate(UseSVE > 0);
 425   match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
 426   ins_cost(SVE_COST);
 427   format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) (D)" %}
 428   ins_encode %{
 429     __ sve_bic(as_PRegister($pd$$reg), ptrue,
 430                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 431   %}
 432   ins_pipe(pipe_slow);
 433 %}
 434 
 435 // vector reinterpret
 436 
 437 instruct reinterpret(vReg dst) %{
 438   predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() ==
 439                           n->in(1)->bottom_type()->is_vect()->length_in_bytes());  // src == dst
 440   match(Set dst (VectorReinterpret dst));
 441   ins_cost(0);
 442   format %{ "# reinterpret $dst\t# do nothing" %}
 443   ins_encode %{
 444     // empty
 445   %}
 446   ins_pipe(pipe_class_empty);
 447 %}
 448 
 449 instruct reinterpretResize(vReg dst, vReg src, pRegGov pgtmp, rFlagsReg cr) %{
 450   predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() !=
 451                           n->in(1)->bottom_type()->is_vect()->length_in_bytes());  // src != dst
 452   match(Set dst (VectorReinterpret src));
 453   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
 454   ins_cost(3 * SVE_COST);
 455   format %{ "reinterpretResize $dst, $src\t# vector (sve)" %}
 456   ins_encode %{
 457     uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
 458     uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
 459     uint length_in_bytes_resize = length_in_bytes_src < length_in_bytes_dst ?
 460                                   length_in_bytes_src : length_in_bytes_dst;
 461     assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
 462            "invalid vector length");
 463     __ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ B, length_in_bytes_resize);
 464     __ sve_dup(as_FloatRegister($dst$$reg), __ B, 0);
 465     __ sve_sel(as_FloatRegister($dst$$reg), __ B, as_PRegister($pgtmp$$reg),
 466                as_FloatRegister($src$$reg), as_FloatRegister($dst$$reg));
 467   %}
 468   ins_pipe(pipe_slow);
 469 %}
 470 
 471 // vector mask reinterpret
 472 
 473 instruct vmask_reinterpret_same_esize(pRegGov dst_src) %{
 474   predicate(UseSVE > 0 &&
 475             n->as_Vector()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
 476             n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
 477   match(Set dst_src (VectorReinterpret dst_src));
 478   ins_cost(0);
 479   format %{ "# vmask_reinterpret $dst_src\t# do nothing" %}
 480   ins_encode %{
 481     // empty
 482   %}
 483   ins_pipe(pipe_class_empty);
 484 %}
 485 
 486 instruct vmask_reinterpret_diff_esize(pRegGov dst, pRegGov src, vReg tmp, rFlagsReg cr) %{
 487   predicate(UseSVE > 0 &&
 488             n->as_Vector()->length() != n->in(1)->bottom_type()->is_vect()->length() &&
 489             n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
 490   match(Set dst (VectorReinterpret src));
 491   effect(TEMP tmp, KILL cr);
 492   ins_cost(2 * SVE_COST);
 493   format %{ "# vmask_reinterpret $dst, $src\t# vector (sve)" %}
 494   ins_encode %{
 495     BasicType from_bt = Matcher::vector_element_basic_type(this, $src);
 496     Assembler::SIMD_RegVariant from_size = __ elemType_to_regVariant(from_bt);
 497     BasicType to_bt = Matcher::vector_element_basic_type(this);
 498     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
 499     __ sve_cpy(as_FloatRegister($tmp$$reg), from_size, as_PRegister($src$$reg), -1, false);
 500     __ sve_cmp(Assembler::EQ, as_PRegister($dst$$reg), to_size, ptrue, as_FloatRegister($tmp$$reg), -1);
 501   %}
 502   ins_pipe(pipe_slow);
 503 %}
 504 
 505 // vector abs
 506 
 507 instruct vabsB(vReg dst, vReg src) %{
 508   predicate(UseSVE > 0 &&
 509             !n->as_Vector()->is_predicated_vector());
 510   match(Set dst (AbsVB src));
 511   ins_cost(SVE_COST);
 512   format %{ "sve_abs $dst, $src\t# vector (sve) (B)" %}
 513   ins_encode %{
 514     __ sve_abs(as_FloatRegister($dst$$reg), __ B,
 515          ptrue, as_FloatRegister($src$$reg));
 516   %}
 517   ins_pipe(pipe_slow);
 518 %}
 519 
 520 instruct vabsS(vReg dst, vReg src) %{
 521   predicate(UseSVE > 0 &&
 522             !n->as_Vector()->is_predicated_vector());
 523   match(Set dst (AbsVS src));
 524   ins_cost(SVE_COST);
 525   format %{ "sve_abs $dst, $src\t# vector (sve) (H)" %}
 526   ins_encode %{
 527     __ sve_abs(as_FloatRegister($dst$$reg), __ H,
 528          ptrue, as_FloatRegister($src$$reg));
 529   %}
 530   ins_pipe(pipe_slow);
 531 %}
 532 
 533 instruct vabsI(vReg dst, vReg src) %{
 534   predicate(UseSVE > 0 &&
 535             !n->as_Vector()->is_predicated_vector());
 536   match(Set dst (AbsVI src));
 537   ins_cost(SVE_COST);
 538   format %{ "sve_abs $dst, $src\t# vector (sve) (S)" %}
 539   ins_encode %{
 540     __ sve_abs(as_FloatRegister($dst$$reg), __ S,
 541          ptrue, as_FloatRegister($src$$reg));
 542   %}
 543   ins_pipe(pipe_slow);
 544 %}
 545 
 546 instruct vabsL(vReg dst, vReg src) %{
 547   predicate(UseSVE > 0 &&
 548             !n->as_Vector()->is_predicated_vector());
 549   match(Set dst (AbsVL src));
 550   ins_cost(SVE_COST);
 551   format %{ "sve_abs $dst, $src\t# vector (sve) (D)" %}
 552   ins_encode %{
 553     __ sve_abs(as_FloatRegister($dst$$reg), __ D,
 554          ptrue, as_FloatRegister($src$$reg));
 555   %}
 556   ins_pipe(pipe_slow);
 557 %}
 558 
 559 instruct vabsF(vReg dst, vReg src) %{
 560   predicate(UseSVE > 0 &&
 561             !n->as_Vector()->is_predicated_vector());
 562   match(Set dst (AbsVF src));
 563   ins_cost(SVE_COST);
 564   format %{ "sve_fabs $dst, $src\t# vector (sve) (S)" %}
 565   ins_encode %{
 566     __ sve_fabs(as_FloatRegister($dst$$reg), __ S,
 567          ptrue, as_FloatRegister($src$$reg));
 568   %}
 569   ins_pipe(pipe_slow);
 570 %}
 571 
 572 instruct vabsD(vReg dst, vReg src) %{
 573   predicate(UseSVE > 0 &&
 574             !n->as_Vector()->is_predicated_vector());
 575   match(Set dst (AbsVD src));
 576   ins_cost(SVE_COST);
 577   format %{ "sve_fabs $dst, $src\t# vector (sve) (D)" %}
 578   ins_encode %{
 579     __ sve_fabs(as_FloatRegister($dst$$reg), __ D,
 580          ptrue, as_FloatRegister($src$$reg));
 581   %}
 582   ins_pipe(pipe_slow);
 583 %}
 584 
 585 // vector abs - predicated
 586 
 587 instruct vabsB_masked(vReg dst_src, pRegGov pg) %{
 588   predicate(UseSVE > 0);
 589   match(Set dst_src (AbsVB dst_src pg));
 590   ins_cost(SVE_COST);
 591   format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (B)" %}
 592   ins_encode %{
 593     __ sve_abs(as_FloatRegister($dst_src$$reg), __ B,
 594             as_PRegister($pg$$reg),
 595             as_FloatRegister($dst_src$$reg));
 596   %}
 597   ins_pipe(pipe_slow);
 598 %}
 599 
 600 instruct vabsS_masked(vReg dst_src, pRegGov pg) %{
 601   predicate(UseSVE > 0);
 602   match(Set dst_src (AbsVS dst_src pg));
 603   ins_cost(SVE_COST);
 604   format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (H)" %}
 605   ins_encode %{
 606     __ sve_abs(as_FloatRegister($dst_src$$reg), __ H,
 607             as_PRegister($pg$$reg),
 608             as_FloatRegister($dst_src$$reg));
 609   %}
 610   ins_pipe(pipe_slow);
 611 %}
 612 
 613 instruct vabsI_masked(vReg dst_src, pRegGov pg) %{
 614   predicate(UseSVE > 0);
 615   match(Set dst_src (AbsVI dst_src pg));
 616   ins_cost(SVE_COST);
 617   format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
 618   ins_encode %{
 619     __ sve_abs(as_FloatRegister($dst_src$$reg), __ S,
 620             as_PRegister($pg$$reg),
 621             as_FloatRegister($dst_src$$reg));
 622   %}
 623   ins_pipe(pipe_slow);
 624 %}
 625 
 626 instruct vabsL_masked(vReg dst_src, pRegGov pg) %{
 627   predicate(UseSVE > 0);
 628   match(Set dst_src (AbsVL dst_src pg));
 629   ins_cost(SVE_COST);
 630   format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
 631   ins_encode %{
 632     __ sve_abs(as_FloatRegister($dst_src$$reg), __ D,
 633             as_PRegister($pg$$reg),
 634             as_FloatRegister($dst_src$$reg));
 635   %}
 636   ins_pipe(pipe_slow);
 637 %}
 638 
 639 instruct vabsF_masked(vReg dst_src, pRegGov pg) %{
 640   predicate(UseSVE > 0);
 641   match(Set dst_src (AbsVF dst_src pg));
 642   ins_cost(SVE_COST);
 643   format %{ "sve_fabs $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
 644   ins_encode %{
 645     __ sve_fabs(as_FloatRegister($dst_src$$reg), __ S,
 646             as_PRegister($pg$$reg),
 647             as_FloatRegister($dst_src$$reg));
 648   %}
 649   ins_pipe(pipe_slow);
 650 %}
 651 
 652 instruct vabsD_masked(vReg dst_src, pRegGov pg) %{
 653   predicate(UseSVE > 0);
 654   match(Set dst_src (AbsVD dst_src pg));
 655   ins_cost(SVE_COST);
 656   format %{ "sve_fabs $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
 657   ins_encode %{
 658     __ sve_fabs(as_FloatRegister($dst_src$$reg), __ D,
 659             as_PRegister($pg$$reg),
 660             as_FloatRegister($dst_src$$reg));
 661   %}
 662   ins_pipe(pipe_slow);
 663 %}
 664 
 665 // vector add
 666 
 667 instruct vaddB(vReg dst, vReg src1, vReg src2) %{
 668   predicate(UseSVE > 0);
 669   match(Set dst (AddVB src1 src2));
 670   ins_cost(SVE_COST);
 671   format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (B)" %}
 672   ins_encode %{
 673     __ sve_add(as_FloatRegister($dst$$reg), __ B,
 674          as_FloatRegister($src1$$reg),
 675          as_FloatRegister($src2$$reg));
 676   %}
 677   ins_pipe(pipe_slow);
 678 %}
 679 
 680 instruct vaddS(vReg dst, vReg src1, vReg src2) %{
 681   predicate(UseSVE > 0);
 682   match(Set dst (AddVS src1 src2));
 683   ins_cost(SVE_COST);
 684   format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (H)" %}
 685   ins_encode %{
 686     __ sve_add(as_FloatRegister($dst$$reg), __ H,
 687          as_FloatRegister($src1$$reg),
 688          as_FloatRegister($src2$$reg));
 689   %}
 690   ins_pipe(pipe_slow);
 691 %}
 692 
 693 instruct vaddI(vReg dst, vReg src1, vReg src2) %{
 694   predicate(UseSVE > 0);
 695   match(Set dst (AddVI src1 src2));
 696   ins_cost(SVE_COST);
 697   format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (S)" %}
 698   ins_encode %{
 699     __ sve_add(as_FloatRegister($dst$$reg), __ S,
 700          as_FloatRegister($src1$$reg),
 701          as_FloatRegister($src2$$reg));
 702   %}
 703   ins_pipe(pipe_slow);
 704 %}
 705 
 706 instruct vaddL(vReg dst, vReg src1, vReg src2) %{
 707   predicate(UseSVE > 0);
 708   match(Set dst (AddVL src1 src2));
 709   ins_cost(SVE_COST);
 710   format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (D)" %}
 711   ins_encode %{
 712     __ sve_add(as_FloatRegister($dst$$reg), __ D,
 713          as_FloatRegister($src1$$reg),
 714          as_FloatRegister($src2$$reg));
 715   %}
 716   ins_pipe(pipe_slow);
 717 %}
 718 
 719 instruct vaddF(vReg dst, vReg src1, vReg src2) %{
 720   predicate(UseSVE > 0);
 721   match(Set dst (AddVF src1 src2));
 722   ins_cost(SVE_COST);
 723   format %{ "sve_fadd $dst, $src1, $src2\t # vector (sve) (S)" %}
 724   ins_encode %{
 725     __ sve_fadd(as_FloatRegister($dst$$reg), __ S,
 726          as_FloatRegister($src1$$reg),
 727          as_FloatRegister($src2$$reg));
 728   %}
 729   ins_pipe(pipe_slow);
 730 %}
 731 
 732 instruct vaddD(vReg dst, vReg src1, vReg src2) %{
 733   predicate(UseSVE > 0);
 734   match(Set dst (AddVD src1 src2));
 735   ins_cost(SVE_COST);
 736   format %{ "sve_fadd $dst, $src1, $src2\t # vector (sve) (D)" %}
 737   ins_encode %{
 738     __ sve_fadd(as_FloatRegister($dst$$reg), __ D,
 739          as_FloatRegister($src1$$reg),
 740          as_FloatRegister($src2$$reg));
 741   %}
 742   ins_pipe(pipe_slow);
 743 %}
 744 
 745 // vector add - predicated
 746 
 747 instruct vaddB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 748   predicate(UseSVE > 0);
 749   match(Set dst_src1 (AddVB (Binary dst_src1 src2) pg));
 750   ins_cost(SVE_COST);
 751   format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
 752   ins_encode %{
 753     __ sve_add(as_FloatRegister($dst_src1$$reg), __ B,
 754             as_PRegister($pg$$reg),
 755             as_FloatRegister($src2$$reg));
 756   %}
 757   ins_pipe(pipe_slow);
 758 %}
 759 
 760 instruct vaddS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 761   predicate(UseSVE > 0);
 762   match(Set dst_src1 (AddVS (Binary dst_src1 src2) pg));
 763   ins_cost(SVE_COST);
 764   format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
 765   ins_encode %{
 766     __ sve_add(as_FloatRegister($dst_src1$$reg), __ H,
 767             as_PRegister($pg$$reg),
 768             as_FloatRegister($src2$$reg));
 769   %}
 770   ins_pipe(pipe_slow);
 771 %}
 772 
 773 instruct vaddI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 774   predicate(UseSVE > 0);
 775   match(Set dst_src1 (AddVI (Binary dst_src1 src2) pg));
 776   ins_cost(SVE_COST);
 777   format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
 778   ins_encode %{
 779     __ sve_add(as_FloatRegister($dst_src1$$reg), __ S,
 780             as_PRegister($pg$$reg),
 781             as_FloatRegister($src2$$reg));
 782   %}
 783   ins_pipe(pipe_slow);
 784 %}
 785 
 786 instruct vaddL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 787   predicate(UseSVE > 0);
 788   match(Set dst_src1 (AddVL (Binary dst_src1 src2) pg));
 789   ins_cost(SVE_COST);
 790   format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
 791   ins_encode %{
 792     __ sve_add(as_FloatRegister($dst_src1$$reg), __ D,
 793             as_PRegister($pg$$reg),
 794             as_FloatRegister($src2$$reg));
 795   %}
 796   ins_pipe(pipe_slow);
 797 %}
 798 
 799 instruct vaddF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 800   predicate(UseSVE > 0);
 801   match(Set dst_src1 (AddVF (Binary dst_src1 src2) pg));
 802   ins_cost(SVE_COST);
 803   format %{ "sve_fadd $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
 804   ins_encode %{
 805     __ sve_fadd(as_FloatRegister($dst_src1$$reg), __ S,
 806             as_PRegister($pg$$reg),
 807             as_FloatRegister($src2$$reg));
 808   %}
 809   ins_pipe(pipe_slow);
 810 %}
 811 
 812 instruct vaddD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 813   predicate(UseSVE > 0);
 814   match(Set dst_src1 (AddVD (Binary dst_src1 src2) pg));
 815   ins_cost(SVE_COST);
 816   format %{ "sve_fadd $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
 817   ins_encode %{
 818     __ sve_fadd(as_FloatRegister($dst_src1$$reg), __ D,
 819             as_PRegister($pg$$reg),
 820             as_FloatRegister($src2$$reg));
 821   %}
 822   ins_pipe(pipe_slow);
 823 %}
 824 
 825 // vector add reg imm (unpredicated)
 826 
 827 instruct vaddImmB(vReg dst_src, immBAddSubV con) %{
 828   predicate(UseSVE > 0);
 829   match(Set dst_src (AddVB dst_src (ReplicateB con)));
 830   ins_cost(SVE_COST);
 831   format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) (B)" %}
 832   ins_encode %{
 833     int32_t val = $con$$constant;
 834     if (val > 0){
 835       __ sve_add(as_FloatRegister($dst_src$$reg), __ B, val);
 836     } else if (val < 0){
 837       __ sve_sub(as_FloatRegister($dst_src$$reg), __ B, -val);
 838     }
 839   %}
 840   ins_pipe(pipe_slow);
 841 %}
 842 
 843 instruct vaddImmS(vReg dst_src, immIAddSubV con) %{
 844   predicate(UseSVE > 0);
 845   match(Set dst_src (AddVS dst_src (ReplicateS con)));
 846   ins_cost(SVE_COST);
 847   format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) (H)" %}
 848   ins_encode %{
 849     int32_t val = $con$$constant;
 850     if (val > 0){
 851       __ sve_add(as_FloatRegister($dst_src$$reg), __ H, val);
 852     } else if (val < 0){
 853       __ sve_sub(as_FloatRegister($dst_src$$reg), __ H, -val);
 854     }
 855   %}
 856   ins_pipe(pipe_slow);
 857 %}
 858 
 859 instruct vaddImmI(vReg dst_src, immIAddSubV con) %{
 860   predicate(UseSVE > 0);
 861   match(Set dst_src (AddVI dst_src (ReplicateI con)));
 862   ins_cost(SVE_COST);
 863   format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) (S)" %}
 864   ins_encode %{
 865     int32_t val = $con$$constant;
 866     if (val > 0){
 867       __ sve_add(as_FloatRegister($dst_src$$reg), __ S, val);
 868     } else if (val < 0){
 869       __ sve_sub(as_FloatRegister($dst_src$$reg), __ S, -val);
 870     }
 871   %}
 872   ins_pipe(pipe_slow);
 873 %}
 874 
 875 instruct vaddImmL(vReg dst_src, immLAddSubV con) %{
 876   predicate(UseSVE > 0);
 877   match(Set dst_src (AddVL dst_src (ReplicateL con)));
 878   ins_cost(SVE_COST);
 879   format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) (D)" %}
 880   ins_encode %{
 881     int32_t val = $con$$constant;
 882     if (val > 0){
 883       __ sve_add(as_FloatRegister($dst_src$$reg), __ D, val);
 884     } else if (val < 0){
 885       __ sve_sub(as_FloatRegister($dst_src$$reg), __ D, -val);
 886     }
 887   %}
 888   ins_pipe(pipe_slow);
 889 %}
 890 
 891 // vector binary op reg imm (unpredicated)
 892 
 893 instruct vandB(vReg dst_src, immBLog con) %{
 894   predicate(UseSVE > 0);
 895   match(Set dst_src (AndV dst_src (ReplicateB con)));
 896   ins_cost(SVE_COST);
 897   format %{ "sve_and $dst_src, $dst_src, $con\t # vector (sve) (B)" %}
 898   ins_encode %{
 899     __ sve_and(as_FloatRegister($dst_src$$reg), __ B,
 900          (uint64_t)($con$$constant));
 901   %}
 902   ins_pipe(pipe_slow);
 903 %}
 904 
 905 instruct vandH(vReg dst_src, immSLog con) %{
 906   predicate(UseSVE > 0);
 907   match(Set dst_src (AndV dst_src (ReplicateS con)));
 908   ins_cost(SVE_COST);
 909   format %{ "sve_and $dst_src, $dst_src, $con\t # vector (sve) (H)" %}
 910   ins_encode %{
 911     __ sve_and(as_FloatRegister($dst_src$$reg), __ H,
 912          (uint64_t)($con$$constant));
 913   %}
 914   ins_pipe(pipe_slow);
 915 %}
 916 
 917 instruct vandS(vReg dst_src, immILog con) %{
 918   predicate(UseSVE > 0);
 919   match(Set dst_src (AndV dst_src (ReplicateI con)));
 920   ins_cost(SVE_COST);
 921   format %{ "sve_and $dst_src, $dst_src, $con\t # vector (sve) (S)" %}
 922   ins_encode %{
 923     __ sve_and(as_FloatRegister($dst_src$$reg), __ S,
 924          (uint64_t)($con$$constant));
 925   %}
 926   ins_pipe(pipe_slow);
 927 %}
 928 
 929 instruct vandD(vReg dst_src, immLLog con) %{
 930   predicate(UseSVE > 0);
 931   match(Set dst_src (AndV dst_src (ReplicateL con)));
 932   ins_cost(SVE_COST);
 933   format %{ "sve_and $dst_src, $dst_src, $con\t # vector (sve) (D)" %}
 934   ins_encode %{
 935     __ sve_and(as_FloatRegister($dst_src$$reg), __ D,
 936          (uint64_t)($con$$constant));
 937   %}
 938   ins_pipe(pipe_slow);
 939 %}
 940 
 941 instruct vorB(vReg dst_src, immBLog con) %{
 942   predicate(UseSVE > 0);
 943   match(Set dst_src (OrV dst_src (ReplicateB con)));
 944   ins_cost(SVE_COST);
 945   format %{ "sve_orr $dst_src, $dst_src, $con\t # vector (sve) (B)" %}
 946   ins_encode %{
 947     __ sve_orr(as_FloatRegister($dst_src$$reg), __ B,
 948          (uint64_t)($con$$constant));
 949   %}
 950   ins_pipe(pipe_slow);
 951 %}
 952 
 953 instruct vorH(vReg dst_src, immSLog con) %{
 954   predicate(UseSVE > 0);
 955   match(Set dst_src (OrV dst_src (ReplicateS con)));
 956   ins_cost(SVE_COST);
 957   format %{ "sve_orr $dst_src, $dst_src, $con\t # vector (sve) (H)" %}
 958   ins_encode %{
 959     __ sve_orr(as_FloatRegister($dst_src$$reg), __ H,
 960          (uint64_t)($con$$constant));
 961   %}
 962   ins_pipe(pipe_slow);
 963 %}
 964 
 965 instruct vorS(vReg dst_src, immILog con) %{
 966   predicate(UseSVE > 0);
 967   match(Set dst_src (OrV dst_src (ReplicateI con)));
 968   ins_cost(SVE_COST);
 969   format %{ "sve_orr $dst_src, $dst_src, $con\t # vector (sve) (S)" %}
 970   ins_encode %{
 971     __ sve_orr(as_FloatRegister($dst_src$$reg), __ S,
 972          (uint64_t)($con$$constant));
 973   %}
 974   ins_pipe(pipe_slow);
 975 %}
 976 
 977 instruct vorD(vReg dst_src, immLLog con) %{
 978   predicate(UseSVE > 0);
 979   match(Set dst_src (OrV dst_src (ReplicateL con)));
 980   ins_cost(SVE_COST);
 981   format %{ "sve_orr $dst_src, $dst_src, $con\t # vector (sve) (D)" %}
 982   ins_encode %{
 983     __ sve_orr(as_FloatRegister($dst_src$$reg), __ D,
 984          (uint64_t)($con$$constant));
 985   %}
 986   ins_pipe(pipe_slow);
 987 %}
 988 
 989 instruct vxorB(vReg dst_src, immBLog con) %{
 990   predicate(UseSVE > 0);
 991   match(Set dst_src (XorV dst_src (ReplicateB con)));
 992   ins_cost(SVE_COST);
 993   format %{ "sve_eor $dst_src, $dst_src, $con\t # vector (sve) (B)" %}
 994   ins_encode %{
 995     __ sve_eor(as_FloatRegister($dst_src$$reg), __ B,
 996          (uint64_t)($con$$constant));
 997   %}
 998   ins_pipe(pipe_slow);
 999 %}
1000 
1001 instruct vxorH(vReg dst_src, immSLog con) %{
1002   predicate(UseSVE > 0);
1003   match(Set dst_src (XorV dst_src (ReplicateS con)));
1004   ins_cost(SVE_COST);
1005   format %{ "sve_eor $dst_src, $dst_src, $con\t # vector (sve) (H)" %}
1006   ins_encode %{
1007     __ sve_eor(as_FloatRegister($dst_src$$reg), __ H,
1008          (uint64_t)($con$$constant));
1009   %}
1010   ins_pipe(pipe_slow);
1011 %}
1012 
1013 instruct vxorS(vReg dst_src, immILog con) %{
1014   predicate(UseSVE > 0);
1015   match(Set dst_src (XorV dst_src (ReplicateI con)));
1016   ins_cost(SVE_COST);
1017   format %{ "sve_eor $dst_src, $dst_src, $con\t # vector (sve) (S)" %}
1018   ins_encode %{
1019     __ sve_eor(as_FloatRegister($dst_src$$reg), __ S,
1020          (uint64_t)($con$$constant));
1021   %}
1022   ins_pipe(pipe_slow);
1023 %}
1024 
1025 instruct vxorD(vReg dst_src, immLLog con) %{
1026   predicate(UseSVE > 0);
1027   match(Set dst_src (XorV dst_src (ReplicateL con)));
1028   ins_cost(SVE_COST);
1029   format %{ "sve_eor $dst_src, $dst_src, $con\t # vector (sve) (D)" %}
1030   ins_encode %{
1031     __ sve_eor(as_FloatRegister($dst_src$$reg), __ D,
1032          (uint64_t)($con$$constant));
1033   %}
1034   ins_pipe(pipe_slow);
1035 %}
1036 // vector and
1037 
1038 instruct vand(vReg dst, vReg src1, vReg src2) %{
1039   predicate(UseSVE > 0);
1040   match(Set dst (AndV src1 src2));
1041   ins_cost(SVE_COST);
1042   format %{ "sve_and  $dst, $src1, $src2\t# vector (sve)" %}
1043   ins_encode %{
1044     __ sve_and(as_FloatRegister($dst$$reg),
1045          as_FloatRegister($src1$$reg),
1046          as_FloatRegister($src2$$reg));
1047   %}
1048   ins_pipe(pipe_slow);
1049 %}
1050 
1051 // vector or
1052 
1053 instruct vor(vReg dst, vReg src1, vReg src2) %{
1054   predicate(UseSVE > 0);
1055   match(Set dst (OrV src1 src2));
1056   ins_cost(SVE_COST);
1057   format %{ "sve_orr  $dst, $src1, $src2\t# vector (sve)" %}
1058   ins_encode %{
1059     __ sve_orr(as_FloatRegister($dst$$reg),
1060          as_FloatRegister($src1$$reg),
1061          as_FloatRegister($src2$$reg));
1062   %}
1063   ins_pipe(pipe_slow);
1064 %}
1065 
1066 // vector xor
1067 
1068 instruct vxor(vReg dst, vReg src1, vReg src2) %{
1069   predicate(UseSVE > 0);
1070   match(Set dst (XorV src1 src2));
1071   ins_cost(SVE_COST);
1072   format %{ "sve_eor  $dst, $src1, $src2\t# vector (sve)" %}
1073   ins_encode %{
1074     __ sve_eor(as_FloatRegister($dst$$reg),
1075          as_FloatRegister($src1$$reg),
1076          as_FloatRegister($src2$$reg));
1077   %}
1078   ins_pipe(pipe_slow);
1079 %}
1080 
1081 // vector and - predicated
1082 
1083 instruct vand_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1084   predicate(UseSVE > 0);
1085   match(Set dst_src1 (AndV (Binary dst_src1 src2) pg));
1086   ins_cost(SVE_COST);
1087   format %{ "sve_and $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
1088   ins_encode %{
1089     BasicType bt = Matcher::vector_element_basic_type(this);
1090     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1091     __ sve_and(as_FloatRegister($dst_src1$$reg), size,
1092           as_PRegister($pg$$reg),
1093           as_FloatRegister($src2$$reg));
1094   %}
1095   ins_pipe(pipe_slow);
1096 %}
1097 
1098 // vector or - predicated
1099 
1100 instruct vor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1101   predicate(UseSVE > 0);
1102   match(Set dst_src1 (OrV (Binary dst_src1 src2) pg));
1103   ins_cost(SVE_COST);
1104   format %{ "sve_orr $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
1105   ins_encode %{
1106     BasicType bt = Matcher::vector_element_basic_type(this);
1107     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1108     __ sve_orr(as_FloatRegister($dst_src1$$reg), size,
1109           as_PRegister($pg$$reg),
1110           as_FloatRegister($src2$$reg));
1111   %}
1112   ins_pipe(pipe_slow);
1113 %}
1114 
1115 // vector xor - predicated
1116 
1117 instruct vxor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1118   predicate(UseSVE > 0);
1119   match(Set dst_src1 (XorV (Binary dst_src1 src2) pg));
1120   ins_cost(SVE_COST);
1121   format %{ "sve_eor $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
1122   ins_encode %{
1123     BasicType bt = Matcher::vector_element_basic_type(this);
1124     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1125     __ sve_eor(as_FloatRegister($dst_src1$$reg), size,
1126           as_PRegister($pg$$reg),
1127           as_FloatRegister($src2$$reg));
1128   %}
1129   ins_pipe(pipe_slow);
1130 %}
1131 
1132 // vector not
1133 
1134 instruct vnotI(vReg dst, vReg src, immI_M1 m1) %{
1135   predicate(UseSVE > 0);
1136   match(Set dst (XorV src (ReplicateB m1)));
1137   match(Set dst (XorV src (ReplicateS m1)));
1138   match(Set dst (XorV src (ReplicateI m1)));
1139   ins_cost(SVE_COST);
1140   format %{ "sve_not $dst, $src\t# vector (sve) B/H/S" %}
1141   ins_encode %{
1142     __ sve_not(as_FloatRegister($dst$$reg), __ D,
1143                ptrue, as_FloatRegister($src$$reg));
1144   %}
1145   ins_pipe(pipe_slow);
1146 %}
1147 
1148 instruct vnotL(vReg dst, vReg src, immL_M1 m1) %{
1149   predicate(UseSVE > 0);
1150   match(Set dst (XorV src (ReplicateL m1)));
1151   ins_cost(SVE_COST);
1152   format %{ "sve_not $dst, $src\t# vector (sve) D" %}
1153   ins_encode %{
1154     __ sve_not(as_FloatRegister($dst$$reg), __ D,
1155                ptrue, as_FloatRegister($src$$reg));
1156   %}
1157   ins_pipe(pipe_slow);
1158 %}
1159 
1160 // vector not - predicated
1161 
1162 instruct vnotI_masked(vReg dst, vReg src, immI_M1 m1, pRegGov pg) %{
1163   predicate(UseSVE > 0);
1164   match(Set dst (XorV (Binary src (ReplicateB m1)) pg));
1165   match(Set dst (XorV (Binary src (ReplicateS m1)) pg));
1166   match(Set dst (XorV (Binary src (ReplicateI m1)) pg));
1167   ins_cost(SVE_COST);
1168   format %{ "sve_not $dst, $pg, $src\t# vector (sve) B/H/S" %}
1169   ins_encode %{
1170     BasicType bt = Matcher::vector_element_basic_type(this);
1171     __ sve_not(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
1172                as_PRegister($pg$$reg), as_FloatRegister($src$$reg));
1173   %}
1174   ins_pipe(pipe_slow);
1175 %}
1176 
1177 instruct vnotL_masked(vReg dst, vReg src, immL_M1 m1, pRegGov pg) %{
1178   predicate(UseSVE > 0);
1179   match(Set dst (XorV (Binary src (ReplicateL m1)) pg));
1180   ins_cost(SVE_COST);
1181   format %{ "sve_not $dst, $pg, $src\t# vector (sve) D" %}
1182   ins_encode %{
1183     BasicType bt = Matcher::vector_element_basic_type(this);
1184     __ sve_not(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
1185                as_PRegister($pg$$reg), as_FloatRegister($src$$reg));
1186   %}
1187   ins_pipe(pipe_slow);
1188 %}
1189 
1190 // vector and_not
1191 
1192 instruct vand_notI(vReg dst, vReg src1, vReg src2, immI_M1 m1) %{
1193   predicate(UseSVE > 0);
1194   match(Set dst (AndV src1 (XorV src2 (ReplicateB m1))));
1195   match(Set dst (AndV src1 (XorV src2 (ReplicateS m1))));
1196   match(Set dst (AndV src1 (XorV src2 (ReplicateI m1))));
1197   ins_cost(SVE_COST);
1198   format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) B/H/S" %}
1199   ins_encode %{
1200     __ sve_bic(as_FloatRegister($dst$$reg),
1201                as_FloatRegister($src1$$reg),
1202                as_FloatRegister($src2$$reg));
1203   %}
1204   ins_pipe(pipe_slow);
1205 %}
1206 
1207 instruct vand_notL(vReg dst, vReg src1, vReg src2, immL_M1 m1) %{
1208   predicate(UseSVE > 0);
1209   match(Set dst (AndV src1 (XorV src2 (ReplicateL m1))));
1210   ins_cost(SVE_COST);
1211   format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) D" %}
1212   ins_encode %{
1213     __ sve_bic(as_FloatRegister($dst$$reg),
1214                as_FloatRegister($src1$$reg),
1215                as_FloatRegister($src2$$reg));
1216   %}
1217   ins_pipe(pipe_slow);
1218 %}
1219 
1220 // vector and_not - predicated
1221 
1222 instruct vand_notI_masked(vReg dst_src1, vReg src2, immI_M1 m1, pRegGov pg) %{
1223   predicate(UseSVE > 0);
1224   match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateB m1))) pg));
1225   match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateS m1))) pg));
1226   match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateI m1))) pg));
1227   ins_cost(SVE_COST);
1228   format %{ "sve_bic $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) B/H/S" %}
1229   ins_encode %{
1230     BasicType bt = Matcher::vector_element_basic_type(this);
1231     __ sve_bic(as_FloatRegister($dst_src1$$reg), __ elemType_to_regVariant(bt),
1232                as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1233   %}
1234   ins_pipe(pipe_slow);
1235 %}
1236 
1237 instruct vand_notL_masked(vReg dst_src1, vReg src2, immL_M1 m1, pRegGov pg) %{
1238   predicate(UseSVE > 0);
1239   match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateL m1))) pg));
1240   ins_cost(SVE_COST);
1241   format %{ "sve_bic $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) D" %}
1242   ins_encode %{
1243     BasicType bt = Matcher::vector_element_basic_type(this);
1244     __ sve_bic(as_FloatRegister($dst_src1$$reg), __ elemType_to_regVariant(bt),
1245                as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1246   %}
1247   ins_pipe(pipe_slow);
1248 %}
1249 
1250 // vector float div
1251 
1252 instruct vdivF(vReg dst_src1, vReg src2) %{
1253   predicate(UseSVE > 0);
1254   match(Set dst_src1 (DivVF dst_src1 src2));
1255   ins_cost(SVE_COST);
1256   format %{ "sve_fdiv  $dst_src1, $dst_src1, $src2\t# vector (sve) (S)" %}
1257   ins_encode %{
1258     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ S,
1259          ptrue, as_FloatRegister($src2$$reg));
1260   %}
1261   ins_pipe(pipe_slow);
1262 %}
1263 
1264 instruct vdivD(vReg dst_src1, vReg src2) %{
1265   predicate(UseSVE > 0);
1266   match(Set dst_src1 (DivVD dst_src1 src2));
1267   ins_cost(SVE_COST);
1268   format %{ "sve_fdiv  $dst_src1, $dst_src1, $src2\t# vector (sve) (D)" %}
1269   ins_encode %{
1270     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ D,
1271          ptrue, as_FloatRegister($src2$$reg));
1272   %}
1273   ins_pipe(pipe_slow);
1274 %}
1275 
1276 // vector float div - predicated
1277 
1278 instruct vfdivF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1279   predicate(UseSVE > 0);
1280   match(Set dst_src1 (DivVF (Binary dst_src1 src2) pg));
1281   ins_cost(SVE_COST);
1282   format %{ "sve_fdiv $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
1283   ins_encode %{
1284     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ S,
1285             as_PRegister($pg$$reg),
1286             as_FloatRegister($src2$$reg));
1287   %}
1288   ins_pipe(pipe_slow);
1289 %}
1290 
1291 instruct vfdivD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1292   predicate(UseSVE > 0);
1293   match(Set dst_src1 (DivVD (Binary dst_src1 src2) pg));
1294   ins_cost(SVE_COST);
1295   format %{ "sve_fdiv $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
1296   ins_encode %{
1297     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ D,
1298             as_PRegister($pg$$reg),
1299             as_FloatRegister($src2$$reg));
1300   %}
1301   ins_pipe(pipe_slow);
1302 %}
1303 
1304 // vector min/max
1305 
1306 instruct vmin(vReg dst_src1, vReg src2) %{
1307   predicate(UseSVE > 0);
1308   match(Set dst_src1 (MinV dst_src1 src2));
1309   ins_cost(SVE_COST);
1310   format %{ "sve_min $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
1311   ins_encode %{
1312     BasicType bt = Matcher::vector_element_basic_type(this);
1313     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1314     if (is_floating_point_type(bt)) {
1315       __ sve_fmin(as_FloatRegister($dst_src1$$reg), size,
1316                   ptrue, as_FloatRegister($src2$$reg));
1317     } else {
1318       assert(is_integral_type(bt), "unsupported type");
1319       __ sve_smin(as_FloatRegister($dst_src1$$reg), size,
1320                   ptrue, as_FloatRegister($src2$$reg));
1321     }
1322   %}
1323   ins_pipe(pipe_slow);
1324 %}
1325 
1326 instruct vmax(vReg dst_src1, vReg src2) %{
1327   predicate(UseSVE > 0);
1328   match(Set dst_src1 (MaxV dst_src1 src2));
1329   ins_cost(SVE_COST);
1330   format %{ "sve_max $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
1331   ins_encode %{
1332     BasicType bt = Matcher::vector_element_basic_type(this);
1333     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1334     if (is_floating_point_type(bt)) {
1335       __ sve_fmax(as_FloatRegister($dst_src1$$reg), size,
1336                   ptrue, as_FloatRegister($src2$$reg));
1337     } else {
1338       assert(is_integral_type(bt), "unsupported type");
1339       __ sve_smax(as_FloatRegister($dst_src1$$reg), size,
1340                   ptrue, as_FloatRegister($src2$$reg));
1341     }
1342   %}
1343   ins_pipe(pipe_slow);
1344 %}
1345 
1346 // vector min/max - predicated
1347 
1348 instruct vmin_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1349   predicate(UseSVE > 0);
1350   match(Set dst_src1 (MinV (Binary dst_src1 src2) pg));
1351   ins_cost(SVE_COST);
1352   format %{ "sve_min $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %}
1353   ins_encode %{
1354     BasicType bt = Matcher::vector_element_basic_type(this);
1355     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1356     if (is_floating_point_type(bt)) {
1357       __ sve_fmin(as_FloatRegister($dst_src1$$reg), size,
1358                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1359     } else {
1360       assert(is_integral_type(bt), "unsupported type");
1361       __ sve_smin(as_FloatRegister($dst_src1$$reg), size,
1362                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1363     }
1364   %}
1365   ins_pipe(pipe_slow);
1366 %}
1367 
1368 instruct vmax_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1369   predicate(UseSVE > 0);
1370   match(Set dst_src1 (MaxV (Binary dst_src1 src2) pg));
1371   ins_cost(SVE_COST);
1372   format %{ "sve_max $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %}
1373   ins_encode %{
1374     BasicType bt = Matcher::vector_element_basic_type(this);
1375     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1376     if (is_floating_point_type(bt)) {
1377       __ sve_fmax(as_FloatRegister($dst_src1$$reg), size,
1378                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1379     } else {
1380       assert(is_integral_type(bt), "unsupported type");
1381       __ sve_smax(as_FloatRegister($dst_src1$$reg), size,
1382                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1383     }
1384   %}
1385   ins_pipe(pipe_slow);
1386 %}
1387 
1388 // vector fmla
1389 
1390 // dst_src1 = dst_src1 + src2 * src3
1391 instruct vfmlaF(vReg dst_src1, vReg src2, vReg src3) %{
1392   predicate(UseFMA && UseSVE > 0);
1393   match(Set dst_src1 (FmaVF dst_src1 (Binary src2 src3)));
1394   ins_cost(SVE_COST);
1395   format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1396   ins_encode %{
1397     __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ S,
1398          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1399   %}
1400   ins_pipe(pipe_slow);
1401 %}
1402 
1403 // dst_src1 = dst_src1 + src2 * src3
1404 instruct vfmlaD(vReg dst_src1, vReg src2, vReg src3) %{
1405   predicate(UseFMA && UseSVE > 0);
1406   match(Set dst_src1 (FmaVD dst_src1 (Binary src2 src3)));
1407   ins_cost(SVE_COST);
1408   format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1409   ins_encode %{
1410     __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ D,
1411          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1412   %}
1413   ins_pipe(pipe_slow);
1414 %}
1415 
1416 // vector fmad - predicated
1417 
1418 // dst_src1 = dst_src1 * src2 + src3
1419 instruct vfmadF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1420   predicate(UseFMA && UseSVE > 0);
1421   match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary src3 pg)));
1422   ins_cost(SVE_COST);
1423   format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) (S)" %}
1424   ins_encode %{
1425     __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
1426          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1427   %}
1428   ins_pipe(pipe_slow);
1429 %}
1430 
1431 // dst_src1 = dst_src1 * src2 + src3
1432 instruct vfmadD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1433   predicate(UseFMA && UseSVE > 0);
1434   match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary src3 pg)));
1435   ins_cost(SVE_COST);
1436   format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) (D)" %}
1437   ins_encode %{
1438     __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
1439          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1440   %}
1441   ins_pipe(pipe_slow);
1442 %}
1443 
1444 // vector fmls
1445 
1446 // dst_src1 = dst_src1 + -src2 * src3
1447 // The NegVF must not be predicated.
1448 instruct vfmlsF1(vReg dst_src1, vReg src2, vReg src3) %{
1449   predicate(UseFMA && UseSVE > 0 &&
1450             !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1451   match(Set dst_src1 (FmaVF dst_src1 (Binary (NegVF src2) src3)));
1452   ins_cost(SVE_COST);
1453   format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1454   ins_encode %{
1455     __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ S,
1456          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1457   %}
1458   ins_pipe(pipe_slow);
1459 %}
1460 
1461 // dst_src1 = dst_src1 + src2 * -src3
1462 // The NegVF must not be predicated.
1463 instruct vfmlsF2(vReg dst_src1, vReg src2, vReg src3) %{
1464   predicate(UseFMA && UseSVE > 0 &&
1465             !n->in(2)->in(2)->as_Vector()->is_predicated_vector());
1466   match(Set dst_src1 (FmaVF dst_src1 (Binary src2 (NegVF src3))));
1467   ins_cost(SVE_COST);
1468   format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1469   ins_encode %{
1470     __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ S,
1471          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1472   %}
1473   ins_pipe(pipe_slow);
1474 %}
1475 
1476 // dst_src1 = dst_src1 + -src2 * src3
1477 // The NegVD must not be predicated.
1478 instruct vfmlsD1(vReg dst_src1, vReg src2, vReg src3) %{
1479   predicate(UseFMA && UseSVE > 0 &&
1480             !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1481   match(Set dst_src1 (FmaVD dst_src1 (Binary (NegVD src2) src3)));
1482   ins_cost(SVE_COST);
1483   format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1484   ins_encode %{
1485     __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ D,
1486          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1487   %}
1488   ins_pipe(pipe_slow);
1489 %}
1490 
1491 // dst_src1 = dst_src1 + src2 * -src3
1492 // The NegVD must not be predicated.
1493 instruct vfmlsD2(vReg dst_src1, vReg src2, vReg src3) %{
1494   predicate(UseFMA && UseSVE > 0 &&
1495             !n->in(2)->in(2)->as_Vector()->is_predicated_vector());
1496   match(Set dst_src1 (FmaVD dst_src1 (Binary src2 (NegVD src3))));
1497   ins_cost(SVE_COST);
1498   format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1499   ins_encode %{
1500     __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ D,
1501          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1502   %}
1503   ins_pipe(pipe_slow);
1504 %}
1505 
1506 // vector fmsb - predicated
1507 
1508 // dst_src1 = dst_src1 * -src2 + src3
1509 // The NegVF must not be predicated.
1510 instruct vfmsbF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1511   predicate(UseFMA && UseSVE > 0 &&
1512             !n->in(1)->in(2)->as_Vector()->is_predicated_vector());
1513   match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary src3 pg)));
1514   ins_cost(SVE_COST);
1515   format %{ "sve_fmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) (S)" %}
1516   ins_encode %{
1517     __ sve_fmsb(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
1518          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1519   %}
1520   ins_pipe(pipe_slow);
1521 %}
1522 
1523 // dst_src1 = dst_src1 * -src2 + src3
1524 // The NegVD must not be predicated.
1525 instruct vfmsbD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1526   predicate(UseFMA && UseSVE > 0 &&
1527             !n->in(1)->in(2)->as_Vector()->is_predicated_vector());
1528   match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary src3 pg)));
1529   ins_cost(SVE_COST);
1530   format %{ "sve_fmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) (D)" %}
1531   ins_encode %{
1532     __ sve_fmsb(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
1533          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1534   %}
1535   ins_pipe(pipe_slow);
1536 %}
1537 
1538 // vector fnmla
1539 
1540 // dst_src1 = -dst_src1 + -src2 * src3
1541 // The NegVF must not be predicated.
1542 instruct vfnmlaF1(vReg dst_src1, vReg src2, vReg src3) %{
1543   predicate(UseFMA && UseSVE > 0 &&
1544             !n->in(1)->as_Vector()->is_predicated_vector() &&
1545             !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1546   match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary (NegVF src2) src3)));
1547   ins_cost(SVE_COST);
1548   format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1549   ins_encode %{
1550     __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ S,
1551          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1552   %}
1553   ins_pipe(pipe_slow);
1554 %}
1555 
1556 // dst_src1 = -dst_src1 + src2 * -src3
1557 // The NegVF must not be predicated.
1558 instruct vfnmlaF2(vReg dst_src1, vReg src2, vReg src3) %{
1559   predicate(UseFMA && UseSVE > 0 &&
1560             !n->in(1)->as_Vector()->is_predicated_vector() &&
1561             !n->in(2)->in(2)->as_Vector()->is_predicated_vector());
1562   match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 (NegVF src3))));
1563   ins_cost(SVE_COST);
1564   format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1565   ins_encode %{
1566     __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ S,
1567          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1568   %}
1569   ins_pipe(pipe_slow);
1570 %}
1571 
1572 // dst_src1 = -dst_src1 + -src2 * src3
1573 // The NegVD must not be predicated.
1574 instruct vfnmlaD1(vReg dst_src1, vReg src2, vReg src3) %{
1575   predicate(UseFMA && UseSVE > 0 &&
1576             !n->in(1)->as_Vector()->is_predicated_vector() &&
1577             !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1578   match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary (NegVD src2) src3)));
1579   ins_cost(SVE_COST);
1580   format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1581   ins_encode %{
1582     __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ D,
1583          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1584   %}
1585   ins_pipe(pipe_slow);
1586 %}
1587 
1588 // dst_src1 = -dst_src1 + src2 * -src3
1589 // The NegVD must not be predicated.
1590 instruct vfnmlaD2(vReg dst_src1, vReg src2, vReg src3) %{
1591   predicate(UseFMA && UseSVE > 0 &&
1592             !n->in(1)->as_Vector()->is_predicated_vector() &&
1593             !n->in(2)->in(2)->as_Vector()->is_predicated_vector());
1594   match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 (NegVD src3))));
1595   ins_cost(SVE_COST);
1596   format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1597   ins_encode %{
1598     __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ D,
1599          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1600   %}
1601   ins_pipe(pipe_slow);
1602 %}
1603 
1604 // vector fnmad - predicated
1605 
1606 // dst_src1 = -src3 + dst_src1 * -src2
1607 // The NegVF must not be predicated.
1608 instruct vfnmadF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1609   predicate(UseFMA && UseSVE > 0 &&
1610             !n->in(1)->in(2)->as_Vector()->is_predicated_vector() &&
1611             !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1612   match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary (NegVF src3) pg)));
1613   ins_cost(SVE_COST);
1614   format %{ "sve_fnmad $dst_src1, $pg, $src2, $src3\t # vector (sve) (S)" %}
1615   ins_encode %{
1616     __ sve_fnmad(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
1617          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1618   %}
1619   ins_pipe(pipe_slow);
1620 %}
1621 
1622 // dst_src1 = -src3 + dst_src1 * -src2
1623 // The NegVD must not be predicated.
1624 instruct vfnmadD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1625   predicate(UseFMA && UseSVE > 0 &&
1626             !n->in(1)->in(2)->as_Vector()->is_predicated_vector() &&
1627             !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1628   match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary (NegVD src3) pg)));
1629   ins_cost(SVE_COST);
1630   format %{ "sve_fnmad $dst_src1, $pg, $src2, $src3\t # vector (sve) (D)" %}
1631   ins_encode %{
1632     __ sve_fnmad(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
1633          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1634   %}
1635   ins_pipe(pipe_slow);
1636 %}
1637 
1638 // vector fnmls
1639 
1640 // dst_src1 = -dst_src1 + src2 * src3
1641 // The NegVF must not be predicated.
1642 instruct vfnmlsF(vReg dst_src1, vReg src2, vReg src3) %{
1643   predicate(UseFMA && UseSVE > 0 &&
1644             !n->in(1)->as_Vector()->is_predicated_vector());
1645   match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 src3)));
1646   ins_cost(SVE_COST);
1647   format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1648   ins_encode %{
1649     __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ S,
1650          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1651   %}
1652   ins_pipe(pipe_slow);
1653 %}
1654 
1655 // dst_src1 = -dst_src1 + src2 * src3
1656 // The NegVD must not be predicated.
1657 instruct vfnmlsD(vReg dst_src1, vReg src2, vReg src3) %{
1658   predicate(UseFMA && UseSVE > 0 &&
1659             !n->in(1)->as_Vector()->is_predicated_vector());
1660   match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 src3)));
1661   ins_cost(SVE_COST);
1662   format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1663   ins_encode %{
1664     __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ D,
1665          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1666   %}
1667   ins_pipe(pipe_slow);
1668 %}
1669 
1670 // vector fnmsb - predicated
1671 
1672 // dst_src1 = -src3 + dst_src1 * src2
1673 // The NegVF must not be predicated.
1674 instruct vfnmsbF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1675   predicate(UseFMA && UseSVE > 0 &&
1676             !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1677   match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary (NegVF src3) pg)));
1678   ins_cost(SVE_COST);
1679   format %{ "sve_fnmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) (S)" %}
1680   ins_encode %{
1681     __ sve_fnmsb(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
1682          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1683   %}
1684   ins_pipe(pipe_slow);
1685 %}
1686 
1687 // dst_src1 = -src3 + dst_src1 * src2
1688 // The NegVD must not be predicated.
1689 instruct vfnmsbD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1690   predicate(UseFMA && UseSVE > 0 &&
1691             !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1692   match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary (NegVD src3) pg)));
1693   ins_cost(SVE_COST);
1694   format %{ "sve_fnmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) (D)" %}
1695   ins_encode %{
1696     __ sve_fnmsb(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
1697          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1698   %}
1699   ins_pipe(pipe_slow);
1700 %}
1701 
1702 // vector mla
1703 
1704 // dst_src1 = dst_src1 + src2 * src3
1705 instruct vmlaB(vReg dst_src1, vReg src2, vReg src3)
1706 %{
1707   predicate(UseSVE > 0);
1708   match(Set dst_src1 (AddVB dst_src1 (MulVB src2 src3)));
1709   ins_cost(SVE_COST);
1710   format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (B)" %}
1711   ins_encode %{
1712     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ B,
1713       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1714   %}
1715   ins_pipe(pipe_slow);
1716 %}
1717 
1718 // dst_src1 = dst_src1 + src2 * src3
1719 instruct vmlaS(vReg dst_src1, vReg src2, vReg src3)
1720 %{
1721   predicate(UseSVE > 0);
1722   match(Set dst_src1 (AddVS dst_src1 (MulVS src2 src3)));
1723   ins_cost(SVE_COST);
1724   format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (H)" %}
1725   ins_encode %{
1726     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ H,
1727       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1728   %}
1729   ins_pipe(pipe_slow);
1730 %}
1731 
1732 // dst_src1 = dst_src1 + src2 * src3
1733 instruct vmlaI(vReg dst_src1, vReg src2, vReg src3)
1734 %{
1735   predicate(UseSVE > 0);
1736   match(Set dst_src1 (AddVI dst_src1 (MulVI src2 src3)));
1737   ins_cost(SVE_COST);
1738   format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (S)" %}
1739   ins_encode %{
1740     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ S,
1741       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1742   %}
1743   ins_pipe(pipe_slow);
1744 %}
1745 
1746 // dst_src1 = dst_src1 + src2 * src3
1747 instruct vmlaL(vReg dst_src1, vReg src2, vReg src3)
1748 %{
1749   predicate(UseSVE > 0);
1750   match(Set dst_src1 (AddVL dst_src1 (MulVL src2 src3)));
1751   ins_cost(SVE_COST);
1752   format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (D)" %}
1753   ins_encode %{
1754     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ D,
1755       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1756   %}
1757   ins_pipe(pipe_slow);
1758 %}
1759 
1760 // vector mla - predicated
1761 
1762 // dst_src1 = dst_src1 + src2 * src3
1763 instruct vmlaB_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1764 %{
1765   predicate(UseSVE > 0);
1766   match(Set dst_src1 (AddVB (Binary dst_src1 (MulVB src2 src3)) pg));
1767   ins_cost(SVE_COST);
1768   format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) (B)" %}
1769   ins_encode %{
1770     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ B, as_PRegister($pg$$reg),
1771          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1772   %}
1773   ins_pipe(pipe_slow);
1774 %}
1775 
1776 // dst_src1 = dst_src1 + src2 * src3
1777 instruct vmlaS_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1778 %{
1779   predicate(UseSVE > 0);
1780   match(Set dst_src1 (AddVS (Binary dst_src1 (MulVS src2 src3)) pg));
1781   ins_cost(SVE_COST);
1782   format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) (H)" %}
1783   ins_encode %{
1784     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ H, as_PRegister($pg$$reg),
1785          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1786   %}
1787   ins_pipe(pipe_slow);
1788 %}
1789 
1790 // dst_src1 = dst_src1 + src2 * src3
1791 instruct vmlaI_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1792 %{
1793   predicate(UseSVE > 0);
1794   match(Set dst_src1 (AddVI (Binary dst_src1 (MulVI src2 src3)) pg));
1795   ins_cost(SVE_COST);
1796   format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) (S)" %}
1797   ins_encode %{
1798     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
1799          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1800   %}
1801   ins_pipe(pipe_slow);
1802 %}
1803 
1804 // dst_src1 = dst_src1 + src2 * src3
1805 instruct vmlaL_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1806 %{
1807   predicate(UseSVE > 0);
1808   match(Set dst_src1 (AddVL (Binary dst_src1 (MulVL src2 src3)) pg));
1809   ins_cost(SVE_COST);
1810   format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) (D)" %}
1811   ins_encode %{
1812     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
1813          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1814   %}
1815   ins_pipe(pipe_slow);
1816 %}
1817 
1818 // vector mls
1819 
1820 // dst_src1 = dst_src1 - src2 * src3
1821 instruct vmlsB(vReg dst_src1, vReg src2, vReg src3)
1822 %{
1823   predicate(UseSVE > 0);
1824   match(Set dst_src1 (SubVB dst_src1 (MulVB src2 src3)));
1825   ins_cost(SVE_COST);
1826   format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (B)" %}
1827   ins_encode %{
1828     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ B,
1829       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1830   %}
1831   ins_pipe(pipe_slow);
1832 %}
1833 
1834 // dst_src1 = dst_src1 - src2 * src3
1835 instruct vmlsS(vReg dst_src1, vReg src2, vReg src3)
1836 %{
1837   predicate(UseSVE > 0);
1838   match(Set dst_src1 (SubVS dst_src1 (MulVS src2 src3)));
1839   ins_cost(SVE_COST);
1840   format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (H)" %}
1841   ins_encode %{
1842     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ H,
1843       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1844   %}
1845   ins_pipe(pipe_slow);
1846 %}
1847 
1848 // dst_src1 = dst_src1 - src2 * src3
1849 instruct vmlsI(vReg dst_src1, vReg src2, vReg src3)
1850 %{
1851   predicate(UseSVE > 0);
1852   match(Set dst_src1 (SubVI dst_src1 (MulVI src2 src3)));
1853   ins_cost(SVE_COST);
1854   format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (S)" %}
1855   ins_encode %{
1856     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ S,
1857       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1858   %}
1859   ins_pipe(pipe_slow);
1860 %}
1861 
1862 // dst_src1 = dst_src1 - src2 * src3
1863 instruct vmlsL(vReg dst_src1, vReg src2, vReg src3)
1864 %{
1865   predicate(UseSVE > 0);
1866   match(Set dst_src1 (SubVL dst_src1 (MulVL src2 src3)));
1867   ins_cost(SVE_COST);
1868   format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (D)" %}
1869   ins_encode %{
1870     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ D,
1871       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1872   %}
1873   ins_pipe(pipe_slow);
1874 %}
1875 
1876 // vector mls - predicated
1877 
1878 // dst_src1 = dst_src1 - src2 * src3
1879 instruct vmlsB_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1880 %{
1881   predicate(UseSVE > 0);
1882   match(Set dst_src1 (SubVB (Binary dst_src1 (MulVB src2 src3)) pg));
1883   ins_cost(SVE_COST);
1884   format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) (B)" %}
1885   ins_encode %{
1886     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ B, as_PRegister($pg$$reg),
1887          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1888   %}
1889   ins_pipe(pipe_slow);
1890 %}
1891 
1892 // dst_src1 = dst_src1 - src2 * src3
1893 instruct vmlsS_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1894 %{
1895   predicate(UseSVE > 0);
1896   match(Set dst_src1 (SubVS (Binary dst_src1 (MulVS src2 src3)) pg));
1897   ins_cost(SVE_COST);
1898   format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) (H)" %}
1899   ins_encode %{
1900     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ H, as_PRegister($pg$$reg),
1901          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1902   %}
1903   ins_pipe(pipe_slow);
1904 %}
1905 
1906 // dst_src1 = dst_src1 - src2 * src3
1907 instruct vmlsI_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1908 %{
1909   predicate(UseSVE > 0);
1910   match(Set dst_src1 (SubVI (Binary dst_src1 (MulVI src2 src3)) pg));
1911   ins_cost(SVE_COST);
1912   format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) (S)" %}
1913   ins_encode %{
1914     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
1915          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1916   %}
1917   ins_pipe(pipe_slow);
1918 %}
1919 
1920 // dst_src1 = dst_src1 - src2 * src3
1921 instruct vmlsL_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1922 %{
1923   predicate(UseSVE > 0);
1924   match(Set dst_src1 (SubVL (Binary dst_src1 (MulVL src2 src3)) pg));
1925   ins_cost(SVE_COST);
1926   format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) (D)" %}
1927   ins_encode %{
1928     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
1929          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1930   %}
1931   ins_pipe(pipe_slow);
1932 %}
1933 
1934 // vector mul
1935 
1936 instruct vmulB(vReg dst_src1, vReg src2) %{
1937   predicate(UseSVE > 0);
1938   match(Set dst_src1 (MulVB dst_src1 src2));
1939   ins_cost(SVE_COST);
1940   format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (B)" %}
1941   ins_encode %{
1942     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ B,
1943          ptrue, as_FloatRegister($src2$$reg));
1944   %}
1945   ins_pipe(pipe_slow);
1946 %}
1947 
1948 instruct vmulS(vReg dst_src1, vReg src2) %{
1949   predicate(UseSVE > 0);
1950   match(Set dst_src1 (MulVS dst_src1 src2));
1951   ins_cost(SVE_COST);
1952   format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (H)" %}
1953   ins_encode %{
1954     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ H,
1955          ptrue, as_FloatRegister($src2$$reg));
1956   %}
1957   ins_pipe(pipe_slow);
1958 %}
1959 
1960 instruct vmulI(vReg dst_src1, vReg src2) %{
1961   predicate(UseSVE > 0);
1962   match(Set dst_src1 (MulVI dst_src1 src2));
1963   ins_cost(SVE_COST);
1964   format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (S)" %}
1965   ins_encode %{
1966     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ S,
1967          ptrue, as_FloatRegister($src2$$reg));
1968   %}
1969   ins_pipe(pipe_slow);
1970 %}
1971 
1972 instruct vmulL(vReg dst_src1, vReg src2) %{
1973   predicate(UseSVE > 0);
1974   match(Set dst_src1 (MulVL dst_src1 src2));
1975   ins_cost(SVE_COST);
1976   format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (D)" %}
1977   ins_encode %{
1978     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ D,
1979          ptrue, as_FloatRegister($src2$$reg));
1980   %}
1981   ins_pipe(pipe_slow);
1982 %}
1983 
1984 instruct vmulF(vReg dst, vReg src1, vReg src2) %{
1985   predicate(UseSVE > 0);
1986   match(Set dst (MulVF src1 src2));
1987   ins_cost(SVE_COST);
1988   format %{ "sve_fmul $dst, $src1, $src2\t # vector (sve) (S)" %}
1989   ins_encode %{
1990     __ sve_fmul(as_FloatRegister($dst$$reg), __ S,
1991          as_FloatRegister($src1$$reg),
1992          as_FloatRegister($src2$$reg));
1993   %}
1994   ins_pipe(pipe_slow);
1995 %}
1996 
1997 instruct vmulD(vReg dst, vReg src1, vReg src2) %{
1998   predicate(UseSVE > 0);
1999   match(Set dst (MulVD src1 src2));
2000   ins_cost(SVE_COST);
2001   format %{ "sve_fmul $dst, $src1, $src2\t # vector (sve) (D)" %}
2002   ins_encode %{
2003     __ sve_fmul(as_FloatRegister($dst$$reg), __ D,
2004          as_FloatRegister($src1$$reg),
2005          as_FloatRegister($src2$$reg));
2006   %}
2007   ins_pipe(pipe_slow);
2008 %}
2009 
2010 // vector mul - predicated
2011 
2012 instruct vmulB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
2013   predicate(UseSVE > 0);
2014   match(Set dst_src1 (MulVB (Binary dst_src1 src2) pg));
2015   ins_cost(SVE_COST);
2016   format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
2017   ins_encode %{
2018     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ B,
2019             as_PRegister($pg$$reg),
2020             as_FloatRegister($src2$$reg));
2021   %}
2022   ins_pipe(pipe_slow);
2023 %}
2024 
2025 instruct vmulS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
2026   predicate(UseSVE > 0);
2027   match(Set dst_src1 (MulVS (Binary dst_src1 src2) pg));
2028   ins_cost(SVE_COST);
2029   format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
2030   ins_encode %{
2031     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ H,
2032             as_PRegister($pg$$reg),
2033             as_FloatRegister($src2$$reg));
2034   %}
2035   ins_pipe(pipe_slow);
2036 %}
2037 
2038 instruct vmulI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
2039   predicate(UseSVE > 0);
2040   match(Set dst_src1 (MulVI (Binary dst_src1 src2) pg));
2041   ins_cost(SVE_COST);
2042   format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
2043   ins_encode %{
2044     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ S,
2045             as_PRegister($pg$$reg),
2046             as_FloatRegister($src2$$reg));
2047   %}
2048   ins_pipe(pipe_slow);
2049 %}
2050 
2051 instruct vmulL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
2052   predicate(UseSVE > 0);
2053   match(Set dst_src1 (MulVL (Binary dst_src1 src2) pg));
2054   ins_cost(SVE_COST);
2055   format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
2056   ins_encode %{
2057     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ D,
2058             as_PRegister($pg$$reg),
2059             as_FloatRegister($src2$$reg));
2060   %}
2061   ins_pipe(pipe_slow);
2062 %}
2063 
2064 instruct vmulF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
2065   predicate(UseSVE > 0);
2066   match(Set dst_src1 (MulVF (Binary dst_src1 src2) pg));
2067   ins_cost(SVE_COST);
2068   format %{ "sve_fmul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
2069   ins_encode %{
2070     __ sve_fmul(as_FloatRegister($dst_src1$$reg), __ S,
2071             as_PRegister($pg$$reg),
2072             as_FloatRegister($src2$$reg));
2073   %}
2074   ins_pipe(pipe_slow);
2075 %}
2076 
2077 instruct vmulD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
2078   predicate(UseSVE > 0);
2079   match(Set dst_src1 (MulVD (Binary dst_src1 src2) pg));
2080   ins_cost(SVE_COST);
2081   format %{ "sve_fmul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
2082   ins_encode %{
2083     __ sve_fmul(as_FloatRegister($dst_src1$$reg), __ D,
2084             as_PRegister($pg$$reg),
2085             as_FloatRegister($src2$$reg));
2086   %}
2087   ins_pipe(pipe_slow);
2088 %}
2089 
2090 // vector neg
2091 
2092 instruct vnegI(vReg dst, vReg src) %{
2093   predicate(UseSVE > 0 &&
2094             !n->as_Vector()->is_predicated_vector());
2095   match(Set dst (NegVI src));
2096   ins_cost(SVE_COST);
2097   format %{ "sve_neg $dst, $src\t# vector (sve) (B/H/S)" %}
2098   ins_encode %{
2099     BasicType bt = Matcher::vector_element_basic_type(this);
2100     __ sve_neg(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
2101          ptrue, as_FloatRegister($src$$reg));
2102   %}
2103   ins_pipe(pipe_slow);
2104 %}
2105 
2106 instruct vnegL(vReg dst, vReg src) %{
2107   predicate(UseSVE > 0 &&
2108             !n->as_Vector()->is_predicated_vector());
2109   match(Set dst (NegVL src));
2110   ins_cost(SVE_COST);
2111   format %{ "sve_neg $dst, $src\t# vector (sve) (D)" %}
2112   ins_encode %{
2113     __ sve_neg(as_FloatRegister($dst$$reg), __ D,
2114          ptrue, as_FloatRegister($src$$reg));
2115   %}
2116   ins_pipe(pipe_slow);
2117 %}
2118 
2119 instruct vnegF(vReg dst, vReg src) %{
2120   predicate(UseSVE > 0 &&
2121             !n->as_Vector()->is_predicated_vector());
2122   match(Set dst (NegVF src));
2123   ins_cost(SVE_COST);
2124   format %{ "sve_fneg $dst, $src\t# vector (sve) (S)" %}
2125   ins_encode %{
2126     __ sve_fneg(as_FloatRegister($dst$$reg), __ S,
2127          ptrue, as_FloatRegister($src$$reg));
2128   %}
2129   ins_pipe(pipe_slow);
2130 %}
2131 
2132 instruct vnegD(vReg dst, vReg src) %{
2133   predicate(UseSVE > 0 &&
2134             !n->as_Vector()->is_predicated_vector());
2135   match(Set dst (NegVD src));
2136   ins_cost(SVE_COST);
2137   format %{ "sve_fneg $dst, $src\t# vector (sve) (D)" %}
2138   ins_encode %{
2139     __ sve_fneg(as_FloatRegister($dst$$reg), __ D,
2140          ptrue, as_FloatRegister($src$$reg));
2141   %}
2142   ins_pipe(pipe_slow);
2143 %}
2144 
2145 // vector neg - predicated
2146 
2147 instruct vnegI_masked(vReg dst_src, pRegGov pg) %{
2148   predicate(UseSVE > 0);
2149   match(Set dst_src (NegVI dst_src pg));
2150   ins_cost(SVE_COST);
2151   format %{ "sve_neg $dst_src, $pg, $dst_src\t# vector (sve) (B/H/S)" %}
2152   ins_encode %{
2153     BasicType bt = Matcher::vector_element_basic_type(this);
2154     __ sve_neg(as_FloatRegister($dst_src$$reg), __ elemType_to_regVariant(bt),
2155             as_PRegister($pg$$reg),
2156             as_FloatRegister($dst_src$$reg));
2157   %}
2158   ins_pipe(pipe_slow);
2159 %}
2160 
2161 instruct vnegL_masked(vReg dst_src, pRegGov pg) %{
2162   predicate(UseSVE > 0);
2163   match(Set dst_src (NegVL dst_src pg));
2164   ins_cost(SVE_COST);
2165   format %{ "sve_neg $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
2166   ins_encode %{
2167     __ sve_neg(as_FloatRegister($dst_src$$reg), __ D,
2168             as_PRegister($pg$$reg),
2169             as_FloatRegister($dst_src$$reg));
2170   %}
2171   ins_pipe(pipe_slow);
2172 %}
2173 
2174 instruct vnegF_masked(vReg dst_src, pRegGov pg) %{
2175   predicate(UseSVE > 0);
2176   match(Set dst_src (NegVF dst_src pg));
2177   ins_cost(SVE_COST);
2178   format %{ "sve_fneg $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
2179   ins_encode %{
2180     __ sve_fneg(as_FloatRegister($dst_src$$reg), __ S,
2181             as_PRegister($pg$$reg),
2182             as_FloatRegister($dst_src$$reg));
2183   %}
2184   ins_pipe(pipe_slow);
2185 %}
2186 
2187 instruct vnegD_masked(vReg dst_src, pRegGov pg) %{
2188   predicate(UseSVE > 0);
2189   match(Set dst_src (NegVD dst_src pg));
2190   ins_cost(SVE_COST);
2191   format %{ "sve_fneg $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
2192   ins_encode %{
2193     __ sve_fneg(as_FloatRegister($dst_src$$reg), __ D,
2194             as_PRegister($pg$$reg),
2195             as_FloatRegister($dst_src$$reg));
2196   %}
2197   ins_pipe(pipe_slow);
2198 %}
2199 
2200 // popcount vector
2201 
2202 instruct vpopcountI(vReg dst, vReg src) %{
2203   predicate(UseSVE > 0);
2204   match(Set dst (PopCountVI src));
2205   format %{ "sve_cnt $dst, $src\t# vector (sve) (S)\n\t" %}
2206   ins_encode %{
2207      __ sve_cnt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg));
2208   %}
2209   ins_pipe(pipe_slow);
2210 %}
2211 
2212 // vector blend
2213 
2214 instruct vblend(vReg dst, vReg src1, vReg src2, pRegGov pg) %{
2215   predicate(UseSVE > 0);
2216   match(Set dst (VectorBlend (Binary src1 src2) pg));
2217   ins_cost(SVE_COST);
2218   format %{ "sve_sel $dst, $pg, $src2, $src1\t# vector blend (sve)" %}
2219   ins_encode %{
2220     Assembler::SIMD_RegVariant size =
2221                __ elemType_to_regVariant(Matcher::vector_element_basic_type(this));
2222     __ sve_sel(as_FloatRegister($dst$$reg), size, as_PRegister($pg$$reg),
2223                as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
2224   %}
2225   ins_pipe(pipe_slow);
2226 %}
2227 
2228 // vector store mask
2229 
2230 instruct vstoremaskB(vReg dst, pRegGov src, immI_1 size) %{
2231   predicate(UseSVE > 0);
2232   match(Set dst (VectorStoreMask src size));
2233   ins_cost(SVE_COST);
2234   format %{ "vstoremask $dst, $src\t# vector store mask (sve) (B)" %}
2235   ins_encode %{
2236     __ sve_cpy(as_FloatRegister($dst$$reg), __ B, as_PRegister($src$$reg), 1, false);
2237   %}
2238   ins_pipe(pipe_slow);
2239 %}
2240 
2241 instruct vstoremask_narrow(vReg dst, pRegGov src, vReg tmp, immI_gt_1 size) %{
2242   predicate(UseSVE > 0);
2243   match(Set dst (VectorStoreMask src size));
2244   effect(TEMP_DEF dst, TEMP tmp);
2245   ins_cost(3 * SVE_COST);
2246   format %{ "vstoremask $dst, $src\t# vector store mask (sve) (H/S/D)" %}
2247   ins_encode %{
2248     Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant((int)$size$$constant);
2249     __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($src$$reg), 1, false);
2250     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
2251                          as_FloatRegister($dst$$reg), size, as_FloatRegister($tmp$$reg));
2252   %}
2253   ins_pipe(pipe_slow);
2254 %}
2255 
2256 // Combine LoadVector+VectorLoadMask when the vector element type is not T_BYTE
2257 
2258 instruct vloadmask_loadV(pRegGov dst, indirect mem, vReg tmp, rFlagsReg cr) %{
2259   predicate(UseSVE > 0 &&
2260             n->as_Vector()->length_in_bytes() == MaxVectorSize &&
2261             type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
2262   match(Set dst (VectorLoadMask (LoadVector mem)));
2263   effect(TEMP tmp, KILL cr);
2264   ins_cost(3 * SVE_COST);
2265   format %{ "sve_ld1b $tmp, $mem\n\t"
2266             "sve_cmpne $dst, $tmp, 0\t# load vector mask (sve) (H/S/D)" %}
2267   ins_encode %{
2268     // Load mask values which are boolean type, and extend them to the
2269     // expected vector element type. Convert the vector to predicate.
2270     BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
2271     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($tmp$$reg),
2272                           ptrue, T_BOOLEAN, to_vect_bt, $mem->opcode(),
2273                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2274     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ elemType_to_regVariant(to_vect_bt),
2275                ptrue, as_FloatRegister($tmp$$reg), 0);
2276   %}
2277   ins_pipe(pipe_slow);
2278 %}
2279 
2280 instruct vloadmask_loadV_partial(pRegGov dst, indirect mem, vReg vtmp, pRegGov ptmp, rFlagsReg cr) %{
2281   predicate(UseSVE > 0 &&
2282             n->as_Vector()->length_in_bytes() > 16 &&
2283             n->as_Vector()->length_in_bytes() < MaxVectorSize &&
2284             type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
2285   match(Set dst (VectorLoadMask (LoadVector mem)));
2286   effect(TEMP vtmp, TEMP ptmp, KILL cr);
2287   ins_cost(6 * SVE_COST);
2288   format %{ "vloadmask_loadV $dst, $mem\t# load vector mask partial (sve) (H/S/D)" %}
2289   ins_encode %{
2290     // Load valid mask values which are boolean type, and extend them to the
2291     // expected vector element type. Convert the vector to predicate.
2292     BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
2293     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(to_vect_bt);
2294     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this));
2295     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($vtmp$$reg),
2296                           as_PRegister($ptmp$$reg), T_BOOLEAN, to_vect_bt, $mem->opcode(),
2297                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2298     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($vtmp$$reg), 0);
2299   %}
2300   ins_pipe(pipe_slow);
2301 %}
2302 
2303 // Combine VectorStoreMask+StoreVector when the vector element type is not T_BYTE
2304 
2305 instruct storeV_vstoremask(indirect mem, pRegGov src, vReg tmp, immI_gt_1 esize) %{
2306   predicate(UseSVE > 0 &&
2307             Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
2308   match(Set mem (StoreVector mem (VectorStoreMask src esize)));
2309   effect(TEMP tmp);
2310   ins_cost(3 * SVE_COST);
2311   format %{ "sve_cpy $tmp, $src, 1\n\t"
2312             "sve_st1b $tmp, $mem\t# store vector mask (sve) (H/S/D)" %}
2313   ins_encode %{
2314     BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
2315     assert(type2aelembytes(from_vect_bt) == (int)$esize$$constant, "unsupported type.");
2316     Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
2317     __ sve_cpy(as_FloatRegister($tmp$$reg), size, as_PRegister($src$$reg), 1, false);
2318     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($tmp$$reg),
2319                           ptrue, T_BOOLEAN, from_vect_bt, $mem->opcode(),
2320                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2321   %}
2322   ins_pipe(pipe_slow);
2323 %}
2324 
2325 instruct storeV_vstoremask_partial(indirect mem, pRegGov src, vReg vtmp,
2326                                    immI_gt_1 esize, pRegGov ptmp, rFlagsReg cr) %{
2327   predicate(UseSVE > 0 &&
2328             n->as_StoreVector()->memory_size() > 16 &&
2329             type2aelembytes(n->as_StoreVector()->vect_type()->element_basic_type()) > 1 &&
2330             Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
2331   match(Set mem (StoreVector mem (VectorStoreMask src esize)));
2332   effect(TEMP vtmp, TEMP ptmp, KILL cr);
2333   format %{ "storeV_vstoremask $src, $mem\t# store vector mask partial (sve) (H/S/D)" %}
2334   ins_cost(6 * SVE_COST);
2335   ins_encode %{
2336     // Convert the valid src predicate to vector, and store the vector
2337     // elements as boolean values.
2338     BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
2339     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(from_vect_bt);
2340     __ sve_cpy(as_FloatRegister($vtmp$$reg), size, as_PRegister($src$$reg), 1, false);
2341     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
2342     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($vtmp$$reg),
2343                           as_PRegister($ptmp$$reg), T_BOOLEAN, from_vect_bt, $mem->opcode(),
2344                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2345   %}
2346   ins_pipe(pipe_slow);
2347 %}
2348 
2349 // vector add reduction
2350 
2351 instruct reduce_addI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
2352   predicate(UseSVE > 0 &&
2353             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2354   match(Set dst (AddReductionVI src1 src2));
2355   effect(TEMP_DEF dst, TEMP tmp);
2356   ins_cost(SVE_COST);
2357   format %{ "sve_reduce_addI $dst, $src1, $src2\t# addI reduction (sve) (may extend)" %}
2358   ins_encode %{
2359     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2360     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2361                            $src1$$Register, as_FloatRegister($src2$$reg),
2362                            ptrue, as_FloatRegister($tmp$$reg));
2363   %}
2364   ins_pipe(pipe_slow);
2365 %}
2366 
2367 instruct reduce_addL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
2368   predicate(UseSVE > 0 &&
2369             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2370   match(Set dst (AddReductionVL src1 src2));
2371   effect(TEMP_DEF dst, TEMP tmp);
2372   ins_cost(SVE_COST);
2373   format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction (sve)" %}
2374   ins_encode %{
2375     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2376                            $src1$$Register, as_FloatRegister($src2$$reg),
2377                            ptrue, as_FloatRegister($tmp$$reg));
2378   %}
2379   ins_pipe(pipe_slow);
2380 %}
2381 
2382 instruct reduce_addF(vRegF src1_dst, vReg src2) %{
2383   predicate(UseSVE > 0 &&
2384             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2385   match(Set src1_dst (AddReductionVF src1_dst src2));
2386   ins_cost(SVE_COST);
2387   format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (S)" %}
2388   ins_encode %{
2389     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
2390          ptrue, as_FloatRegister($src2$$reg));
2391   %}
2392   ins_pipe(pipe_slow);
2393 %}
2394 
2395 instruct reduce_addD(vRegD src1_dst, vReg src2) %{
2396   predicate(UseSVE > 0 &&
2397             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2398   match(Set src1_dst (AddReductionVD src1_dst src2));
2399   ins_cost(SVE_COST);
2400   format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (D)" %}
2401   ins_encode %{
2402     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
2403          ptrue, as_FloatRegister($src2$$reg));
2404   %}
2405   ins_pipe(pipe_slow);
2406 %}
2407 
2408 instruct reduce_addI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2409                              pRegGov ptmp, rFlagsReg cr) %{
2410   predicate(UseSVE > 0 &&
2411             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2412   match(Set dst (AddReductionVI src1 src2));
2413   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2414   ins_cost(2 * SVE_COST);
2415   format %{ "sve_reduce_addI $dst, $src1, $src2\t# addI reduction partial (sve) (may extend)" %}
2416   ins_encode %{
2417     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2418     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2419     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
2420     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2421                            $src1$$Register, as_FloatRegister($src2$$reg),
2422                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2423   %}
2424   ins_pipe(pipe_slow);
2425 %}
2426 
2427 instruct reduce_addL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2428                              pRegGov ptmp, rFlagsReg cr) %{
2429   predicate(UseSVE > 0 &&
2430             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2431   match(Set dst (AddReductionVL src1 src2));
2432   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2433   ins_cost(2 * SVE_COST);
2434   format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction partial (sve)" %}
2435   ins_encode %{
2436     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
2437     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2438                            $src1$$Register, as_FloatRegister($src2$$reg),
2439                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2440   %}
2441   ins_pipe(pipe_slow);
2442 %}
2443 
2444 instruct reduce_addF_partial(vRegF src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
2445   predicate(UseSVE > 0 &&
2446             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2447   match(Set src1_dst (AddReductionVF src1_dst src2));
2448   ins_cost(SVE_COST);
2449   effect(TEMP ptmp, KILL cr);
2450   format %{ "sve_reduce_addF $src1_dst, $src1_dst, $src2\t# addF reduction partial (sve) (S)" %}
2451   ins_encode %{
2452     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src2));
2453     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
2454                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2455   %}
2456   ins_pipe(pipe_slow);
2457 %}
2458 
2459 instruct reduce_addD_partial(vRegD src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
2460   predicate(UseSVE > 0 &&
2461             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2462   match(Set src1_dst (AddReductionVD src1_dst src2));
2463   ins_cost(SVE_COST);
2464   effect(TEMP ptmp, KILL cr);
2465   format %{ "sve_reduce_addD $src1_dst, $src1_dst, $src2\t# addD reduction partial (sve) (D)" %}
2466   ins_encode %{
2467     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
2468     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
2469                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2470   %}
2471   ins_pipe(pipe_slow);
2472 %}
2473 
2474 // vector add reduction - predicated
2475 
2476 instruct reduce_addI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2477   predicate(UseSVE > 0);
2478   match(Set dst (AddReductionVI (Binary src1 src2) pg));
2479   effect(TEMP_DEF dst, TEMP tmp);
2480   ins_cost(SVE_COST);
2481   format %{ "sve_reduce_addI $dst, $src1, $pg, $src2\t# addI reduction predicated (sve) (may extend)" %}
2482   ins_encode %{
2483     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2484     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2485                            $src1$$Register, as_FloatRegister($src2$$reg),
2486                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2487   %}
2488   ins_pipe(pipe_slow);
2489 %}
2490 
2491 instruct reduce_addL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2492   predicate(UseSVE > 0);
2493   match(Set dst (AddReductionVL (Binary src1 src2) pg));
2494   effect(TEMP_DEF dst, TEMP tmp);
2495   ins_cost(SVE_COST);
2496   format %{ "sve_reduce_addL $dst, $src1, $pg, $src2\t# addL reduction predicated (sve)" %}
2497   ins_encode %{
2498     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2499                            $src1$$Register, as_FloatRegister($src2$$reg),
2500                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2501   %}
2502   ins_pipe(pipe_slow);
2503 %}
2504 
2505 instruct reduce_addF_masked(vRegF src1_dst, vReg src2, pRegGov pg) %{
2506   predicate(UseSVE > 0);
2507   match(Set src1_dst (AddReductionVF (Binary src1_dst src2) pg));
2508   ins_cost(SVE_COST);
2509   format %{ "sve_reduce_addF $src1_dst, $pg, $src2\t# addF reduction predicated (sve)" %}
2510   ins_encode %{
2511     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
2512                  as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
2513   %}
2514   ins_pipe(pipe_slow);
2515 %}
2516 
2517 instruct reduce_addD_masked(vRegD src1_dst, vReg src2, pRegGov pg) %{
2518   predicate(UseSVE > 0);
2519   match(Set src1_dst (AddReductionVD (Binary src1_dst src2) pg));
2520   ins_cost(SVE_COST);
2521   format %{ "sve_reduce_addD $src1_dst, $pg, $src2\t# addD reduction predicated (sve)" %}
2522   ins_encode %{
2523     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
2524                  as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
2525   %}
2526   ins_pipe(pipe_slow);
2527 %}
2528 
2529 // vector and reduction
2530 
2531 instruct reduce_andI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
2532   predicate(UseSVE > 0 &&
2533             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2534             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2535   match(Set dst (AndReductionV src1 src2));
2536   effect(TEMP_DEF dst, TEMP tmp);
2537   ins_cost(SVE_COST);
2538   format %{ "sve_reduce_andI $dst, $src1, $src2\t# andI reduction (sve) (may extend)" %}
2539   ins_encode %{
2540     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2541     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2542                            $src1$$Register, as_FloatRegister($src2$$reg),
2543                            ptrue, as_FloatRegister($tmp$$reg));
2544   %}
2545   ins_pipe(pipe_slow);
2546 %}
2547 
2548 instruct reduce_andL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
2549   predicate(UseSVE > 0 &&
2550             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2551             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2552   match(Set dst (AndReductionV src1 src2));
2553   effect(TEMP_DEF dst, TEMP tmp);
2554   ins_cost(SVE_COST);
2555   format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction (sve)" %}
2556   ins_encode %{
2557     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2558                            $src1$$Register, as_FloatRegister($src2$$reg),
2559                            ptrue, as_FloatRegister($tmp$$reg));
2560   %}
2561   ins_pipe(pipe_slow);
2562 %}
2563 
2564 instruct reduce_andI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2565                              pRegGov ptmp, rFlagsReg cr) %{
2566   predicate(UseSVE > 0 &&
2567             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2568             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2569   match(Set dst (AndReductionV src1 src2));
2570   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2571   ins_cost(2 * SVE_COST);
2572   format %{ "sve_reduce_andI $dst, $src1, $src2\t# andI reduction partial (sve) (may extend)" %}
2573   ins_encode %{
2574     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2575     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2576     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
2577     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2578                            $src1$$Register, as_FloatRegister($src2$$reg),
2579                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2580   %}
2581   ins_pipe(pipe_slow);
2582 %}
2583 
2584 instruct reduce_andL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2585                              pRegGov ptmp, rFlagsReg cr) %{
2586   predicate(UseSVE > 0 &&
2587             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2588             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2589   match(Set dst (AndReductionV src1 src2));
2590   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2591   ins_cost(2 * SVE_COST);
2592   format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction partial (sve)" %}
2593   ins_encode %{
2594     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
2595     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2596                            $src1$$Register, as_FloatRegister($src2$$reg),
2597                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2598   %}
2599   ins_pipe(pipe_slow);
2600 %}
2601 
2602 // vector and reduction - predicated
2603 
2604 instruct reduce_andI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2605   predicate(UseSVE > 0 &&
2606             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG);
2607   match(Set dst (AndReductionV (Binary src1 src2) pg));
2608   effect(TEMP_DEF dst, TEMP tmp);
2609   ins_cost(SVE_COST);
2610   format %{ "sve_reduce_andI $dst, $src1, $pg, $src2\t# andI reduction predicated (sve) (may extend)" %}
2611   ins_encode %{
2612     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2613     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2614                            $src1$$Register, as_FloatRegister($src2$$reg),
2615                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2616   %}
2617   ins_pipe(pipe_slow);
2618 %}
2619 
2620 instruct reduce_andL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2621   predicate(UseSVE > 0 &&
2622             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2623   match(Set dst (AndReductionV (Binary src1 src2) pg));
2624   effect(TEMP_DEF dst, TEMP tmp);
2625   ins_cost(SVE_COST);
2626   format %{ "sve_reduce_andL $dst, $src1, $pg, $src2\t# andL reduction predicated (sve)" %}
2627   ins_encode %{
2628     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2629                            $src1$$Register, as_FloatRegister($src2$$reg),
2630                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2631   %}
2632   ins_pipe(pipe_slow);
2633 %}
2634 
2635 // vector or reduction
2636 
2637 instruct reduce_orI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
2638   predicate(UseSVE > 0 &&
2639             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2640             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2641   match(Set dst (OrReductionV src1 src2));
2642   effect(TEMP_DEF dst, TEMP tmp);
2643   ins_cost(SVE_COST);
2644   format %{ "sve_reduce_orI $dst, $src1, $src2\t# orI reduction (sve) (may extend)" %}
2645   ins_encode %{
2646     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2647     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2648                            $src1$$Register, as_FloatRegister($src2$$reg),
2649                            ptrue, as_FloatRegister($tmp$$reg));
2650   %}
2651   ins_pipe(pipe_slow);
2652 %}
2653 
2654 instruct reduce_orL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
2655   predicate(UseSVE > 0 &&
2656             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2657             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2658   match(Set dst (OrReductionV src1 src2));
2659   effect(TEMP_DEF dst, TEMP tmp);
2660   ins_cost(SVE_COST);
2661   format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction (sve)" %}
2662   ins_encode %{
2663     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2664                            $src1$$Register, as_FloatRegister($src2$$reg),
2665                            ptrue, as_FloatRegister($tmp$$reg));
2666   %}
2667   ins_pipe(pipe_slow);
2668 %}
2669 
2670 instruct reduce_orI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2671                              pRegGov ptmp, rFlagsReg cr) %{
2672   predicate(UseSVE > 0 &&
2673             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2674             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2675   match(Set dst (OrReductionV src1 src2));
2676   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2677   ins_cost(2 * SVE_COST);
2678   format %{ "sve_reduce_orI $dst, $src1, $src2\t# orI reduction partial (sve) (may extend)" %}
2679   ins_encode %{
2680     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2681     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2682     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
2683     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2684                            $src1$$Register, as_FloatRegister($src2$$reg),
2685                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2686   %}
2687   ins_pipe(pipe_slow);
2688 %}
2689 
2690 instruct reduce_orL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2691                              pRegGov ptmp, rFlagsReg cr) %{
2692   predicate(UseSVE > 0 &&
2693             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2694             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2695   match(Set dst (OrReductionV src1 src2));
2696   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2697   ins_cost(2 * SVE_COST);
2698   format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction partial (sve)" %}
2699   ins_encode %{
2700     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
2701     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2702                            $src1$$Register, as_FloatRegister($src2$$reg),
2703                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2704   %}
2705   ins_pipe(pipe_slow);
2706 %}
2707 
2708 // vector or reduction - predicated
2709 
2710 instruct reduce_orI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2711   predicate(UseSVE > 0 &&
2712             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG);
2713   match(Set dst (OrReductionV (Binary src1 src2) pg));
2714   effect(TEMP_DEF dst, TEMP tmp);
2715   ins_cost(SVE_COST);
2716   format %{ "sve_reduce_orI $dst, $src1, $pg, $src2\t# orI reduction predicated (sve) (may extend)" %}
2717   ins_encode %{
2718     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2719     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2720                            $src1$$Register, as_FloatRegister($src2$$reg),
2721                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2722   %}
2723   ins_pipe(pipe_slow);
2724 %}
2725 
2726 instruct reduce_orL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2727   predicate(UseSVE > 0 &&
2728             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2729   match(Set dst (OrReductionV (Binary src1 src2) pg));
2730   effect(TEMP_DEF dst, TEMP tmp);
2731   ins_cost(SVE_COST);
2732   format %{ "sve_reduce_orL $dst, $src1, $pg, $src2\t# orL reduction predicated (sve)" %}
2733   ins_encode %{
2734     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2735                            $src1$$Register, as_FloatRegister($src2$$reg),
2736                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2737   %}
2738   ins_pipe(pipe_slow);
2739 %}
2740 
2741 // vector xor reduction
2742 
2743 instruct reduce_eorI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
2744   predicate(UseSVE > 0 &&
2745             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2746             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2747   match(Set dst (XorReductionV src1 src2));
2748   effect(TEMP_DEF dst, TEMP tmp);
2749   ins_cost(SVE_COST);
2750   format %{ "sve_reduce_eorI $dst, $src1, $src2\t# eorI reduction (sve) (may extend)" %}
2751   ins_encode %{
2752     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2753     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2754                            $src1$$Register, as_FloatRegister($src2$$reg),
2755                            ptrue, as_FloatRegister($tmp$$reg));
2756   %}
2757   ins_pipe(pipe_slow);
2758 %}
2759 
2760 instruct reduce_eorL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
2761   predicate(UseSVE > 0 &&
2762             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2763             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2764   match(Set dst (XorReductionV src1 src2));
2765   effect(TEMP_DEF dst, TEMP tmp);
2766   ins_cost(SVE_COST);
2767   format %{ "sve_reduce_eorL $dst, $src1, $src2\t# eorL reduction (sve)" %}
2768   ins_encode %{
2769     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2770                            $src1$$Register, as_FloatRegister($src2$$reg),
2771                            ptrue, as_FloatRegister($tmp$$reg));
2772   %}
2773   ins_pipe(pipe_slow);
2774 %}
2775 
2776 instruct reduce_eorI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2777                              pRegGov ptmp, rFlagsReg cr) %{
2778   predicate(UseSVE > 0 &&
2779             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2780             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2781   match(Set dst (XorReductionV src1 src2));
2782   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2783   ins_cost(2 * SVE_COST);
2784   format %{ "sve_reduce_eorI $dst, $src1, $src2\t# eorI reduction partial (sve) (may extend)" %}
2785   ins_encode %{
2786     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2787     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2788     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
2789     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2790                            $src1$$Register, as_FloatRegister($src2$$reg),
2791                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2792   %}
2793   ins_pipe(pipe_slow);
2794 %}
2795 
2796 instruct reduce_eorL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2797                              pRegGov ptmp, rFlagsReg cr) %{
2798   predicate(UseSVE > 0 &&
2799             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2800             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2801   match(Set dst (XorReductionV src1 src2));
2802   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2803   ins_cost(2 * SVE_COST);
2804   format %{ "sve_reduce_eorL $dst, $src1, $src2\t# eorL reduction partial (sve)" %}
2805   ins_encode %{
2806     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
2807     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2808                            $src1$$Register, as_FloatRegister($src2$$reg),
2809                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2810   %}
2811   ins_pipe(pipe_slow);
2812 %}
2813 
2814 // vector xor reduction - predicated
2815 
2816 instruct reduce_eorI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2817   predicate(UseSVE > 0 &&
2818             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG);
2819   match(Set dst (XorReductionV (Binary src1 src2) pg));
2820   effect(TEMP_DEF dst, TEMP tmp);
2821   ins_cost(SVE_COST);
2822   format %{ "sve_reduce_eorI $dst, $src1, $pg, $src2\t# eorI reduction predicated (sve) (may extend)" %}
2823   ins_encode %{
2824     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2825     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2826                            $src1$$Register, as_FloatRegister($src2$$reg),
2827                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2828   %}
2829   ins_pipe(pipe_slow);
2830 %}
2831 
2832 instruct reduce_eorL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2833   predicate(UseSVE > 0 &&
2834             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2835   match(Set dst (XorReductionV (Binary src1 src2) pg));
2836   effect(TEMP_DEF dst, TEMP tmp);
2837   ins_cost(SVE_COST);
2838   format %{ "sve_reduce_eorL $dst, $src1, $pg, $src2\t# eorL reduction predicated (sve)" %}
2839   ins_encode %{
2840     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2841                            $src1$$Register, as_FloatRegister($src2$$reg),
2842                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2843   %}
2844   ins_pipe(pipe_slow);
2845 %}
2846 
2847 // vector max reduction
2848 
2849 instruct reduce_maxI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
2850   predicate(UseSVE > 0 &&
2851             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2852             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2853             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
2854   match(Set dst (MaxReductionV src1 src2));
2855   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2856   ins_cost(SVE_COST);
2857   format %{ "sve_reduce_maxI $dst, $src1, $src2\t# maxI reduction (sve)" %}
2858   ins_encode %{
2859     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2860     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2861                            $src1$$Register, as_FloatRegister($src2$$reg),
2862                            ptrue, as_FloatRegister($tmp$$reg));
2863   %}
2864   ins_pipe(pipe_slow);
2865 %}
2866 
2867 instruct reduce_maxL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
2868   predicate(UseSVE > 0 &&
2869             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2870             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2871   match(Set dst (MaxReductionV src1 src2));
2872   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2873   ins_cost(SVE_COST);
2874   format %{ "sve_reduce_maxL $dst, $src1, $src2\t# maxL reduction (sve)" %}
2875   ins_encode %{
2876     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2877                            $src1$$Register, as_FloatRegister($src2$$reg),
2878                            ptrue, as_FloatRegister($tmp$$reg));
2879   %}
2880   ins_pipe(pipe_slow);
2881 %}
2882 
2883 instruct reduce_maxI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2884                              pRegGov ptmp, rFlagsReg cr) %{
2885   predicate(UseSVE > 0 &&
2886             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2887             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2888             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
2889   match(Set dst (MaxReductionV src1 src2));
2890   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2891   ins_cost(2 * SVE_COST);
2892   format %{ "sve_reduce_maxI $dst, $src1, $src2\t# maxI reduction partial (sve)" %}
2893   ins_encode %{
2894     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2895     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2896     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
2897     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2898                            $src1$$Register, as_FloatRegister($src2$$reg),
2899                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2900   %}
2901   ins_pipe(pipe_slow);
2902 %}
2903 
2904 instruct reduce_maxL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2905                              pRegGov ptmp, rFlagsReg cr) %{
2906   predicate(UseSVE > 0 &&
2907             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2908             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2909   match(Set dst (MaxReductionV src1 src2));
2910   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2911   ins_cost(2 * SVE_COST);
2912   format %{ "sve_reduce_maxL $dst, $src1, $src2\t# maxL reduction  partial (sve)" %}
2913   ins_encode %{
2914     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
2915     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2916                            $src1$$Register, as_FloatRegister($src2$$reg),
2917                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2918   %}
2919   ins_pipe(pipe_slow);
2920 %}
2921 
2922 instruct reduce_maxF(vRegF dst, vRegF src1, vReg src2) %{
2923   predicate(UseSVE > 0 &&
2924             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
2925             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2926   match(Set dst (MaxReductionV src1 src2));
2927   ins_cost(INSN_COST);
2928   effect(TEMP_DEF dst);
2929   format %{ "sve_reduce_maxF $dst, $src1, $src2\t# maxF reduction (sve)" %}
2930   ins_encode %{
2931     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src2$$reg));
2932     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2933   %}
2934   ins_pipe(pipe_slow);
2935 %}
2936 
2937 instruct reduce_maxF_partial(vRegF dst, vRegF src1, vReg src2,
2938                              pRegGov ptmp, rFlagsReg cr) %{
2939   predicate(UseSVE > 0 &&
2940             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
2941             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2942   match(Set dst (MaxReductionV src1 src2));
2943   ins_cost(INSN_COST);
2944   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
2945   format %{ "sve_reduce_maxF $dst, $src1, $src2\t# maxF reduction partial (sve)" %}
2946   ins_encode %{
2947     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src2));
2948     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2949     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2950   %}
2951   ins_pipe(pipe_slow);
2952 %}
2953 
2954 instruct reduce_maxD(vRegD dst, vRegD src1, vReg src2) %{
2955   predicate(UseSVE > 0 &&
2956             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
2957             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2958   match(Set dst (MaxReductionV src1 src2));
2959   ins_cost(INSN_COST);
2960   effect(TEMP_DEF dst);
2961   format %{ "sve_reduce_maxD $dst, $src1, $src2\t# maxD reduction (sve)" %}
2962   ins_encode %{
2963     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
2964     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2965   %}
2966   ins_pipe(pipe_slow);
2967 %}
2968 
2969 instruct reduce_maxD_partial(vRegD dst, vRegD src1, vReg src2,
2970                              pRegGov ptmp, rFlagsReg cr) %{
2971   predicate(UseSVE > 0 &&
2972             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
2973             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2974   match(Set dst (MaxReductionV src1 src2));
2975   ins_cost(INSN_COST);
2976   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
2977   format %{ "sve_reduce_maxD $dst, $src1, $src2\t# maxD reduction partial (sve)" %}
2978   ins_encode %{
2979     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
2980     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2981     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2982   %}
2983   ins_pipe(pipe_slow);
2984 %}
2985 
2986 // vector max reduction - predicated
2987 
2988 instruct reduce_maxI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp,
2989                            pRegGov pg, rFlagsReg cr) %{
2990   predicate(UseSVE > 0 &&
2991             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2992             is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
2993   match(Set dst (MaxReductionV (Binary src1 src2) pg));
2994   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2995   ins_cost(SVE_COST);
2996   format %{ "sve_reduce_maxI $dst, $src1, $pg, $src2\t# maxI reduction predicated (sve)" %}
2997   ins_encode %{
2998     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2999     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
3000                            $src1$$Register, as_FloatRegister($src2$$reg),
3001                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
3002   %}
3003   ins_pipe(pipe_slow);
3004 %}
3005 
3006 instruct reduce_maxL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp,
3007                           pRegGov pg, rFlagsReg cr) %{
3008   predicate(UseSVE > 0 &&
3009             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3010   match(Set dst (MaxReductionV (Binary src1 src2) pg));
3011   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3012   ins_cost(SVE_COST);
3013   format %{ "sve_reduce_maxL $dst, $src1, $pg, $src2\t# maxL reduction predicated (sve)" %}
3014   ins_encode %{
3015     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
3016                            $src1$$Register, as_FloatRegister($src2$$reg),
3017                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
3018   %}
3019   ins_pipe(pipe_slow);
3020 %}
3021 
3022 instruct reduce_maxF_masked(vRegF dst, vRegF src1, vReg src2, pRegGov pg) %{
3023   predicate(UseSVE > 0 &&
3024             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
3025   match(Set dst (MaxReductionV (Binary src1 src2) pg));
3026   ins_cost(SVE_COST);
3027   effect(TEMP_DEF dst);
3028   format %{ "sve_reduce_maxF $dst, $src1, $pg, $src2\t# maxF reduction predicated (sve)" %}
3029   ins_encode %{
3030     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
3031     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3032   %}
3033   ins_pipe(pipe_slow);
3034 %}
3035 
3036 instruct reduce_maxD_masked(vRegD dst, vRegD src1, vReg src2, pRegGov pg) %{
3037   predicate(UseSVE > 0 &&
3038             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
3039   match(Set dst (MaxReductionV (Binary src1 src2) pg));
3040   ins_cost(SVE_COST);
3041   effect(TEMP_DEF dst);
3042   format %{ "sve_reduce_maxD $dst, $src1, $pg, $src2\t# maxD reduction predicated (sve)" %}
3043   ins_encode %{
3044     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
3045     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3046   %}
3047   ins_pipe(pipe_slow);
3048 %}
3049 
3050 // vector min reduction
3051 
3052 instruct reduce_minI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
3053   predicate(UseSVE > 0 &&
3054             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
3055             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
3056             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
3057   match(Set dst (MinReductionV src1 src2));
3058   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3059   ins_cost(SVE_COST);
3060   format %{ "sve_reduce_minI $dst, $src1, $src2\t# minI reduction (sve)" %}
3061   ins_encode %{
3062     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
3063     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
3064                            $src1$$Register, as_FloatRegister($src2$$reg),
3065                            ptrue, as_FloatRegister($tmp$$reg));
3066   %}
3067   ins_pipe(pipe_slow);
3068 %}
3069 
3070 instruct reduce_minL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
3071   predicate(UseSVE > 0 &&
3072             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
3073             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3074   match(Set dst (MinReductionV src1 src2));
3075   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3076   ins_cost(SVE_COST);
3077   format %{ "sve_reduce_minL $dst, $src1, $src2\t# minL reduction (sve)" %}
3078   ins_encode %{
3079     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
3080                            $src1$$Register, as_FloatRegister($src2$$reg),
3081                            ptrue, as_FloatRegister($tmp$$reg));
3082   %}
3083   ins_pipe(pipe_slow);
3084 %}
3085 
3086 instruct reduce_minI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
3087                              pRegGov ptmp, rFlagsReg cr) %{
3088   predicate(UseSVE > 0 &&
3089             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
3090             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
3091             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
3092   match(Set dst (MinReductionV src1 src2));
3093   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
3094   ins_cost(2 * SVE_COST);
3095   format %{ "sve_reduce_minI $dst, $src1, $src2\t# minI reduction partial (sve)" %}
3096   ins_encode %{
3097     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
3098     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
3099     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
3100     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
3101                            $src1$$Register, as_FloatRegister($src2$$reg),
3102                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
3103   %}
3104   ins_pipe(pipe_slow);
3105 %}
3106 
3107 instruct reduce_minL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
3108                              pRegGov ptmp, rFlagsReg cr) %{
3109   predicate(UseSVE > 0 &&
3110             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
3111             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3112   match(Set dst (MinReductionV src1 src2));
3113   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
3114   ins_cost(2 * SVE_COST);
3115   format %{ "sve_reduce_minL $dst, $src1, $src2\t# minL reduction  partial (sve)" %}
3116   ins_encode %{
3117     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
3118     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
3119                            $src1$$Register, as_FloatRegister($src2$$reg),
3120                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
3121   %}
3122   ins_pipe(pipe_slow);
3123 %}
3124 
3125 instruct reduce_minF(vRegF dst, vRegF src1, vReg src2) %{
3126   predicate(UseSVE > 0 &&
3127             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3128             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3129   match(Set dst (MinReductionV src1 src2));
3130   ins_cost(INSN_COST);
3131   effect(TEMP_DEF dst);
3132   format %{ "sve_reduce_minF $dst, $src1, $src2\t# minF reduction (sve)" %}
3133   ins_encode %{
3134     __ sve_fminv(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src2$$reg));
3135     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3136   %}
3137   ins_pipe(pipe_slow);
3138 %}
3139 
3140 instruct reduce_minF_partial(vRegF dst, vRegF src1, vReg src2,
3141                              pRegGov ptmp, rFlagsReg cr) %{
3142   predicate(UseSVE > 0 &&
3143             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3144             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3145   match(Set dst (MinReductionV src1 src2));
3146   ins_cost(INSN_COST);
3147   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3148   format %{ "sve_reduce_minF $dst, $src1, $src2\t# minF reduction partial (sve)" %}
3149   ins_encode %{
3150     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src2));
3151     __ sve_fminv(as_FloatRegister($dst$$reg), __ S, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
3152     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3153   %}
3154   ins_pipe(pipe_slow);
3155 %}
3156 
3157 instruct reduce_minD(vRegD dst, vRegD src1, vReg src2) %{
3158   predicate(UseSVE > 0 &&
3159             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3160             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3161   match(Set dst (MinReductionV src1 src2));
3162   ins_cost(INSN_COST);
3163   effect(TEMP_DEF dst);
3164   format %{ "sve_reduce_minD $dst, $src1, $src2\t# minD reduction (sve)" %}
3165   ins_encode %{
3166     __ sve_fminv(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
3167     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3168   %}
3169   ins_pipe(pipe_slow);
3170 %}
3171 
3172 instruct reduce_minD_partial(vRegD dst, vRegD src1, vReg src2,
3173                              pRegGov ptmp, rFlagsReg cr) %{
3174   predicate(UseSVE > 0 &&
3175             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3176             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3177   match(Set dst (MinReductionV src1 src2));
3178   ins_cost(INSN_COST);
3179   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3180   format %{ "sve_reduce_minD $dst, $src1, $src2\t# minD reduction partial (sve)" %}
3181   ins_encode %{
3182     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
3183     __ sve_fminv(as_FloatRegister($dst$$reg), __ D, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
3184     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3185   %}
3186   ins_pipe(pipe_slow);
3187 %}
3188 
3189 // vector min reduction - predicated
3190 
3191 instruct reduce_minI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp,
3192                            pRegGov pg, rFlagsReg cr) %{
3193   predicate(UseSVE > 0 &&
3194             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
3195             is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
3196   match(Set dst (MinReductionV (Binary src1 src2) pg));
3197   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3198   ins_cost(SVE_COST);
3199   format %{ "sve_reduce_minI $dst, $src1, $pg, $src2\t# minI reduction predicated (sve)" %}
3200   ins_encode %{
3201     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
3202     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
3203                            $src1$$Register, as_FloatRegister($src2$$reg),
3204                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
3205   %}
3206   ins_pipe(pipe_slow);
3207 %}
3208 
3209 instruct reduce_minL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp,
3210                           pRegGov pg, rFlagsReg cr) %{
3211   predicate(UseSVE > 0 &&
3212             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3213   match(Set dst (MinReductionV (Binary src1 src2) pg));
3214   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3215   ins_cost(SVE_COST);
3216   format %{ "sve_reduce_minL $dst, $src1, $pg, $src2\t# minL reduction predicated (sve)" %}
3217   ins_encode %{
3218     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
3219                            $src1$$Register, as_FloatRegister($src2$$reg),
3220                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
3221   %}
3222   ins_pipe(pipe_slow);
3223 %}
3224 
3225 instruct reduce_minF_masked(vRegF dst, vRegF src1, vReg src2, pRegGov pg) %{
3226   predicate(UseSVE > 0 &&
3227             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
3228   match(Set dst (MinReductionV (Binary src1 src2) pg));
3229   ins_cost(SVE_COST);
3230   effect(TEMP_DEF dst);
3231   format %{ "sve_reduce_minF $dst, $src1, $pg, $src2\t# minF reduction predicated (sve)" %}
3232   ins_encode %{
3233     __ sve_fminv(as_FloatRegister($dst$$reg), __ S, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
3234     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3235   %}
3236   ins_pipe(pipe_slow);
3237 %}
3238 
3239 instruct reduce_minD_masked(vRegD dst, vRegD src1, vReg src2, pRegGov pg) %{
3240   predicate(UseSVE > 0 &&
3241             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
3242   match(Set dst (MinReductionV (Binary src1 src2) pg));
3243   ins_cost(SVE_COST);
3244   effect(TEMP_DEF dst);
3245   format %{ "sve_reduce_minD $dst, $src1, $pg, $src2\t# minD reduction predicated (sve)" %}
3246   ins_encode %{
3247     __ sve_fminv(as_FloatRegister($dst$$reg), __ D, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
3248     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3249   %}
3250   ins_pipe(pipe_slow);
3251 %}
3252 
3253 // vector Math.rint, floor, ceil
3254 
3255 instruct vroundD(vReg dst, vReg src, immI rmode) %{
3256   predicate(UseSVE > 0 &&
3257             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
3258   match(Set dst (RoundDoubleModeV src rmode));
3259   format %{ "sve_frint $dst, $src, $rmode\t# vector (sve) (D)" %}
3260   ins_encode %{
3261     switch ($rmode$$constant) {
3262       case RoundDoubleModeNode::rmode_rint:
3263         __ sve_frintn(as_FloatRegister($dst$$reg), __ D,
3264              ptrue, as_FloatRegister($src$$reg));
3265         break;
3266       case RoundDoubleModeNode::rmode_floor:
3267         __ sve_frintm(as_FloatRegister($dst$$reg), __ D,
3268              ptrue, as_FloatRegister($src$$reg));
3269         break;
3270       case RoundDoubleModeNode::rmode_ceil:
3271         __ sve_frintp(as_FloatRegister($dst$$reg), __ D,
3272              ptrue, as_FloatRegister($src$$reg));
3273         break;
3274     }
3275   %}
3276   ins_pipe(pipe_slow);
3277 %}
3278 
3279 instruct vroundFtoI(vReg dst, vReg src, vReg tmp1, vReg tmp2, vReg tmp3, pRegGov ptmp)
3280 %{
3281   predicate(UseSVE > 0);
3282   match(Set dst (RoundVF src));
3283   effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP ptmp);
3284   format %{ "sve_vround  $dst, S, $src\t# round F to I vector" %}
3285   ins_encode %{
3286     BasicType bt = Matcher::vector_element_basic_type(this);
3287     int vlen = Matcher::vector_length_in_bytes(this);
3288     if (vlen > 16) {
3289       __ vector_round_sve(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3290                           as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg),
3291                           as_PRegister($ptmp$$reg), __ S);
3292     } else {
3293       __ vector_round_neon(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3294                            as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg),
3295                            as_FloatRegister($tmp3$$reg),
3296                            __ esize2arrangement(type2aelembytes(bt),
3297                               /*isQ*/ vlen == 16));
3298     }
3299   %}
3300   ins_pipe(pipe_class_default);
3301 %}
3302 
3303 instruct vroundDtoL(vReg dst, vReg src, vReg tmp1, vReg tmp2, vReg tmp3, pRegGov ptmp)
3304 %{
3305   predicate(UseSVE > 0);
3306   match(Set dst (RoundVD src));
3307   effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP ptmp);
3308   format %{ "sve_vround  $dst, D, $src\t# round D to L vector" %}
3309   ins_encode %{
3310     BasicType bt = Matcher::vector_element_basic_type(this);
3311     int vlen = Matcher::vector_length_in_bytes(this);
3312     if (vlen > 16) {
3313       __ vector_round_sve(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3314                           as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg),
3315                           as_PRegister($ptmp$$reg), __ D);
3316     } else {
3317       __ vector_round_neon(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3318                            as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg),
3319                            as_FloatRegister($tmp3$$reg),
3320                            __ esize2arrangement(type2aelembytes(bt),
3321                               /*isQ*/ vlen == 16));
3322     }
3323   %}
3324   ins_pipe(pipe_class_default);
3325 %}
3326 
3327 // vector replicate
3328 
3329 instruct replicateB(vReg dst, iRegIorL2I src) %{
3330   predicate(UseSVE > 0);
3331   match(Set dst (ReplicateB src));
3332   ins_cost(SVE_COST);
3333   format %{ "sve_dup  $dst, $src\t# vector (sve) (B)" %}
3334   ins_encode %{
3335     __ sve_dup(as_FloatRegister($dst$$reg), __ B, as_Register($src$$reg));
3336   %}
3337   ins_pipe(pipe_slow);
3338 %}
3339 
3340 instruct replicateS(vReg dst, iRegIorL2I src) %{
3341   predicate(UseSVE > 0);
3342   match(Set dst (ReplicateS src));
3343   ins_cost(SVE_COST);
3344   format %{ "sve_dup  $dst, $src\t# vector (sve) (H)" %}
3345   ins_encode %{
3346     __ sve_dup(as_FloatRegister($dst$$reg), __ H, as_Register($src$$reg));
3347   %}
3348   ins_pipe(pipe_slow);
3349 %}
3350 
3351 instruct replicateI(vReg dst, iRegIorL2I src) %{
3352   predicate(UseSVE > 0);
3353   match(Set dst (ReplicateI src));
3354   ins_cost(SVE_COST);
3355   format %{ "sve_dup  $dst, $src\t# vector (sve) (S)" %}
3356   ins_encode %{
3357     __ sve_dup(as_FloatRegister($dst$$reg), __ S, as_Register($src$$reg));
3358   %}
3359   ins_pipe(pipe_slow);
3360 %}
3361 
3362 instruct replicateL(vReg dst, iRegL src) %{
3363   predicate(UseSVE > 0);
3364   match(Set dst (ReplicateL src));
3365   ins_cost(SVE_COST);
3366   format %{ "sve_dup  $dst, $src\t# vector (sve) (D)" %}
3367   ins_encode %{
3368     __ sve_dup(as_FloatRegister($dst$$reg), __ D, as_Register($src$$reg));
3369   %}
3370   ins_pipe(pipe_slow);
3371 %}
3372 
3373 instruct replicateB_imm8(vReg dst, immI8 con) %{
3374   predicate(UseSVE > 0);
3375   match(Set dst (ReplicateB con));
3376   ins_cost(SVE_COST);
3377   format %{ "sve_dup  $dst, $con\t# vector (sve) (B)" %}
3378   ins_encode %{
3379     __ sve_dup(as_FloatRegister($dst$$reg), __ B, $con$$constant);
3380   %}
3381   ins_pipe(pipe_slow);
3382 %}
3383 
3384 instruct replicateS_imm8(vReg dst, immI8_shift8 con) %{
3385   predicate(UseSVE > 0);
3386   match(Set dst (ReplicateS con));
3387   ins_cost(SVE_COST);
3388   format %{ "sve_dup  $dst, $con\t# vector (sve) (H)" %}
3389   ins_encode %{
3390     __ sve_dup(as_FloatRegister($dst$$reg), __ H, $con$$constant);
3391   %}
3392   ins_pipe(pipe_slow);
3393 %}
3394 
3395 instruct replicateI_imm8(vReg dst, immI8_shift8 con) %{
3396   predicate(UseSVE > 0);
3397   match(Set dst (ReplicateI con));
3398   ins_cost(SVE_COST);
3399   format %{ "sve_dup  $dst, $con\t# vector (sve) (S)" %}
3400   ins_encode %{
3401     __ sve_dup(as_FloatRegister($dst$$reg), __ S, $con$$constant);
3402   %}
3403   ins_pipe(pipe_slow);
3404 %}
3405 
3406 instruct replicateL_imm8(vReg dst, immL8_shift8 con) %{
3407   predicate(UseSVE > 0);
3408   match(Set dst (ReplicateL con));
3409   ins_cost(SVE_COST);
3410   format %{ "sve_dup  $dst, $con\t# vector (sve) (D)" %}
3411   ins_encode %{
3412     __ sve_dup(as_FloatRegister($dst$$reg), __ D, $con$$constant);
3413   %}
3414   ins_pipe(pipe_slow);
3415 %}
3416 
3417 instruct replicateF(vReg dst, vRegF src) %{
3418   predicate(UseSVE > 0);
3419   match(Set dst (ReplicateF src));
3420   ins_cost(SVE_COST);
3421   format %{ "sve_cpy  $dst, $src\t# vector (sve) (S)" %}
3422   ins_encode %{
3423     __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
3424          ptrue, as_FloatRegister($src$$reg));
3425   %}
3426   ins_pipe(pipe_slow);
3427 %}
3428 
3429 instruct replicateD(vReg dst, vRegD src) %{
3430   predicate(UseSVE > 0);
3431   match(Set dst (ReplicateD src));
3432   ins_cost(SVE_COST);
3433   format %{ "sve_cpy  $dst, $src\t# vector (sve) (D)" %}
3434   ins_encode %{
3435     __ sve_cpy(as_FloatRegister($dst$$reg), __ D,
3436          ptrue, as_FloatRegister($src$$reg));
3437   %}
3438   ins_pipe(pipe_slow);
3439 %}
3440 
3441 // vector shift
3442 
3443 instruct vasrB(vReg dst, vReg shift) %{
3444   predicate(UseSVE > 0);
3445   match(Set dst (RShiftVB dst shift));
3446   ins_cost(SVE_COST);
3447   format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (B)" %}
3448   ins_encode %{
3449     __ sve_asr(as_FloatRegister($dst$$reg), __ B,
3450          ptrue, as_FloatRegister($shift$$reg));
3451   %}
3452   ins_pipe(pipe_slow);
3453 %}
3454 
3455 instruct vasrS(vReg dst, vReg shift) %{
3456   predicate(UseSVE > 0);
3457   match(Set dst (RShiftVS dst shift));
3458   ins_cost(SVE_COST);
3459   format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (H)" %}
3460   ins_encode %{
3461     __ sve_asr(as_FloatRegister($dst$$reg), __ H,
3462          ptrue, as_FloatRegister($shift$$reg));
3463   %}
3464   ins_pipe(pipe_slow);
3465 %}
3466 
3467 instruct vasrI(vReg dst, vReg shift) %{
3468   predicate(UseSVE > 0);
3469   match(Set dst (RShiftVI dst shift));
3470   ins_cost(SVE_COST);
3471   format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (S)" %}
3472   ins_encode %{
3473     __ sve_asr(as_FloatRegister($dst$$reg), __ S,
3474          ptrue, as_FloatRegister($shift$$reg));
3475   %}
3476   ins_pipe(pipe_slow);
3477 %}
3478 
3479 instruct vasrL(vReg dst, vReg shift) %{
3480   predicate(UseSVE > 0);
3481   match(Set dst (RShiftVL dst shift));
3482   ins_cost(SVE_COST);
3483   format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (D)" %}
3484   ins_encode %{
3485     __ sve_asr(as_FloatRegister($dst$$reg), __ D,
3486          ptrue, as_FloatRegister($shift$$reg));
3487   %}
3488   ins_pipe(pipe_slow);
3489 %}
3490 
3491 instruct vlslB(vReg dst, vReg shift) %{
3492   predicate(UseSVE > 0);
3493   match(Set dst (LShiftVB dst shift));
3494   ins_cost(SVE_COST);
3495   format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (B)" %}
3496   ins_encode %{
3497     __ sve_lsl(as_FloatRegister($dst$$reg), __ B,
3498          ptrue, as_FloatRegister($shift$$reg));
3499   %}
3500   ins_pipe(pipe_slow);
3501 %}
3502 
3503 instruct vlslS(vReg dst, vReg shift) %{
3504   predicate(UseSVE > 0);
3505   match(Set dst (LShiftVS dst shift));
3506   ins_cost(SVE_COST);
3507   format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (H)" %}
3508   ins_encode %{
3509     __ sve_lsl(as_FloatRegister($dst$$reg), __ H,
3510          ptrue, as_FloatRegister($shift$$reg));
3511   %}
3512   ins_pipe(pipe_slow);
3513 %}
3514 
3515 instruct vlslI(vReg dst, vReg shift) %{
3516   predicate(UseSVE > 0);
3517   match(Set dst (LShiftVI dst shift));
3518   ins_cost(SVE_COST);
3519   format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (S)" %}
3520   ins_encode %{
3521     __ sve_lsl(as_FloatRegister($dst$$reg), __ S,
3522          ptrue, as_FloatRegister($shift$$reg));
3523   %}
3524   ins_pipe(pipe_slow);
3525 %}
3526 
3527 instruct vlslL(vReg dst, vReg shift) %{
3528   predicate(UseSVE > 0);
3529   match(Set dst (LShiftVL dst shift));
3530   ins_cost(SVE_COST);
3531   format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (D)" %}
3532   ins_encode %{
3533     __ sve_lsl(as_FloatRegister($dst$$reg), __ D,
3534          ptrue, as_FloatRegister($shift$$reg));
3535   %}
3536   ins_pipe(pipe_slow);
3537 %}
3538 
3539 instruct vlsrB(vReg dst, vReg shift) %{
3540   predicate(UseSVE > 0);
3541   match(Set dst (URShiftVB dst shift));
3542   ins_cost(SVE_COST);
3543   format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (B)" %}
3544   ins_encode %{
3545     __ sve_lsr(as_FloatRegister($dst$$reg), __ B,
3546          ptrue, as_FloatRegister($shift$$reg));
3547   %}
3548   ins_pipe(pipe_slow);
3549 %}
3550 
3551 instruct vlsrS(vReg dst, vReg shift) %{
3552   predicate(UseSVE > 0);
3553   match(Set dst (URShiftVS dst shift));
3554   ins_cost(SVE_COST);
3555   format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (H)" %}
3556   ins_encode %{
3557     __ sve_lsr(as_FloatRegister($dst$$reg), __ H,
3558          ptrue, as_FloatRegister($shift$$reg));
3559   %}
3560   ins_pipe(pipe_slow);
3561 %}
3562 
3563 instruct vlsrI(vReg dst, vReg shift) %{
3564   predicate(UseSVE > 0);
3565   match(Set dst (URShiftVI dst shift));
3566   ins_cost(SVE_COST);
3567   format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (S)" %}
3568   ins_encode %{
3569     __ sve_lsr(as_FloatRegister($dst$$reg), __ S,
3570          ptrue, as_FloatRegister($shift$$reg));
3571   %}
3572   ins_pipe(pipe_slow);
3573 %}
3574 
3575 instruct vlsrL(vReg dst, vReg shift) %{
3576   predicate(UseSVE > 0);
3577   match(Set dst (URShiftVL dst shift));
3578   ins_cost(SVE_COST);
3579   format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (D)" %}
3580   ins_encode %{
3581     __ sve_lsr(as_FloatRegister($dst$$reg), __ D,
3582          ptrue, as_FloatRegister($shift$$reg));
3583   %}
3584   ins_pipe(pipe_slow);
3585 %}
3586 
3587 instruct vasrB_imm(vReg dst, vReg src, immI shift) %{
3588   predicate(UseSVE > 0);
3589   match(Set dst (RShiftVB src (RShiftCntV shift)));
3590   ins_cost(SVE_COST);
3591   format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (B)" %}
3592   ins_encode %{
3593     int con = (int)$shift$$constant;
3594     if (con == 0) {
3595       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3596            as_FloatRegister($src$$reg));
3597       return;
3598     }
3599     if (con >= 8) con = 7;
3600     __ sve_asr(as_FloatRegister($dst$$reg), __ B,
3601          as_FloatRegister($src$$reg), con);
3602   %}
3603   ins_pipe(pipe_slow);
3604 %}
3605 
3606 instruct vasrS_imm(vReg dst, vReg src, immI shift) %{
3607   predicate(UseSVE > 0);
3608   match(Set dst (RShiftVS src (RShiftCntV shift)));
3609   ins_cost(SVE_COST);
3610   format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (H)" %}
3611   ins_encode %{
3612     int con = (int)$shift$$constant;
3613     if (con == 0) {
3614       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3615            as_FloatRegister($src$$reg));
3616       return;
3617     }
3618     if (con >= 16) con = 15;
3619     __ sve_asr(as_FloatRegister($dst$$reg), __ H,
3620          as_FloatRegister($src$$reg), con);
3621   %}
3622   ins_pipe(pipe_slow);
3623 %}
3624 
3625 instruct vasrI_imm(vReg dst, vReg src, immI shift) %{
3626   predicate(UseSVE > 0);
3627   match(Set dst (RShiftVI src (RShiftCntV shift)));
3628   ins_cost(SVE_COST);
3629   format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (S)" %}
3630   ins_encode %{
3631     int con = (int)$shift$$constant;
3632     if (con == 0) {
3633       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3634            as_FloatRegister($src$$reg));
3635       return;
3636     }
3637     __ sve_asr(as_FloatRegister($dst$$reg), __ S,
3638          as_FloatRegister($src$$reg), con);
3639   %}
3640   ins_pipe(pipe_slow);
3641 %}
3642 
3643 instruct vasrL_imm(vReg dst, vReg src, immI shift) %{
3644   predicate(UseSVE > 0);
3645   match(Set dst (RShiftVL src (RShiftCntV shift)));
3646   ins_cost(SVE_COST);
3647   format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (D)" %}
3648   ins_encode %{
3649     int con = (int)$shift$$constant;
3650     if (con == 0) {
3651       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3652            as_FloatRegister($src$$reg));
3653       return;
3654     }
3655     __ sve_asr(as_FloatRegister($dst$$reg), __ D,
3656          as_FloatRegister($src$$reg), con);
3657   %}
3658   ins_pipe(pipe_slow);
3659 %}
3660 
3661 instruct vlsrB_imm(vReg dst, vReg src, immI shift) %{
3662   predicate(UseSVE > 0);
3663   match(Set dst (URShiftVB src (RShiftCntV shift)));
3664   ins_cost(SVE_COST);
3665   format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (B)" %}
3666   ins_encode %{
3667     int con = (int)$shift$$constant;
3668     if (con == 0) {
3669       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3670            as_FloatRegister($src$$reg));
3671       return;
3672     }
3673     if (con >= 8) {
3674       __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3675            as_FloatRegister($src$$reg));
3676       return;
3677     }
3678     __ sve_lsr(as_FloatRegister($dst$$reg), __ B,
3679          as_FloatRegister($src$$reg), con);
3680   %}
3681   ins_pipe(pipe_slow);
3682 %}
3683 
3684 instruct vlsrS_imm(vReg dst, vReg src, immI shift) %{
3685   predicate(UseSVE > 0);
3686   match(Set dst (URShiftVS src (RShiftCntV shift)));
3687   ins_cost(SVE_COST);
3688   format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (H)" %}
3689   ins_encode %{
3690     int con = (int)$shift$$constant;
3691     if (con == 0) {
3692       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3693            as_FloatRegister($src$$reg));
3694       return;
3695     }
3696     if (con >= 16) {
3697       __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3698            as_FloatRegister($src$$reg));
3699       return;
3700     }
3701     __ sve_lsr(as_FloatRegister($dst$$reg), __ H,
3702          as_FloatRegister($src$$reg), con);
3703   %}
3704   ins_pipe(pipe_slow);
3705 %}
3706 
3707 instruct vlsrI_imm(vReg dst, vReg src, immI shift) %{
3708   predicate(UseSVE > 0);
3709   match(Set dst (URShiftVI src (RShiftCntV shift)));
3710   ins_cost(SVE_COST);
3711   format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (S)" %}
3712   ins_encode %{
3713     int con = (int)$shift$$constant;
3714     if (con == 0) {
3715       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3716            as_FloatRegister($src$$reg));
3717       return;
3718     }
3719     __ sve_lsr(as_FloatRegister($dst$$reg), __ S,
3720          as_FloatRegister($src$$reg), con);
3721   %}
3722   ins_pipe(pipe_slow);
3723 %}
3724 
3725 instruct vlsrL_imm(vReg dst, vReg src, immI shift) %{
3726   predicate(UseSVE > 0);
3727   match(Set dst (URShiftVL src (RShiftCntV shift)));
3728   ins_cost(SVE_COST);
3729   format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (D)" %}
3730   ins_encode %{
3731     int con = (int)$shift$$constant;
3732     if (con == 0) {
3733       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3734            as_FloatRegister($src$$reg));
3735       return;
3736     }
3737     __ sve_lsr(as_FloatRegister($dst$$reg), __ D,
3738          as_FloatRegister($src$$reg), con);
3739   %}
3740   ins_pipe(pipe_slow);
3741 %}
3742 
3743 instruct vlslB_imm(vReg dst, vReg src, immI shift) %{
3744   predicate(UseSVE > 0);
3745   match(Set dst (LShiftVB src (LShiftCntV shift)));
3746   ins_cost(SVE_COST);
3747   format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (B)" %}
3748   ins_encode %{
3749     int con = (int)$shift$$constant;
3750     if (con >= 8) {
3751       __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3752            as_FloatRegister($src$$reg));
3753       return;
3754     }
3755     __ sve_lsl(as_FloatRegister($dst$$reg), __ B,
3756          as_FloatRegister($src$$reg), con);
3757   %}
3758   ins_pipe(pipe_slow);
3759 %}
3760 
3761 instruct vlslS_imm(vReg dst, vReg src, immI shift) %{
3762   predicate(UseSVE > 0);
3763   match(Set dst (LShiftVS src (LShiftCntV shift)));
3764   ins_cost(SVE_COST);
3765   format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (H)" %}
3766   ins_encode %{
3767     int con = (int)$shift$$constant;
3768     if (con >= 16) {
3769       __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3770            as_FloatRegister($src$$reg));
3771       return;
3772     }
3773     __ sve_lsl(as_FloatRegister($dst$$reg), __ H,
3774          as_FloatRegister($src$$reg), con);
3775   %}
3776   ins_pipe(pipe_slow);
3777 %}
3778 
3779 instruct vlslI_imm(vReg dst, vReg src, immI shift) %{
3780   predicate(UseSVE > 0);
3781   match(Set dst (LShiftVI src (LShiftCntV shift)));
3782   ins_cost(SVE_COST);
3783   format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (S)" %}
3784   ins_encode %{
3785     int con = (int)$shift$$constant;
3786     __ sve_lsl(as_FloatRegister($dst$$reg), __ S,
3787          as_FloatRegister($src$$reg), con);
3788   %}
3789   ins_pipe(pipe_slow);
3790 %}
3791 
3792 instruct vlslL_imm(vReg dst, vReg src, immI shift) %{
3793   predicate(UseSVE > 0);
3794   match(Set dst (LShiftVL src (LShiftCntV shift)));
3795   ins_cost(SVE_COST);
3796   format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (D)" %}
3797   ins_encode %{
3798     int con = (int)$shift$$constant;
3799     __ sve_lsl(as_FloatRegister($dst$$reg), __ D,
3800          as_FloatRegister($src$$reg), con);
3801   %}
3802   ins_pipe(pipe_slow);
3803 %}
3804 
3805 instruct vshiftcntB(vReg dst, iRegIorL2I cnt) %{
3806   predicate(UseSVE > 0 &&
3807             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE));
3808   match(Set dst (LShiftCntV cnt));
3809   match(Set dst (RShiftCntV cnt));
3810   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (B)" %}
3811   ins_encode %{
3812     __ sve_dup(as_FloatRegister($dst$$reg), __ B, as_Register($cnt$$reg));
3813   %}
3814   ins_pipe(pipe_slow);
3815 %}
3816 
3817 instruct vshiftcntS(vReg dst, iRegIorL2I cnt) %{
3818   predicate(UseSVE > 0 &&
3819             (n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
3820             (n->bottom_type()->is_vect()->element_basic_type() == T_CHAR)));
3821   match(Set dst (LShiftCntV cnt));
3822   match(Set dst (RShiftCntV cnt));
3823   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (H)" %}
3824   ins_encode %{
3825     __ sve_dup(as_FloatRegister($dst$$reg), __ H, as_Register($cnt$$reg));
3826   %}
3827   ins_pipe(pipe_slow);
3828 %}
3829 
3830 instruct vshiftcntI(vReg dst, iRegIorL2I cnt) %{
3831   predicate(UseSVE > 0 &&
3832             (n->bottom_type()->is_vect()->element_basic_type() == T_INT));
3833   match(Set dst (LShiftCntV cnt));
3834   match(Set dst (RShiftCntV cnt));
3835   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (S)" %}
3836   ins_encode %{
3837     __ sve_dup(as_FloatRegister($dst$$reg), __ S, as_Register($cnt$$reg));
3838   %}
3839   ins_pipe(pipe_slow);
3840 %}
3841 
3842 instruct vshiftcntL(vReg dst, iRegIorL2I cnt) %{
3843   predicate(UseSVE > 0 &&
3844             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
3845   match(Set dst (LShiftCntV cnt));
3846   match(Set dst (RShiftCntV cnt));
3847   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (D)" %}
3848   ins_encode %{
3849     __ sve_dup(as_FloatRegister($dst$$reg), __ D, as_Register($cnt$$reg));
3850   %}
3851   ins_pipe(pipe_slow);
3852 %}
3853 
3854 // vector shift - predicated
3855 
3856 instruct vasrB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3857   predicate(UseSVE > 0);
3858   match(Set dst_src1 (RShiftVB (Binary dst_src1 src2) pg));
3859   ins_cost(SVE_COST);
3860   format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
3861   ins_encode %{
3862     __ sve_asr(as_FloatRegister($dst_src1$$reg), __ B,
3863             as_PRegister($pg$$reg),
3864             as_FloatRegister($src2$$reg));
3865   %}
3866   ins_pipe(pipe_slow);
3867 %}
3868 
3869 instruct vasrS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3870   predicate(UseSVE > 0);
3871   match(Set dst_src1 (RShiftVS (Binary dst_src1 src2) pg));
3872   ins_cost(SVE_COST);
3873   format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
3874   ins_encode %{
3875     __ sve_asr(as_FloatRegister($dst_src1$$reg), __ H,
3876             as_PRegister($pg$$reg),
3877             as_FloatRegister($src2$$reg));
3878   %}
3879   ins_pipe(pipe_slow);
3880 %}
3881 
3882 instruct vasrI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3883   predicate(UseSVE > 0);
3884   match(Set dst_src1 (RShiftVI (Binary dst_src1 src2) pg));
3885   ins_cost(SVE_COST);
3886   format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
3887   ins_encode %{
3888     __ sve_asr(as_FloatRegister($dst_src1$$reg), __ S,
3889             as_PRegister($pg$$reg),
3890             as_FloatRegister($src2$$reg));
3891   %}
3892   ins_pipe(pipe_slow);
3893 %}
3894 
3895 instruct vasrL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3896   predicate(UseSVE > 0);
3897   match(Set dst_src1 (RShiftVL (Binary dst_src1 src2) pg));
3898   ins_cost(SVE_COST);
3899   format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
3900   ins_encode %{
3901     __ sve_asr(as_FloatRegister($dst_src1$$reg), __ D,
3902             as_PRegister($pg$$reg),
3903             as_FloatRegister($src2$$reg));
3904   %}
3905   ins_pipe(pipe_slow);
3906 %}
3907 
3908 instruct vlslB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3909   predicate(UseSVE > 0);
3910   match(Set dst_src1 (LShiftVB (Binary dst_src1 src2) pg));
3911   ins_cost(SVE_COST);
3912   format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
3913   ins_encode %{
3914     __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ B,
3915             as_PRegister($pg$$reg),
3916             as_FloatRegister($src2$$reg));
3917   %}
3918   ins_pipe(pipe_slow);
3919 %}
3920 
3921 instruct vlslS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3922   predicate(UseSVE > 0);
3923   match(Set dst_src1 (LShiftVS (Binary dst_src1 src2) pg));
3924   ins_cost(SVE_COST);
3925   format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
3926   ins_encode %{
3927     __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ H,
3928             as_PRegister($pg$$reg),
3929             as_FloatRegister($src2$$reg));
3930   %}
3931   ins_pipe(pipe_slow);
3932 %}
3933 
3934 instruct vlslI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3935   predicate(UseSVE > 0);
3936   match(Set dst_src1 (LShiftVI (Binary dst_src1 src2) pg));
3937   ins_cost(SVE_COST);
3938   format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
3939   ins_encode %{
3940     __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ S,
3941             as_PRegister($pg$$reg),
3942             as_FloatRegister($src2$$reg));
3943   %}
3944   ins_pipe(pipe_slow);
3945 %}
3946 
3947 instruct vlslL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3948   predicate(UseSVE > 0);
3949   match(Set dst_src1 (LShiftVL (Binary dst_src1 src2) pg));
3950   ins_cost(SVE_COST);
3951   format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
3952   ins_encode %{
3953     __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ D,
3954             as_PRegister($pg$$reg),
3955             as_FloatRegister($src2$$reg));
3956   %}
3957   ins_pipe(pipe_slow);
3958 %}
3959 
3960 instruct vlsrB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3961   predicate(UseSVE > 0);
3962   match(Set dst_src1 (URShiftVB (Binary dst_src1 src2) pg));
3963   ins_cost(SVE_COST);
3964   format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
3965   ins_encode %{
3966     __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ B,
3967             as_PRegister($pg$$reg),
3968             as_FloatRegister($src2$$reg));
3969   %}
3970   ins_pipe(pipe_slow);
3971 %}
3972 
3973 instruct vlsrS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3974   predicate(UseSVE > 0);
3975   match(Set dst_src1 (URShiftVS (Binary dst_src1 src2) pg));
3976   ins_cost(SVE_COST);
3977   format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
3978   ins_encode %{
3979     __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ H,
3980             as_PRegister($pg$$reg),
3981             as_FloatRegister($src2$$reg));
3982   %}
3983   ins_pipe(pipe_slow);
3984 %}
3985 
3986 instruct vlsrI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3987   predicate(UseSVE > 0);
3988   match(Set dst_src1 (URShiftVI (Binary dst_src1 src2) pg));
3989   ins_cost(SVE_COST);
3990   format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
3991   ins_encode %{
3992     __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ S,
3993             as_PRegister($pg$$reg),
3994             as_FloatRegister($src2$$reg));
3995   %}
3996   ins_pipe(pipe_slow);
3997 %}
3998 
3999 instruct vlsrL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4000   predicate(UseSVE > 0);
4001   match(Set dst_src1 (URShiftVL (Binary dst_src1 src2) pg));
4002   ins_cost(SVE_COST);
4003   format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
4004   ins_encode %{
4005     __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ D,
4006             as_PRegister($pg$$reg),
4007             as_FloatRegister($src2$$reg));
4008   %}
4009   ins_pipe(pipe_slow);
4010 %}
4011 
4012 instruct vasrB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4013   predicate(UseSVE > 0);
4014   match(Set dst_src (RShiftVB (Binary dst_src (RShiftCntV shift)) pg));
4015   ins_cost(SVE_COST);
4016   format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %}
4017   ins_encode %{
4018     int con = (int)$shift$$constant;
4019     assert(con > 0 && con < 8, "invalid shift immediate");
4020     __ sve_asr(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con);
4021   %}
4022   ins_pipe(pipe_slow);
4023 %}
4024 
4025 instruct vasrS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4026   predicate(UseSVE > 0);
4027   match(Set dst_src (RShiftVS (Binary dst_src (RShiftCntV shift)) pg));
4028   ins_cost(SVE_COST);
4029   format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %}
4030   ins_encode %{
4031     int con = (int)$shift$$constant;
4032     assert(con > 0 && con < 16, "invalid shift immediate");
4033     __ sve_asr(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con);
4034   %}
4035   ins_pipe(pipe_slow);
4036 %}
4037 
4038 instruct vasrI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4039   predicate(UseSVE > 0);
4040   match(Set dst_src (RShiftVI (Binary dst_src (RShiftCntV shift)) pg));
4041   ins_cost(SVE_COST);
4042   format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %}
4043   ins_encode %{
4044     int con = (int)$shift$$constant;
4045     assert(con > 0 && con < 32, "invalid shift immediate");
4046     __ sve_asr(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con);
4047   %}
4048   ins_pipe(pipe_slow);
4049 %}
4050 
4051 instruct vasrL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4052   predicate(UseSVE > 0);
4053   match(Set dst_src (RShiftVL (Binary dst_src (RShiftCntV shift)) pg));
4054   ins_cost(SVE_COST);
4055   format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %}
4056   ins_encode %{
4057     int con = (int)$shift$$constant;
4058     assert(con > 0 && con < 64, "invalid shift immediate");
4059     __ sve_asr(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con);
4060   %}
4061   ins_pipe(pipe_slow);
4062 %}
4063 
4064 instruct vlsrB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4065   predicate(UseSVE > 0);
4066   match(Set dst_src (URShiftVB (Binary dst_src (RShiftCntV shift)) pg));
4067   ins_cost(SVE_COST);
4068   format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %}
4069   ins_encode %{
4070     int con = (int)$shift$$constant;
4071     assert(con > 0 && con < 8, "invalid shift immediate");
4072     __ sve_lsr(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con);
4073   %}
4074   ins_pipe(pipe_slow);
4075 %}
4076 
4077 instruct vlsrS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4078   predicate(UseSVE > 0);
4079   match(Set dst_src (URShiftVS (Binary dst_src (RShiftCntV shift)) pg));
4080   ins_cost(SVE_COST);
4081   format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %}
4082   ins_encode %{
4083     int con = (int)$shift$$constant;
4084     assert(con > 0 && con < 16, "invalid shift immediate");
4085     __ sve_lsr(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con);
4086   %}
4087   ins_pipe(pipe_slow);
4088 %}
4089 
4090 instruct vlsrI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4091   predicate(UseSVE > 0);
4092   match(Set dst_src (URShiftVI (Binary dst_src (RShiftCntV shift)) pg));
4093   ins_cost(SVE_COST);
4094   format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %}
4095   ins_encode %{
4096     int con = (int)$shift$$constant;
4097     assert(con > 0 && con < 32, "invalid shift immediate");
4098     __ sve_lsr(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con);
4099   %}
4100   ins_pipe(pipe_slow);
4101 %}
4102 
4103 instruct vlsrL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4104   predicate(UseSVE > 0);
4105   match(Set dst_src (URShiftVL (Binary dst_src (RShiftCntV shift)) pg));
4106   ins_cost(SVE_COST);
4107   format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %}
4108   ins_encode %{
4109     int con = (int)$shift$$constant;
4110     assert(con > 0 && con < 64, "invalid shift immediate");
4111     __ sve_lsr(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con);
4112   %}
4113   ins_pipe(pipe_slow);
4114 %}
4115 
4116 instruct vlslB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4117   predicate(UseSVE > 0);
4118   match(Set dst_src (LShiftVB (Binary dst_src (LShiftCntV shift)) pg));
4119   ins_cost(SVE_COST);
4120   format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %}
4121   ins_encode %{
4122     int con = (int)$shift$$constant;
4123     assert(con >= 0 && con < 8, "invalid shift immediate");
4124     __ sve_lsl(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con);
4125   %}
4126   ins_pipe(pipe_slow);
4127 %}
4128 
4129 instruct vlslS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4130   predicate(UseSVE > 0);
4131   match(Set dst_src (LShiftVS (Binary dst_src (LShiftCntV shift)) pg));
4132   ins_cost(SVE_COST);
4133   format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %}
4134   ins_encode %{
4135     int con = (int)$shift$$constant;
4136     assert(con >= 0 && con < 16, "invalid shift immediate");
4137     __ sve_lsl(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con);
4138   %}
4139   ins_pipe(pipe_slow);
4140 %}
4141 
4142 instruct vlslI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4143   predicate(UseSVE > 0);
4144   match(Set dst_src (LShiftVI (Binary dst_src (LShiftCntV shift)) pg));
4145   ins_cost(SVE_COST);
4146   format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %}
4147   ins_encode %{
4148     int con = (int)$shift$$constant;
4149     assert(con >= 0 && con < 32, "invalid shift immediate");
4150     __ sve_lsl(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con);
4151   %}
4152   ins_pipe(pipe_slow);
4153 %}
4154 
4155 instruct vlslL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4156   predicate(UseSVE > 0);
4157   match(Set dst_src (LShiftVL (Binary dst_src (LShiftCntV shift)) pg));
4158   ins_cost(SVE_COST);
4159   format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %}
4160   ins_encode %{
4161     int con = (int)$shift$$constant;
4162     assert(con >= 0 && con < 64, "invalid shift immediate");
4163     __ sve_lsl(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con);
4164   %}
4165   ins_pipe(pipe_slow);
4166 %}
4167 
4168 // vector sqrt
4169 
4170 instruct vsqrtF(vReg dst, vReg src) %{
4171   predicate(UseSVE > 0 &&
4172             !n->as_Vector()->is_predicated_vector());
4173   match(Set dst (SqrtVF src));
4174   ins_cost(SVE_COST);
4175   format %{ "sve_fsqrt $dst, $src\t# vector (sve) (S)" %}
4176   ins_encode %{
4177     __ sve_fsqrt(as_FloatRegister($dst$$reg), __ S,
4178          ptrue, as_FloatRegister($src$$reg));
4179   %}
4180   ins_pipe(pipe_slow);
4181 %}
4182 
4183 instruct vsqrtD(vReg dst, vReg src) %{
4184   predicate(UseSVE > 0 &&
4185             !n->as_Vector()->is_predicated_vector());
4186   match(Set dst (SqrtVD src));
4187   ins_cost(SVE_COST);
4188   format %{ "sve_fsqrt $dst, $src\t# vector (sve) (D)" %}
4189   ins_encode %{
4190     __ sve_fsqrt(as_FloatRegister($dst$$reg), __ D,
4191          ptrue, as_FloatRegister($src$$reg));
4192   %}
4193   ins_pipe(pipe_slow);
4194 %}
4195 
4196 // vector sqrt - predicated
4197 
4198 instruct vsqrtF_masked(vReg dst_src, pRegGov pg) %{
4199   predicate(UseSVE > 0);
4200   match(Set dst_src (SqrtVF dst_src pg));
4201   ins_cost(SVE_COST);
4202   format %{ "sve_fsqrt $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
4203   ins_encode %{
4204     __ sve_fsqrt(as_FloatRegister($dst_src$$reg), __ S,
4205             as_PRegister($pg$$reg),
4206             as_FloatRegister($dst_src$$reg));
4207   %}
4208   ins_pipe(pipe_slow);
4209 %}
4210 
4211 instruct vsqrtD_masked(vReg dst_src, pRegGov pg) %{
4212   predicate(UseSVE > 0);
4213   match(Set dst_src (SqrtVD dst_src pg));
4214   ins_cost(SVE_COST);
4215   format %{ "sve_fsqrt $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
4216   ins_encode %{
4217     __ sve_fsqrt(as_FloatRegister($dst_src$$reg), __ D,
4218             as_PRegister($pg$$reg),
4219             as_FloatRegister($dst_src$$reg));
4220   %}
4221   ins_pipe(pipe_slow);
4222 %}
4223 
4224 // vector sub
4225 
4226 instruct vsubB(vReg dst, vReg src1, vReg src2) %{
4227   predicate(UseSVE > 0);
4228   match(Set dst (SubVB src1 src2));
4229   ins_cost(SVE_COST);
4230   format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (B)" %}
4231   ins_encode %{
4232     __ sve_sub(as_FloatRegister($dst$$reg), __ B,
4233          as_FloatRegister($src1$$reg),
4234          as_FloatRegister($src2$$reg));
4235   %}
4236   ins_pipe(pipe_slow);
4237 %}
4238 
4239 instruct vsubS(vReg dst, vReg src1, vReg src2) %{
4240   predicate(UseSVE > 0);
4241   match(Set dst (SubVS src1 src2));
4242   ins_cost(SVE_COST);
4243   format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (H)" %}
4244   ins_encode %{
4245     __ sve_sub(as_FloatRegister($dst$$reg), __ H,
4246          as_FloatRegister($src1$$reg),
4247          as_FloatRegister($src2$$reg));
4248   %}
4249   ins_pipe(pipe_slow);
4250 %}
4251 
4252 instruct vsubI(vReg dst, vReg src1, vReg src2) %{
4253   predicate(UseSVE > 0);
4254   match(Set dst (SubVI src1 src2));
4255   ins_cost(SVE_COST);
4256   format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (S)" %}
4257   ins_encode %{
4258     __ sve_sub(as_FloatRegister($dst$$reg), __ S,
4259          as_FloatRegister($src1$$reg),
4260          as_FloatRegister($src2$$reg));
4261   %}
4262   ins_pipe(pipe_slow);
4263 %}
4264 
4265 instruct vsubL(vReg dst, vReg src1, vReg src2) %{
4266   predicate(UseSVE > 0);
4267   match(Set dst (SubVL src1 src2));
4268   ins_cost(SVE_COST);
4269   format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (D)" %}
4270   ins_encode %{
4271     __ sve_sub(as_FloatRegister($dst$$reg), __ D,
4272          as_FloatRegister($src1$$reg),
4273          as_FloatRegister($src2$$reg));
4274   %}
4275   ins_pipe(pipe_slow);
4276 %}
4277 
4278 instruct vsubF(vReg dst, vReg src1, vReg src2) %{
4279   predicate(UseSVE > 0);
4280   match(Set dst (SubVF src1 src2));
4281   ins_cost(SVE_COST);
4282   format %{ "sve_fsub $dst, $src1, $src2\t # vector (sve) (S)" %}
4283   ins_encode %{
4284     __ sve_fsub(as_FloatRegister($dst$$reg), __ S,
4285          as_FloatRegister($src1$$reg),
4286          as_FloatRegister($src2$$reg));
4287   %}
4288   ins_pipe(pipe_slow);
4289 %}
4290 
4291 instruct vsubD(vReg dst, vReg src1, vReg src2) %{
4292   predicate(UseSVE > 0);
4293   match(Set dst (SubVD src1 src2));
4294   ins_cost(SVE_COST);
4295   format %{ "sve_fsub $dst, $src1, $src2\t # vector (sve) (D)" %}
4296   ins_encode %{
4297     __ sve_fsub(as_FloatRegister($dst$$reg), __ D,
4298          as_FloatRegister($src1$$reg),
4299          as_FloatRegister($src2$$reg));
4300   %}
4301   ins_pipe(pipe_slow);
4302 %}
4303 
4304 // vector sub - predicated
4305 
4306 instruct vsubB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4307   predicate(UseSVE > 0);
4308   match(Set dst_src1 (SubVB (Binary dst_src1 src2) pg));
4309   ins_cost(SVE_COST);
4310   format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
4311   ins_encode %{
4312     __ sve_sub(as_FloatRegister($dst_src1$$reg), __ B,
4313             as_PRegister($pg$$reg),
4314             as_FloatRegister($src2$$reg));
4315   %}
4316   ins_pipe(pipe_slow);
4317 %}
4318 
4319 instruct vsubS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4320   predicate(UseSVE > 0);
4321   match(Set dst_src1 (SubVS (Binary dst_src1 src2) pg));
4322   ins_cost(SVE_COST);
4323   format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
4324   ins_encode %{
4325     __ sve_sub(as_FloatRegister($dst_src1$$reg), __ H,
4326             as_PRegister($pg$$reg),
4327             as_FloatRegister($src2$$reg));
4328   %}
4329   ins_pipe(pipe_slow);
4330 %}
4331 
4332 instruct vsubI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4333   predicate(UseSVE > 0);
4334   match(Set dst_src1 (SubVI (Binary dst_src1 src2) pg));
4335   ins_cost(SVE_COST);
4336   format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
4337   ins_encode %{
4338     __ sve_sub(as_FloatRegister($dst_src1$$reg), __ S,
4339             as_PRegister($pg$$reg),
4340             as_FloatRegister($src2$$reg));
4341   %}
4342   ins_pipe(pipe_slow);
4343 %}
4344 
4345 instruct vsubL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4346   predicate(UseSVE > 0);
4347   match(Set dst_src1 (SubVL (Binary dst_src1 src2) pg));
4348   ins_cost(SVE_COST);
4349   format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
4350   ins_encode %{
4351     __ sve_sub(as_FloatRegister($dst_src1$$reg), __ D,
4352             as_PRegister($pg$$reg),
4353             as_FloatRegister($src2$$reg));
4354   %}
4355   ins_pipe(pipe_slow);
4356 %}
4357 
4358 instruct vsubF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4359   predicate(UseSVE > 0);
4360   match(Set dst_src1 (SubVF (Binary dst_src1 src2) pg));
4361   ins_cost(SVE_COST);
4362   format %{ "sve_fsub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
4363   ins_encode %{
4364     __ sve_fsub(as_FloatRegister($dst_src1$$reg), __ S,
4365             as_PRegister($pg$$reg),
4366             as_FloatRegister($src2$$reg));
4367   %}
4368   ins_pipe(pipe_slow);
4369 %}
4370 
4371 instruct vsubD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4372   predicate(UseSVE > 0);
4373   match(Set dst_src1 (SubVD (Binary dst_src1 src2) pg));
4374   ins_cost(SVE_COST);
4375   format %{ "sve_fsub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
4376   ins_encode %{
4377     __ sve_fsub(as_FloatRegister($dst_src1$$reg), __ D,
4378             as_PRegister($pg$$reg),
4379             as_FloatRegister($src2$$reg));
4380   %}
4381   ins_pipe(pipe_slow);
4382 %}
4383 
4384 // ------------------------------ Vector mask cast --------------------------
4385 
4386 instruct vmaskcast(pRegGov dst_src) %{
4387   predicate(UseSVE > 0 &&
4388             n->bottom_type()->is_vect()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
4389             n->bottom_type()->is_vect()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
4390   match(Set dst_src (VectorMaskCast dst_src));
4391   ins_cost(0);
4392   format %{ "vmaskcast $dst_src\t# empty (sve)" %}
4393   ins_encode %{
4394     // empty
4395   %}
4396   ins_pipe(pipe_class_empty);
4397 %}
4398 
4399 instruct vmaskcast_extend(pRegGov dst, pReg src)
4400 %{
4401   predicate(UseSVE > 0 &&
4402             (Matcher::vector_length_in_bytes(n) == 2 * Matcher::vector_length_in_bytes(n->in(1)) ||
4403              Matcher::vector_length_in_bytes(n) == 4 * Matcher::vector_length_in_bytes(n->in(1)) ||
4404              Matcher::vector_length_in_bytes(n) == 8 * Matcher::vector_length_in_bytes(n->in(1))));
4405   match(Set dst (VectorMaskCast src));
4406   ins_cost(SVE_COST * 3);
4407   format %{ "sve_vmaskcast_extend  $dst, $src\t# extend predicate $src" %}
4408   ins_encode %{
4409     __ sve_vmaskcast_extend(as_PRegister($dst$$reg), as_PRegister($src$$reg),
4410                             Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
4411   %}
4412   ins_pipe(pipe_slow);
4413 %}
4414 
4415 instruct vmaskcast_narrow(pRegGov dst, pReg src)
4416 %{
4417   predicate(UseSVE > 0 &&
4418             (Matcher::vector_length_in_bytes(n) * 2 == Matcher::vector_length_in_bytes(n->in(1)) ||
4419              Matcher::vector_length_in_bytes(n) * 4 == Matcher::vector_length_in_bytes(n->in(1)) ||
4420              Matcher::vector_length_in_bytes(n) * 8 == Matcher::vector_length_in_bytes(n->in(1))));
4421   match(Set dst (VectorMaskCast src));
4422   ins_cost(SVE_COST * 3);
4423   format %{ "sve_vmaskcast_narrow  $dst, $src\t# narrow predicate $src" %}
4424   ins_encode %{
4425     __ sve_vmaskcast_narrow(as_PRegister($dst$$reg), as_PRegister($src$$reg),
4426                             Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
4427   %}
4428   ins_pipe(pipe_slow);
4429 %}
4430 
4431 // ------------------------------ Vector cast -------------------------------
4432 
4433 instruct vcvtBtoX_extend(vReg dst, vReg src)
4434 %{
4435   predicate(UseSVE > 0);
4436   match(Set dst (VectorCastB2X src));
4437   ins_cost(2 * SVE_COST);
4438   format %{ "sve_vectorcast_b2x  $dst, $src\t# convert B to X vector (extend)" %}
4439   ins_encode %{
4440     BasicType to_bt = Matcher::vector_element_basic_type(this);
4441     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4442     __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ B);
4443     if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
4444       __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
4445     }
4446   %}
4447   ins_pipe(pipe_slow);
4448 %}
4449 
4450 instruct vcvtStoB(vReg dst, vReg src, vReg tmp)
4451 %{
4452   predicate(UseSVE > 0 &&
4453             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
4454   match(Set dst (VectorCastS2X src));
4455   effect(TEMP tmp);
4456   ins_cost(2 * SVE_COST);
4457   format %{ "sve_vectorcast_s2b  $dst, $src\t# convert H to B vector" %}
4458   ins_encode %{
4459     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
4460                          as_FloatRegister($src$$reg), __ H, as_FloatRegister($tmp$$reg));
4461   %}
4462   ins_pipe(pipe_slow);
4463 %}
4464 
4465 instruct vcvtStoX_extend(vReg dst, vReg src)
4466 %{
4467   predicate(UseSVE > 0 &&
4468             type2aelembytes(Matcher::vector_element_basic_type(n)) > 2);
4469   match(Set dst (VectorCastS2X src));
4470   ins_cost(2 * SVE_COST);
4471   format %{ "sve_vectorcast_s2x  $dst, $src\t# convert H to X vector (extend)" %}
4472   ins_encode %{
4473     BasicType to_bt = Matcher::vector_element_basic_type(this);
4474     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4475     __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ H);
4476     if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
4477       __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
4478     }
4479   %}
4480   ins_pipe(pipe_slow);
4481 %}
4482 
4483 instruct vcvtItoB(vReg dst, vReg src, vReg tmp)
4484 %{
4485   predicate(UseSVE > 0 &&
4486             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
4487   match(Set dst (VectorCastI2X src));
4488   effect(TEMP_DEF dst, TEMP tmp);
4489   ins_cost(3 * SVE_COST);
4490   format %{ "sve_vectorcast_i2b  $dst, $src\t# convert I to B vector" %}
4491   ins_encode %{
4492     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
4493                          as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
4494   %}
4495   ins_pipe(pipe_slow);
4496 %}
4497 
4498 instruct vcvtItoS(vReg dst, vReg src, vReg tmp)
4499 %{
4500   predicate(UseSVE > 0 &&
4501             n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
4502   match(Set dst (VectorCastI2X src));
4503   effect(TEMP tmp);
4504   ins_cost(2 * SVE_COST);
4505   format %{ "sve_vectorcast_i2s $dst, $src\t# convert I to H vector" %}
4506   ins_encode %{
4507     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ H,
4508                          as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
4509   %}
4510   ins_pipe(pipe_slow);
4511 %}
4512 
4513 instruct vcvtItoL(vReg dst, vReg src)
4514 %{
4515   predicate(UseSVE > 0 &&
4516             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
4517   match(Set dst (VectorCastI2X src));
4518   ins_cost(SVE_COST);
4519   format %{ "sve_vectorcast_i2l  $dst, $src\t# convert I to L vector" %}
4520   ins_encode %{
4521     __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
4522   %}
4523   ins_pipe(pipe_slow);
4524 %}
4525 
4526 instruct vcvtItoF(vReg dst, vReg src)
4527 %{
4528   predicate(UseSVE > 0 &&
4529             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4530   match(Set dst (VectorCastI2X src));
4531   ins_cost(SVE_COST);
4532   format %{ "sve_vectorcast_i2f  $dst, $src\t# convert I to F vector" %}
4533   ins_encode %{
4534     __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
4535   %}
4536   ins_pipe(pipe_slow);
4537 %}
4538 
4539 instruct vcvtItoD(vReg dst, vReg src)
4540 %{
4541   predicate(UseSVE > 0 &&
4542             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
4543   match(Set dst (VectorCastI2X src));
4544   ins_cost(2 * SVE_COST);
4545   format %{ "sve_vectorcast_i2d  $dst, $src\t# convert I to D vector" %}
4546   ins_encode %{
4547     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
4548     __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ D);
4549   %}
4550   ins_pipe(pipe_slow);
4551 %}
4552 
4553 instruct vcvtLtoX_narrow(vReg dst, vReg src, vReg tmp)
4554 %{
4555   predicate(UseSVE > 0 && is_integral_type(Matcher::vector_element_basic_type(n)));
4556   match(Set dst (VectorCastL2X src));
4557   effect(TEMP_DEF dst, TEMP tmp);
4558   ins_cost(2 * SVE_COST);
4559   format %{ "sve_vectorcast_l2x  $dst, $src\t# convert L to B/H/S vector (narrow)" %}
4560   ins_encode %{
4561     BasicType to_bt = Matcher::vector_element_basic_type(this);
4562     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4563     __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
4564                          as_FloatRegister($src$$reg), __ D, as_FloatRegister($tmp$$reg));
4565   %}
4566   ins_pipe(pipe_slow);
4567 %}
4568 
4569 instruct vcvtLtoF(vReg dst, vReg src, vReg tmp)
4570 %{
4571   predicate(UseSVE > 0 &&
4572             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4573   match(Set dst (VectorCastL2X src));
4574   effect(TEMP_DEF dst, TEMP tmp);
4575   ins_cost(3 * SVE_COST);
4576   format %{ "sve_vectorcast_l2f  $dst, $src\t# convert L to F vector" %}
4577   ins_encode %{
4578     __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
4579     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
4580                          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
4581 
4582   %}
4583   ins_pipe(pipe_slow);
4584 %}
4585 
4586 instruct vcvtLtoD(vReg dst, vReg src)
4587 %{
4588   predicate(UseSVE > 0 &&
4589             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
4590   match(Set dst (VectorCastL2X src));
4591   ins_cost(SVE_COST);
4592   format %{ "sve_vectorcast_l2d  $dst, $src\t# convert L to D vector" %}
4593   ins_encode %{
4594     __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
4595   %}
4596   ins_pipe(pipe_slow);
4597 %}
4598 
4599 instruct vcvtFtoX_narrow(vReg dst, vReg src, vReg tmp)
4600 %{
4601   predicate(UseSVE > 0 &&
4602             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4603              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT));
4604   match(Set dst (VectorCastF2X src));
4605   effect(TEMP_DEF dst, TEMP tmp);
4606   ins_cost(3 * SVE_COST);
4607   format %{ "sve_vectorcast_f2x  $dst, $src\t# convert F to B/H vector" %}
4608   ins_encode %{
4609     BasicType to_bt = Matcher::vector_element_basic_type(this);
4610     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4611     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
4612     __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
4613                          as_FloatRegister($dst$$reg), __ S, as_FloatRegister($tmp$$reg));
4614   %}
4615   ins_pipe(pipe_slow);
4616 %}
4617 
4618 instruct vcvtFtoI(vReg dst, vReg src)
4619 %{
4620   predicate(UseSVE > 0 &&
4621             (n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4622   match(Set dst (VectorCastF2X src));
4623   ins_cost(SVE_COST);
4624   format %{ "sve_vectorcast_f2x  $dst, $src\t# convert F to I vector" %}
4625   ins_encode %{
4626     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
4627   %}
4628   ins_pipe(pipe_slow);
4629 %}
4630 
4631 instruct vcvtFtoL(vReg dst, vReg src)
4632 %{
4633   predicate(UseSVE > 0 &&
4634             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
4635   match(Set dst (VectorCastF2X src));
4636   ins_cost(SVE_COST * 2);
4637   format %{ "sve_vectorcast_f2x  $dst, $src\t# convert F to L vector" %}
4638   ins_encode %{
4639     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
4640     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);
4641   %}
4642   ins_pipe(pipe_slow);
4643 %}
4644 
4645 instruct vcvtFtoD(vReg dst, vReg src)
4646 %{
4647   predicate(UseSVE > 0 &&
4648             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
4649   match(Set dst (VectorCastF2X src));
4650   ins_cost(2 * SVE_COST);
4651   format %{ "sve_vectorcast_f2d  $dst, $dst\t# convert F to D vector" %}
4652   ins_encode %{
4653     __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
4654     __ sve_fcvt(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);
4655   %}
4656   ins_pipe(pipe_slow);
4657 %}
4658 
4659 instruct vcvtDtoX_narrow(vReg dst, vReg src, vReg tmp)
4660 %{
4661   predicate(UseSVE > 0 &&
4662             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4663              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
4664              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4665   match(Set dst (VectorCastD2X src));
4666   effect(TEMP_DEF dst, TEMP tmp);
4667   ins_cost(3 * SVE_COST);
4668   format %{ "sve_vectorcast_d2x  $dst, $src\t# convert D to X vector (narrow)" %}
4669   ins_encode %{
4670     BasicType to_bt = Matcher::vector_element_basic_type(this);
4671     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4672     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
4673     __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
4674                          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
4675   %}
4676   ins_pipe(pipe_slow);
4677 %}
4678 
4679 instruct vcvtDtoL(vReg dst, vReg src)
4680 %{
4681   predicate(UseSVE > 0 &&
4682             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
4683   match(Set dst (VectorCastD2X src));
4684   ins_cost(SVE_COST);
4685   format %{ "sve_vectorcast_d2l  $dst, $src\t# convert D to L vector" %}
4686   ins_encode %{
4687     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
4688   %}
4689   ins_pipe(pipe_slow);
4690 %}
4691 
4692 instruct vcvtDtoF(vReg dst, vReg src, vReg tmp)
4693 %{
4694   predicate(UseSVE > 0 &&
4695             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4696   match(Set dst (VectorCastD2X src));
4697   effect(TEMP_DEF dst, TEMP tmp);
4698   ins_cost(3 * SVE_COST);
4699   format %{ "sve_vectorcast_d2f  $dst, S, $dst\t# convert D to F vector" %}
4700   ins_encode %{
4701     __ sve_fcvt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
4702     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
4703                          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
4704   %}
4705   ins_pipe(pipe_slow);
4706 %}
4707 
4708 // ------------------------------ Vector extract ---------------------------------
4709 
4710 instruct extractB(iRegINoSp dst, vReg src, immI idx, vReg vtmp)
4711 %{
4712   predicate(UseSVE > 0 && n->in(2)->get_int() >= 16);
4713   match(Set dst (ExtractB src idx));
4714   effect(TEMP vtmp);
4715   ins_cost(2 * SVE_COST);
4716   format %{ "sve_extract_integral $dst, B, $src, $idx\t# extract from vector(B)" %}
4717   ins_encode %{
4718     __ sve_extract_integral(as_Register($dst$$reg), __ B, as_FloatRegister($src$$reg),
4719                             (int)($idx$$constant), /* is_signed */ true, as_FloatRegister($vtmp$$reg));
4720   %}
4721   ins_pipe(pipe_slow);
4722 %}
4723 
4724 instruct extractS(iRegINoSp dst, vReg src, immI idx, vReg vtmp)
4725 %{
4726   predicate(UseSVE > 0 && n->in(2)->get_int() >= 8);
4727   match(Set dst (ExtractS src idx));
4728   effect(TEMP vtmp);
4729   ins_cost(2 * SVE_COST);
4730   format %{ "sve_extract_integral $dst, H, $src, $idx\t# extract from vector(S)" %}
4731   ins_encode %{
4732     __ sve_extract_integral(as_Register($dst$$reg), __ H, as_FloatRegister($src$$reg),
4733                             (int)($idx$$constant), /* is_signed */ true, as_FloatRegister($vtmp$$reg));
4734   %}
4735   ins_pipe(pipe_slow);
4736 %}
4737 
4738 instruct extractI(iRegINoSp dst, vReg src, immI idx, vReg vtmp)
4739 %{
4740   predicate(UseSVE > 0 && n->in(2)->get_int() >= 4);
4741   match(Set dst (ExtractI src idx));
4742   effect(TEMP vtmp);
4743   ins_cost(2 * SVE_COST);
4744   format %{ "sve_extract_integral $dst, S, $src, $idx\t# extract from vector(I)" %}
4745   ins_encode %{
4746     __ sve_extract_integral(as_Register($dst$$reg), __ S, as_FloatRegister($src$$reg),
4747                             (int)($idx$$constant), /* is_signed */ true, as_FloatRegister($vtmp$$reg));
4748   %}
4749   ins_pipe(pipe_slow);
4750 %}
4751 
4752 instruct extractL(iRegLNoSp dst, vReg src, immI idx, vReg vtmp)
4753 %{
4754   predicate(UseSVE > 0 && n->in(2)->get_int() >= 2);
4755   match(Set dst (ExtractL src idx));
4756   effect(TEMP vtmp);
4757   ins_cost(2 * SVE_COST);
4758   format %{ "sve_extract_integral $dst, D, $src, $idx\t# extract from vector(L)" %}
4759   ins_encode %{
4760     __ sve_extract_integral(as_Register($dst$$reg), __ D, as_FloatRegister($src$$reg),
4761                             (int)($idx$$constant), /* is_signed */ false, as_FloatRegister($vtmp$$reg));
4762   %}
4763   ins_pipe(pipe_slow);
4764 %}
4765 
4766 instruct extractB_LT16(iRegINoSp dst, vReg src, immI idx)
4767 %{
4768   predicate(UseSVE > 0 && n->in(2)->get_int() < 16);
4769   match(Set dst (ExtractB src idx));
4770   ins_cost(INSN_COST);
4771   format %{ "smov $dst, B, $src, $idx\t# extract from vector(B)" %}
4772   ins_encode %{
4773     __ smov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ B, $idx$$constant);
4774   %}
4775   ins_pipe(pipe_class_default);
4776 %}
4777 
4778 instruct extractS_LT8(iRegINoSp dst, vReg src, immI idx)
4779 %{
4780   predicate(UseSVE > 0 && n->in(2)->get_int() < 8);
4781   match(Set dst (ExtractS src idx));
4782   ins_cost(INSN_COST);
4783   format %{ "smov $dst, H, $src, $idx\t# extract from vector(S)" %}
4784   ins_encode %{
4785     __ smov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ H, $idx$$constant);
4786   %}
4787   ins_pipe(pipe_class_default);
4788 %}
4789 
4790 instruct extractI_LT4(iRegINoSp dst, vReg src, immI idx)
4791 %{
4792   predicate(UseSVE > 0 && n->in(2)->get_int() < 4);
4793   match(Set dst (ExtractI src idx));
4794   ins_cost(INSN_COST);
4795   format %{ "smov $dst, S, $src, $idx\t# extract from vector(I)" %}
4796   ins_encode %{
4797     __ smov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ S, $idx$$constant);
4798   %}
4799   ins_pipe(pipe_class_default);
4800 %}
4801 
4802 instruct extractL_LT2(iRegLNoSp dst, vReg src, immI idx)
4803 %{
4804   predicate(UseSVE > 0 && n->in(2)->get_int() < 2);
4805   match(Set dst (ExtractL src idx));
4806   ins_cost(INSN_COST);
4807   format %{ "umov $dst, D, $src, $idx\t# extract from vector(L)" %}
4808   ins_encode %{
4809     __ umov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ D, $idx$$constant);
4810   %}
4811   ins_pipe(pipe_class_default);
4812 %}
4813 
4814 instruct extractF(vRegF dst, vReg src, immI idx)
4815 %{
4816   predicate(UseSVE > 0);
4817   match(Set dst (ExtractF src idx));
4818   ins_cost(2 * SVE_COST);
4819   format %{ "sve_extract_f $dst, S, $src, $idx\t# extract from vector(F)" %}
4820   ins_encode %{
4821     if ((as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg)) && ($idx$$constant == 0)) {
4822       /* empty */
4823     } else if ($idx$$constant == 0) {
4824       __ fmovs(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
4825     } else if ($idx$$constant < 4) {
4826       __ ins(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($src$$reg), 0, (int)($idx$$constant));
4827     } else {
4828       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4829       __ sve_ext(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), $idx$$constant << 2);
4830     }
4831   %}
4832   ins_pipe(pipe_slow);
4833 %}
4834 
4835 instruct extractD(vRegD dst, vReg src, immI idx)
4836 %{
4837   predicate(UseSVE > 0);
4838   match(Set dst (ExtractD src idx));
4839   ins_cost(2 * SVE_COST);
4840   format %{ "sve_extract_d $dst, D, $src, $idx\t# extract from vector(D)" %}
4841   ins_encode %{
4842     if ((as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg)) && ($idx$$constant == 0)) {
4843       /* empty */
4844     } else if ($idx$$constant == 0) {
4845       __ fmovd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
4846     } else if ($idx$$constant == 1) {
4847       __ ins(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), 0, 1);
4848     } else {
4849       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4850       __ sve_ext(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), $idx$$constant << 3);
4851     }
4852   %}
4853   ins_pipe(pipe_slow);
4854 %}
4855 
4856 // ------------------------------- VectorTest ----------------------------------
4857 
4858 instruct vtest_alltrue(iRegINoSp dst, pRegGov src1, pRegGov src2, pReg ptmp, rFlagsReg cr)
4859 %{
4860   predicate(UseSVE > 0 &&
4861             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
4862   match(Set dst (VectorTest src1 src2));
4863   effect(TEMP ptmp, KILL cr);
4864   ins_cost(SVE_COST);
4865   format %{ "sve_eors $ptmp, $src1, $src2\t# $src2 is all true mask\n"
4866             "csetw $dst, EQ\t# VectorTest (sve) - alltrue" %}
4867   ins_encode %{
4868     __ sve_eors(as_PRegister($ptmp$$reg), ptrue,
4869                 as_PRegister($src1$$reg), as_PRegister($src2$$reg));
4870     __ csetw(as_Register($dst$$reg), Assembler::EQ);
4871   %}
4872   ins_pipe(pipe_slow);
4873 %}
4874 
4875 instruct vtest_anytrue(iRegINoSp dst, pRegGov src1, pRegGov src2, rFlagsReg cr)
4876 %{
4877   predicate(UseSVE > 0 &&
4878             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
4879   match(Set dst (VectorTest src1 src2));
4880   effect(KILL cr);
4881   ins_cost(SVE_COST);
4882   format %{ "sve_ptest $src1\n\t"
4883             "csetw $dst, NE\t# VectorTest (sve) - anytrue" %}
4884   ins_encode %{
4885     // "src2" is not used for sve.
4886     __ sve_ptest(ptrue, as_PRegister($src1$$reg));
4887     __ csetw(as_Register($dst$$reg), Assembler::NE);
4888   %}
4889   ins_pipe(pipe_slow);
4890 %}
4891 // ------------------------------ Vector insert ---------------------------------
4892 
4893 instruct insertI_le128bits(vReg dst, vReg src, iRegIorL2I val, immI idx) %{
4894   predicate(UseSVE > 0 &&
4895             (Matcher::vector_length_in_bytes(n) == 8 || Matcher::vector_length_in_bytes(n) == 16) &&
4896             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4897              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
4898              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4899   match(Set dst (VectorInsert (Binary src val) idx));
4900   ins_cost(2 * INSN_COST);
4901   format %{ "orr    $dst, T8/16B, $src, $src\n\t"
4902             "mov    $dst, B/H/S, $idx, $val\t# insertI into vector(64/128bits)" %}
4903   ins_encode %{
4904     BasicType bt = Matcher::vector_element_basic_type(this);
4905     if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
4906       __ orr(as_FloatRegister($dst$$reg), Matcher::vector_length_in_bytes(this) == 8 ? __ T8B : __ T16B,
4907              as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4908     }
4909     __ mov(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(Matcher::vector_element_basic_type(this)),
4910            $idx$$constant, $val$$Register);
4911   %}
4912   ins_pipe(pipe_slow);
4913 %}
4914 
4915 instruct insertI_small_index(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg vtmp, pRegGov pgtmp, rFlagsReg cr) %{
4916   predicate(UseSVE > 0 && n->in(2)->get_int() < 32 &&
4917             Matcher::vector_length_in_bytes(n) > 16 &&
4918             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4919              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
4920              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4921   match(Set dst (VectorInsert (Binary src val) idx));
4922   effect(TEMP vtmp, TEMP pgtmp, KILL cr);
4923   ins_cost(4 * SVE_COST);
4924   format %{ "sve_index $vtmp, -16, 1\t# (B/H/S)\n\t"
4925             "sve_cmpeq $pgtmp, $vtmp, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
4926             "sve_orr $dst, $src, $src\n\t"
4927             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
4928   ins_encode %{
4929     BasicType bt = Matcher::vector_element_basic_type(this, $src);
4930     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
4931     __ block_comment("insert into vector (B/H/S) {");
4932       __ sve_index(as_FloatRegister($vtmp$$reg), size, -16, 1);
4933       __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
4934                  as_FloatRegister($vtmp$$reg), (int)($idx$$constant) - 16);
4935       if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
4936         __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4937       }
4938       __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), $val$$Register);
4939     __ block_comment("} insert into vector (B/H/S)");
4940   %}
4941   ins_pipe(pipe_slow);
4942 %}
4943 
4944 instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg vtmp1, vReg vtmp2, pRegGov pgtmp, rFlagsReg cr) %{
4945   predicate(UseSVE > 0 && n->in(2)->get_int() >= 32 &&
4946             Matcher::vector_length_in_bytes(n) > 16 &&
4947             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4948              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
4949              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4950   match(Set dst (VectorInsert (Binary src val) idx));
4951   effect(TEMP vtmp1, TEMP vtmp2, TEMP pgtmp, KILL cr);
4952   ins_cost(5 * SVE_COST);
4953   format %{ "sve_index $vtmp1, 0, 1\t# (B/H/S)\n\t"
4954             "sve_dup $vtmp2, $idx\t# (B/H/S)\n\t"
4955             "sve_cmpeq $pgtmp, $vtmp1, $vtmp2\n\t"
4956             "sve_orr $dst, $src, $src\n\t"
4957             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
4958   ins_encode %{
4959     BasicType bt = Matcher::vector_element_basic_type(this, $src);
4960     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
4961     __ block_comment("insert into vector (B/H/S) {");
4962       __ sve_index(as_FloatRegister($vtmp1$$reg), size, 0, 1);
4963       __ sve_dup(as_FloatRegister($vtmp2$$reg), size, (int)($idx$$constant));
4964       __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
4965                  as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg));
4966       if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
4967         __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4968       }
4969       __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), $val$$Register);
4970     __ block_comment("} insert into vector (B/H/S)");
4971   %}
4972   ins_pipe(pipe_slow);
4973 %}
4974 
4975 instruct insertL_128bits(vReg dst, vReg src, iRegL val, immI idx) %{
4976   predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) == 16 &&
4977             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
4978   match(Set dst (VectorInsert (Binary src val) idx));
4979   ins_cost(2 * INSN_COST);
4980   format %{ "orr    $dst, T16B, $src, $src\n\t"
4981             "mov    $dst, D, $idx, $val\t# insertL into vector(128bits)" %}
4982   ins_encode %{
4983     BasicType bt = Matcher::vector_element_basic_type(this);
4984     if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
4985       __ orr(as_FloatRegister($dst$$reg), __ T16B,
4986              as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4987     }
4988     __ mov(as_FloatRegister($dst$$reg), __ D, $idx$$constant, $val$$Register);
4989   %}
4990   ins_pipe(pipe_slow);
4991 %}
4992 
4993 instruct insertL(vReg dst, vReg src, iRegL val, immI idx, vReg vtmp, pRegGov pgtmp, rFlagsReg cr) %{
4994   predicate(UseSVE > 0 &&
4995             Matcher::vector_length_in_bytes(n) > 16 &&
4996             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
4997   match(Set dst (VectorInsert (Binary src val) idx));
4998   effect(TEMP vtmp, TEMP pgtmp, KILL cr);
4999   ins_cost(4 * SVE_COST);
5000   format %{ "sve_index $vtmp, D, -16, 1\n\t"
5001             "sve_cmpeq $pgtmp, $vtmp, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
5002             "sve_orr $dst, $src, $src\n\t"
5003             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (L)" %}
5004   ins_encode %{
5005     __ block_comment("insert into vector (L) {");
5006       __ sve_index(as_FloatRegister($vtmp$$reg), __ D, -16, 1);
5007       __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
5008                  as_FloatRegister($vtmp$$reg), (int)($idx$$constant) - 16);
5009       if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
5010         __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
5011       }
5012       __ sve_cpy(as_FloatRegister($dst$$reg), __ D,
5013                  as_PRegister($pgtmp$$reg), $val$$Register);
5014     __ block_comment("} insert into vector (L)");
5015   %}
5016   ins_pipe(pipe_slow);
5017 %}
5018 
5019 instruct insertF_le128bits(vReg dst, vReg src, vRegF val, immI idx) %{
5020   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
5021             (Matcher::vector_length_in_bytes(n) == 8 || Matcher::vector_length_in_bytes(n) == 16));
5022   match(Set dst (VectorInsert (Binary src val) idx));
5023   ins_cost(2 * INSN_COST);
5024   effect(TEMP_DEF dst);
5025   format %{ "orr    $dst, T8/16B, $src, $src\n\t"
5026             "ins    $dst, S, $val, $idx, 0\t# insertF into vector(64/128bits)" %}
5027   ins_encode %{
5028     __ orr(as_FloatRegister($dst$$reg), Matcher::vector_length_in_bytes(this) == 8 ? __ T8B : __ T16B,
5029            as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
5030     __ ins(as_FloatRegister($dst$$reg), __ S,
5031            as_FloatRegister($val$$reg), $idx$$constant, 0);
5032   %}
5033   ins_pipe(pipe_slow);
5034 %}
5035 
5036 instruct insertF_small_index(vReg dst, vReg src, vRegF val, immI idx, pRegGov pgtmp, rFlagsReg cr) %{
5037   predicate(UseSVE > 0 && n->in(2)->get_int() < 32 &&
5038             Matcher::vector_length_in_bytes(n) > 16 &&
5039             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
5040   match(Set dst (VectorInsert (Binary src val) idx));
5041   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
5042   ins_cost(4 * SVE_COST);
5043   format %{ "sve_index $dst, S, -16, 1\n\t"
5044             "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
5045             "sve_orr $dst, $src, $src\n\t"
5046             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
5047   ins_encode %{
5048     __ block_comment("insert into vector (F) {");
5049       __ sve_index(as_FloatRegister($dst$$reg), __ S, -16, 1);
5050       __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
5051                  as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
5052       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
5053       __ sve_cpy(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
5054     __ block_comment("} insert into vector (F)");
5055   %}
5056   ins_pipe(pipe_slow);
5057 %}
5058 
5059 instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr) %{
5060   predicate(UseSVE > 0 && n->in(2)->get_int() >= 32 &&
5061             Matcher::vector_length_in_bytes(n) > 16 &&
5062             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
5063   match(Set dst (VectorInsert (Binary src val) idx));
5064   effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
5065   ins_cost(5 * SVE_COST);
5066   format %{ "sve_index $tmp1, S, 0, 1\n\t"
5067             "sve_dup $dst, S, $idx\n\t"
5068             "sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
5069             "sve_orr $dst, $src, $src\n\t"
5070             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
5071   ins_encode %{
5072     __ block_comment("insert into vector (F) {");
5073       __ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
5074       __ sve_dup(as_FloatRegister($dst$$reg), __ S, (int)($idx$$constant));
5075       __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
5076                  as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
5077       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
5078                  as_FloatRegister($src$$reg));
5079       __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
5080                  as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
5081     __ block_comment("} insert into vector (F)");
5082   %}
5083   ins_pipe(pipe_slow);
5084 %}
5085 
5086 instruct insertD_128bits(vReg dst, vReg src, vRegD val, immI idx) %{
5087   predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) == 16 &&
5088             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
5089   match(Set dst (VectorInsert (Binary src val) idx));
5090   ins_cost(2 * INSN_COST);
5091   effect(TEMP_DEF dst);
5092   format %{ "orr    $dst, T16B, $src, $src\n\t"
5093             "ins    $dst, D, $val, $idx, 0\t# insertD into vector(128bits)" %}
5094   ins_encode %{
5095     __ orr(as_FloatRegister($dst$$reg), __ T16B,
5096            as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
5097     __ ins(as_FloatRegister($dst$$reg), __ D,
5098            as_FloatRegister($val$$reg), $idx$$constant, 0);
5099   %}
5100   ins_pipe(pipe_slow);
5101 %}
5102 
5103 instruct insertD(vReg dst, vReg src, vRegD val, immI idx, pRegGov pgtmp, rFlagsReg cr) %{
5104   predicate(UseSVE > 0 &&
5105             Matcher::vector_length_in_bytes(n) > 16 &&
5106             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
5107   match(Set dst (VectorInsert (Binary src val) idx));
5108   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
5109   ins_cost(4 * SVE_COST);
5110   format %{ "sve_index $dst, D, -16, 1\n\t"
5111             "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
5112             "sve_orr $dst, $src, $src\n\t"
5113             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (D)" %}
5114   ins_encode %{
5115     __ block_comment("insert into vector (D) {");
5116       __ sve_index(as_FloatRegister($dst$$reg), __ D, -16, 1);
5117       __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
5118                  as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
5119       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
5120                  as_FloatRegister($src$$reg));
5121       __ sve_cpy(as_FloatRegister($dst$$reg), __ D,
5122                  as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
5123     __ block_comment("} insert into vector (D)");
5124   %}
5125   ins_pipe(pipe_slow);
5126 %}
5127 
5128 // ------------------------------ Vector shuffle -------------------------------
5129 
5130 instruct loadshuffle(vReg dst, vReg src) %{
5131   predicate(UseSVE > 0);
5132   match(Set dst (VectorLoadShuffle src));
5133   ins_cost(SVE_COST);
5134   format %{ "sve_loadshuffle $dst, $src\t# vector load shuffle (B/H/S/D)" %}
5135   ins_encode %{
5136     BasicType bt = Matcher::vector_element_basic_type(this);
5137     if (bt == T_BYTE) {
5138       if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
5139         __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
5140                    as_FloatRegister($src$$reg));
5141       }
5142     } else {
5143       __ sve_vector_extend(as_FloatRegister($dst$$reg),  __ elemType_to_regVariant(bt),
5144                            as_FloatRegister($src$$reg), __ B);
5145     }
5146   %}
5147   ins_pipe(pipe_slow);
5148 %}
5149 
5150 // ------------------------------ Vector rearrange -------------------------------
5151 
5152 instruct rearrange(vReg dst, vReg src, vReg shuffle)
5153 %{
5154   predicate(UseSVE > 0);
5155   match(Set dst (VectorRearrange src shuffle));
5156   ins_cost(SVE_COST);
5157   format %{ "sve_tbl $dst, $src, $shuffle\t# vector rearrange" %}
5158   ins_encode %{
5159     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5160     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5161     __ sve_tbl(as_FloatRegister($dst$$reg), size,
5162                as_FloatRegister($src$$reg), as_FloatRegister($shuffle$$reg));
5163   %}
5164   ins_pipe(pipe_slow);
5165 %}
5166 
5167 // ------------------------------ Vector Load Gather ---------------------------------
5168 
5169 instruct gatherI(vReg dst, indirect mem, vReg idx) %{
5170   predicate(UseSVE > 0 &&
5171             n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
5172             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5173              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5174   match(Set dst (LoadVectorGather mem idx));
5175   ins_cost(SVE_COST);
5176   format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (S)" %}
5177   ins_encode %{
5178     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), ptrue,
5179                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5180   %}
5181   ins_pipe(pipe_slow);
5182 %}
5183 
5184 instruct gatherL(vReg dst, indirect mem, vReg idx) %{
5185   predicate(UseSVE > 0 &&
5186             n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
5187             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5188              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5189   match(Set dst (LoadVectorGather mem idx));
5190   ins_cost(2 * SVE_COST);
5191   format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (D)" %}
5192   ins_encode %{
5193     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5194     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), ptrue, as_Register($mem$$base),
5195                        as_FloatRegister($idx$$reg));
5196   %}
5197   ins_pipe(pipe_slow);
5198 %}
5199 
5200 // ------------------------------ Vector Load Gather Partial-------------------------------
5201 
5202 instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5203   predicate(UseSVE > 0 &&
5204             n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
5205             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5206              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5207   match(Set dst (LoadVectorGather mem idx));
5208   effect(TEMP ptmp, KILL cr);
5209   ins_cost(2 * SVE_COST + INSN_COST);
5210   format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (S)" %}
5211   ins_encode %{
5212     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this));
5213     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
5214                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5215   %}
5216   ins_pipe(pipe_slow);
5217 %}
5218 
5219 instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5220   predicate(UseSVE > 0 &&
5221             n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
5222             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5223              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5224   match(Set dst (LoadVectorGather mem idx));
5225   effect(TEMP ptmp, KILL cr);
5226   ins_cost(3 * SVE_COST + INSN_COST);
5227   format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (D)" %}
5228   ins_encode %{
5229     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this));
5230     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5231     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
5232                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5233   %}
5234   ins_pipe(pipe_slow);
5235 %}
5236 
5237 // ------------------------------ Vector Load Gather Predicated -------------------------------
5238 
5239 instruct gatherI_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
5240   predicate(UseSVE > 0 &&
5241             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5242              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5243   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
5244   ins_cost(SVE_COST);
5245   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (S)" %}
5246   ins_encode %{
5247     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
5248                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5249   %}
5250   ins_pipe(pipe_slow);
5251 %}
5252 
5253 instruct gatherL_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
5254   predicate(UseSVE > 0 &&
5255             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5256              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5257   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
5258   ins_cost(2 * SVE_COST);
5259   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (D)" %}
5260   ins_encode %{
5261     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5262     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
5263                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5264   %}
5265   ins_pipe(pipe_slow);
5266 %}
5267 
5268 // ------------------------------ Vector Store Scatter -------------------------------
5269 
5270 instruct scatterI(indirect mem, vReg src, vReg idx) %{
5271   predicate(UseSVE > 0 &&
5272             n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
5273             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5274              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5275   match(Set mem (StoreVectorScatter mem (Binary src idx)));
5276   ins_cost(SVE_COST);
5277   format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (S)" %}
5278   ins_encode %{
5279     __ sve_st1w_scatter(as_FloatRegister($src$$reg), ptrue,
5280                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5281   %}
5282   ins_pipe(pipe_slow);
5283 %}
5284 
5285 instruct scatterL(indirect mem, vReg src, vReg idx) %{
5286   predicate(UseSVE > 0 &&
5287             n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
5288             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5289              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5290   match(Set mem (StoreVectorScatter mem (Binary src idx)));
5291   ins_cost(2 * SVE_COST);
5292   format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (D)" %}
5293   ins_encode %{
5294     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5295     __ sve_st1d_scatter(as_FloatRegister($src$$reg), ptrue,
5296                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5297   %}
5298   ins_pipe(pipe_slow);
5299 %}
5300 
5301 // ------------------------------ Vector Store Scatter Partial -------------------------------
5302 
5303 instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5304   predicate(UseSVE > 0 &&
5305             n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
5306             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5307              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5308   match(Set mem (StoreVectorScatter mem (Binary src idx)));
5309   effect(TEMP ptmp, KILL cr);
5310   ins_cost(2 * SVE_COST + INSN_COST);
5311   format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (S)" %}
5312   ins_encode %{
5313     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src));
5314     __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
5315                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5316   %}
5317   ins_pipe(pipe_slow);
5318 %}
5319 
5320 instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5321   predicate(UseSVE > 0 &&
5322             n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
5323             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5324              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5325   match(Set mem (StoreVectorScatter mem (Binary src idx)));
5326   effect(TEMP ptmp, KILL cr);
5327   ins_cost(3 * SVE_COST + INSN_COST);
5328   format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (D)" %}
5329   ins_encode %{
5330     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src));
5331     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5332     __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
5333                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5334   %}
5335   ins_pipe(pipe_slow);
5336 %}
5337 
5338 // ------------------------------ Vector Store Scatter Predicated -------------------------------
5339 
5340 instruct scatterI_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
5341   predicate(UseSVE > 0 &&
5342             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5343              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5344   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
5345   ins_cost(SVE_COST);
5346   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicate (S)" %}
5347   ins_encode %{
5348     __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
5349                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5350   %}
5351   ins_pipe(pipe_slow);
5352 %}
5353 
5354 instruct scatterL_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
5355   predicate(UseSVE > 0 &&
5356             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5357              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5358   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
5359   ins_cost(2 * SVE_COST);
5360   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated (D)" %}
5361   ins_encode %{
5362     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5363     __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
5364                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5365   %}
5366   ins_pipe(pipe_slow);
5367 %}
5368 
5369 // ------------------------------ Vector Load Const -------------------------------
5370 
5371 instruct loadconB(vReg dst, immI0 src) %{
5372   predicate(UseSVE > 0 &&
5373             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
5374   match(Set dst (VectorLoadConst src));
5375   ins_cost(SVE_COST);
5376   format %{ "sve_index $dst, 0, 1\t# generate iota indices" %}
5377   ins_encode %{
5378     __ sve_index(as_FloatRegister($dst$$reg), __ B, 0, 1);
5379   %}
5380   ins_pipe(pipe_slow);
5381 %}
5382 
5383 // Intrisics for String.indexOf(char)
5384 
5385 
5386 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
5387                                   iRegI_R0 result, vReg ztmp1, vReg ztmp2,
5388                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
5389 %{
5390   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
5391   predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
5392   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
5393 
5394   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
5395 
5396   ins_encode %{
5397     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
5398                                as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
5399                                as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), true /* isL */);
5400   %}
5401   ins_pipe(pipe_class_memory);
5402 %}
5403 
5404 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
5405                                   iRegI_R0 result, vReg ztmp1, vReg ztmp2,
5406                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
5407 %{
5408   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
5409   predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
5410   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
5411 
5412   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
5413 
5414   ins_encode %{
5415     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
5416                                as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
5417                                as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), false /* isL */);
5418   %}
5419   ins_pipe(pipe_class_memory);
5420 %}
5421 
5422 // ---------------------------- Vector mask reductions ---------------------------
5423 instruct vmask_truecount(iRegINoSp dst, pReg src) %{
5424   predicate(UseSVE > 0);
5425   match(Set dst (VectorMaskTrueCount src));
5426   ins_cost(SVE_COST);
5427   format %{ "vmask_truecount $dst, $src\t# vector mask truecount (sve)" %}
5428   ins_encode %{
5429     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5430     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5431     __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($src$$reg));
5432   %}
5433   ins_pipe(pipe_slow);
5434 %}
5435 
5436 // Return the index of the first mask lane that is set, or vector length if none of
5437 // them are set.
5438 instruct vmask_firsttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
5439   predicate(UseSVE > 0);
5440   match(Set dst (VectorMaskFirstTrue src));
5441   effect(TEMP ptmp);
5442   ins_cost(3 * SVE_COST);
5443   format %{ "vmask_firsttrue $dst, $src\t# vector mask firsttrue (sve)" %}
5444   ins_encode %{
5445     uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
5446     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5447     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5448     // When the input predicate is all-false, the result should be the vector length
5449     // instead of max vector register size.
5450     if (length_in_bytes == MaxVectorSize) {
5451       __ sve_brkb(as_PRegister($ptmp$$reg), ptrue, as_PRegister($src$$reg), false);
5452     } else {
5453       __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
5454       __ sve_brkb(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg), as_PRegister($src$$reg), false);
5455     }
5456     __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($ptmp$$reg));
5457   %}
5458   ins_pipe(pipe_slow);
5459 %}
5460 
5461 instruct vmask_lasttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
5462   predicate(UseSVE > 0);
5463   match(Set dst (VectorMaskLastTrue src));
5464   effect(TEMP ptmp);
5465   ins_cost(3 * SVE_COST);
5466   format %{ "vmask_lasttrue $dst, $src\t# vector mask lasttrue (sve)" %}
5467   ins_encode %{
5468     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5469     __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($src$$reg), as_PRegister($ptmp$$reg));
5470   %}
5471   ins_pipe(pipe_slow);
5472 %}
5473 
5474 instruct vmask_tolong(iRegLNoSp dst, pReg src, vReg vtmp1, vReg vtmp2) %{
5475   predicate(UseSVE > 0 &&
5476             n->in(1)->bottom_type()->is_vect()->length() <= 64);
5477   match(Set dst (VectorMaskToLong src));
5478   effect(TEMP vtmp1, TEMP vtmp2);
5479   ins_cost(13 * SVE_COST);
5480   format %{ "vmask_tolong $dst, $src\t# vector mask tolong (sve)" %}
5481   ins_encode %{
5482     __ sve_vmask_tolong(as_Register($dst$$reg), as_PRegister($src$$reg),
5483                         Matcher::vector_element_basic_type(this, $src),
5484                         Matcher::vector_length(this, $src),
5485                         as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg));
5486   %}
5487   ins_pipe(pipe_slow);
5488 %}
5489 
5490 // ---------------------------- Vector mask generation ---------------------------
5491 // The rules below set predicate registers. They can guarantee the high bits of dst
5492 // are cleared with zero when the vector length is less than the full size of
5493 // hardware vector register width.
5494 
5495 
5496 // maskAll (full or partial predicate size)
5497 
5498 instruct vmaskAll_immI(pRegGov dst, immI src) %{
5499   predicate(UseSVE > 0);
5500   match(Set dst (MaskAll src));
5501   ins_cost(SVE_COST);
5502   format %{ "sve_ptrue_lanecnt/sve_pfalse $dst\t# mask all (sve) (B/H/S)" %}
5503   ins_encode %{
5504     int con = (int)$src$$constant;
5505     if (con == 0) {
5506       __ sve_pfalse(as_PRegister($dst$$reg));
5507     } else {
5508       assert(con == -1, "invalid constant value for mask");
5509       BasicType bt = Matcher::vector_element_basic_type(this);
5510       __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
5511                            Matcher::vector_length(this));
5512     }
5513   %}
5514   ins_pipe(pipe_slow);
5515 %}
5516 
5517 instruct vmaskAllI(pRegGov dst, iRegIorL2I src, vReg tmp, rFlagsReg cr) %{
5518   predicate(UseSVE > 0);
5519   match(Set dst (MaskAll src));
5520   effect(TEMP tmp, KILL cr);
5521   ins_cost(3 * SVE_COST);
5522   format %{ "sve_dup $tmp, $src\n\t"
5523             "sve_ptrue $dst, vector_length\n\t"
5524             "sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) (B/H/S)" %}
5525   ins_encode %{
5526     BasicType bt = Matcher::vector_element_basic_type(this);
5527     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5528     uint length_in_bytes = Matcher::vector_length_in_bytes(this);
5529     __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
5530     if (length_in_bytes < MaxVectorSize) {
5531       __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), size, Matcher::vector_length(this));
5532       __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size,
5533                  as_PRegister($dst$$reg), as_FloatRegister($tmp$$reg), 0);
5534     } else {
5535       __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
5536     }
5537   %}
5538   ins_pipe(pipe_slow);
5539 %}
5540 
5541 instruct vmaskAll_immL(pRegGov dst, immL src) %{
5542   predicate(UseSVE > 0);
5543   match(Set dst (MaskAll src));
5544   ins_cost(SVE_COST);
5545   format %{ "sve_ptrue_lanecnt/sve_pfalse $dst\t# mask all (sve) (D)" %}
5546   ins_encode %{
5547     long con = (long)$src$$constant;
5548     if (con == 0) {
5549       __ sve_pfalse(as_PRegister($dst$$reg));
5550     } else {
5551       assert(con == -1, "invalid constant value for mask");
5552       BasicType bt = Matcher::vector_element_basic_type(this);
5553       __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
5554                            Matcher::vector_length(this));
5555     }
5556   %}
5557   ins_pipe(pipe_slow);
5558 %}
5559 
5560 instruct vmaskAllL(pRegGov dst, iRegL src, vReg tmp, rFlagsReg cr) %{
5561   predicate(UseSVE > 0);
5562   match(Set dst (MaskAll src));
5563   effect(TEMP tmp, KILL cr);
5564   ins_cost(3 * SVE_COST);
5565   format %{ "sve_dup $tmp, $src\n\t"
5566             "sve_ptrue $dst, vector_length\n\t"
5567             "sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) (D)" %}
5568   ins_encode %{
5569     BasicType bt = Matcher::vector_element_basic_type(this);
5570     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5571     uint length_in_bytes = Matcher::vector_length_in_bytes(this);
5572     __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
5573     if (length_in_bytes < MaxVectorSize) {
5574       __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), size, Matcher::vector_length(this));
5575       __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size,
5576                  as_PRegister($dst$$reg), as_FloatRegister($tmp$$reg), 0);
5577     } else {
5578       __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
5579     }
5580   %}
5581   ins_pipe(pipe_slow);
5582 %}
5583 
5584 // vector mask compare
5585 
5586 instruct vmaskcmp(pRegGov dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %{
5587   predicate(UseSVE > 0);
5588   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
5589   effect(KILL cr);
5590   ins_cost(2 * SVE_COST);
5591   format %{ "sve_cmp $dst, $src1, $src2\t# vector mask cmp (sve)" %}
5592   ins_encode %{
5593     uint length_in_bytes = Matcher::vector_length_in_bytes(this);
5594     BasicType bt = Matcher::vector_element_basic_type(this);
5595     if (length_in_bytes == MaxVectorSize) {
5596       __ sve_compare(as_PRegister($dst$$reg), bt, ptrue, as_FloatRegister($src1$$reg),
5597                      as_FloatRegister($src2$$reg), (int)$cond$$constant);
5598     } else {
5599       __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
5600                            Matcher::vector_length(this));
5601       __ sve_compare(as_PRegister($dst$$reg), bt, as_PRegister($dst$$reg), as_FloatRegister($src1$$reg),
5602                      as_FloatRegister($src2$$reg), (int)$cond$$constant);
5603     }
5604   %}
5605   ins_pipe(pipe_slow);
5606 %}
5607 
5608 instruct vmaskcmp_masked(pRegGov dst, vReg src1, vReg src2, immI cond, pRegGov pg, rFlagsReg cr) %{
5609   predicate(UseSVE > 0);
5610   match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond pg)));
5611   effect(KILL cr);
5612   ins_cost(SVE_COST);
5613   format %{ "sve_cmp $dst, $pg, $src1, $src2\t# vector mask cmp (sve)" %}
5614   ins_encode %{
5615     BasicType bt = Matcher::vector_element_basic_type(this);
5616     __ sve_compare(as_PRegister($dst$$reg), bt, as_PRegister($pg$$reg), as_FloatRegister($src1$$reg),
5617                    as_FloatRegister($src2$$reg), (int)$cond$$constant);
5618   %}
5619   ins_pipe(pipe_slow);
5620 %}
5621 
5622 // vector load mask
5623 
5624 instruct vloadmaskB(pRegGov dst, vReg src, rFlagsReg cr) %{
5625   predicate(UseSVE > 0 &&
5626             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
5627   match(Set dst (VectorLoadMask src));
5628   effect(KILL cr);
5629   ins_cost(SVE_COST);
5630   format %{ "vloadmaskB $dst, $src\t# vector load mask (sve) (B)" %}
5631   ins_encode %{
5632     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ B,
5633                ptrue, as_FloatRegister($src$$reg), 0);
5634   %}
5635   ins_pipe(pipe_slow);
5636 %}
5637 
5638 instruct vloadmask_extend(pRegGov dst, vReg src, vReg tmp, rFlagsReg cr) %{
5639   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() != T_BYTE);
5640   match(Set dst (VectorLoadMask src));
5641   effect(TEMP tmp, KILL cr);
5642   ins_cost(3 * SVE_COST);
5643   format %{ "vloadmask $dst, $src\t# vector load mask (sve) (H/S/D)" %}
5644   ins_encode %{
5645     BasicType bt = Matcher::vector_element_basic_type(this);
5646     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5647     __ sve_vector_extend(as_FloatRegister($tmp$$reg), size, as_FloatRegister($src$$reg), __ B);
5648     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
5649   %}
5650   ins_pipe(pipe_slow);
5651 %}
5652 
5653 instruct vmask_gen(pRegGov pg, iRegL len, rFlagsReg cr) %{
5654   predicate(UseSVE > 0);
5655   match(Set pg (VectorMaskGen len));
5656   effect(KILL cr);
5657   ins_cost(SVE_COST);
5658   format %{ "sve_whilelo $pg, zr, $len\t # sve" %}
5659   ins_encode %{
5660     BasicType bt = Matcher::vector_element_basic_type(this);
5661     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5662     __ sve_whilelo(as_PRegister($pg$$reg), size, zr, as_Register($len$$reg));
5663   %}
5664   ins_pipe(pipe_slow);
5665 %}