1 //
   2 // Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
   3 // Copyright (c) 2020, 2022, Arm Limited. All rights reserved.
   4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5 //
   6 // This code is free software; you can redistribute it and/or modify it
   7 // under the terms of the GNU General Public License version 2 only, as
   8 // published by the Free Software Foundation.
   9 //
  10 // This code is distributed in the hope that it will be useful, but WITHOUT
  11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 // version 2 for more details (a copy is included in the LICENSE file that
  14 // accompanied this code).
  15 //
  16 // You should have received a copy of the GNU General Public License version
  17 // 2 along with this work; if not, write to the Free Software Foundation,
  18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21 // or visit www.oracle.com if you need additional information or have any
  22 // questions.
  23 //
  24 //
  25 
  26 // This file is automatically generated by running "m4 aarch64_sve_ad.m4". Do not edit ----
  27 
  28 // AArch64 SVE Architecture Description File
  29 
  30 
  31 // 4 bit signed offset -- for predicated load/store
  32 
  33 operand vmemA_immIOffset4()
  34 %{
  35   // (esize / msize) = 1
  36   predicate(Address::offset_ok_for_sve_immed(n->get_int(), 4,
  37             Matcher::scalable_vector_reg_size(T_BYTE)));
  38   match(ConI);
  39 
  40   op_cost(0);
  41   format %{ %}
  42   interface(CONST_INTER);
  43 %}
  44 
  45 operand vmemA_immLOffset4()
  46 %{
  47   // (esize / msize) = 1
  48   predicate(Address::offset_ok_for_sve_immed(n->get_long(), 4,
  49             Matcher::scalable_vector_reg_size(T_BYTE)));
  50   match(ConL);
  51 
  52   op_cost(0);
  53   format %{ %}
  54   interface(CONST_INTER);
  55 %}
  56 
  57 operand vmemA_indOffI4(iRegP reg, vmemA_immIOffset4 off)
  58 %{
  59   constraint(ALLOC_IN_RC(ptr_reg));
  60   match(AddP reg off);
  61   op_cost(0);
  62   format %{ "[$reg, $off]" %}
  63   interface(MEMORY_INTER) %{
  64     base($reg);
  65     index(0xffffffff);
  66     scale(0x0);
  67     disp($off);
  68   %}
  69 %}
  70 
  71 operand vmemA_indOffL4(iRegP reg, vmemA_immLOffset4 off)
  72 %{
  73   constraint(ALLOC_IN_RC(ptr_reg));
  74   match(AddP reg off);
  75   op_cost(0);
  76   format %{ "[$reg, $off]" %}
  77   interface(MEMORY_INTER) %{
  78     base($reg);
  79     index(0xffffffff);
  80     scale(0x0);
  81     disp($off);
  82   %}
  83 %}
  84 
  85 // The indOff of vmemA is valid only when the vector element (load to/store from)
  86 // size equals to memory element (load from/store to) size.
  87 opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);
  88 
  89 source_hpp %{
  90   bool op_sve_supported(int opcode, int vlen, BasicType bt);
  91   bool masked_op_sve_supported(int opcode, int vlen, BasicType bt);
  92 %}
  93 
  94 source %{
  95 
  96   typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
  97                                                              PRegister Pg, const Address &adr);
  98 
  99   // Predicated load/store, with optional ptrue to all elements of given predicate register.
 100   static void loadStoreA_predicated(C2_MacroAssembler masm, bool is_store, FloatRegister reg,
 101                                     PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
 102                                     int opcode, Register base, int index, int size, int disp) {
 103     sve_mem_insn_predicate insn;
 104     int mesize = type2aelembytes(mem_elem_bt);
 105     if (index == -1) {
 106       assert(size == 0, "unsupported address mode: scale size = %d", size);
 107       switch(mesize) {
 108       case 1:
 109         insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
 110         break;
 111       case 2:
 112         insn = is_store ? &C2_MacroAssembler::sve_st1h : &C2_MacroAssembler::sve_ld1h;
 113         break;
 114       case 4:
 115         insn = is_store ? &C2_MacroAssembler::sve_st1w : &C2_MacroAssembler::sve_ld1w;
 116         break;
 117       case 8:
 118         insn = is_store ? &C2_MacroAssembler::sve_st1d : &C2_MacroAssembler::sve_ld1d;
 119         break;
 120       default:
 121         assert(false, "unsupported");
 122         ShouldNotReachHere();
 123       }
 124       int imm4 = disp / mesize / Matcher::scalable_vector_reg_size(vector_elem_bt);
 125       (masm.*insn)(reg, Assembler::elemType_to_regVariant(vector_elem_bt), pg, Address(base, imm4));
 126     } else {
 127       assert(false, "unimplemented");
 128       ShouldNotReachHere();
 129     }
 130   }
 131 
 132   bool op_sve_supported(int opcode, int vlen, BasicType bt) {
 133     int length_in_bytes = vlen * type2aelembytes(bt);
 134     switch (opcode) {
 135       case Op_MulAddVS2VI:
 136       // No multiply reduction instructions
 137       case Op_MulReductionVD:
 138       case Op_MulReductionVF:
 139       case Op_MulReductionVI:
 140       case Op_MulReductionVL:
 141       // Others
 142       case Op_ExtractC:
 143       case Op_ExtractUB:
 144         return false;
 145       // Vector API specific
 146       case Op_VectorLoadShuffle:
 147       case Op_VectorRearrange:
 148         return vlen >= 4 && length_in_bytes <= MaxVectorSize;
 149       case Op_LoadVector:
 150       case Op_StoreVector:
 151         return Matcher::vector_size_supported(bt, vlen);
 152       case Op_ExpandV:
 153         if (UseSVE < 2 || is_subword_type(bt)) return false;
 154       default:
 155         break;
 156     }
 157     // By default, we only support vector operations with no less than 8 bytes and 2 elements.
 158     return 8 <= length_in_bytes && length_in_bytes <= MaxVectorSize && vlen >= 2;
 159   }
 160 
 161   bool masked_op_sve_supported(int opcode, int vlen, BasicType bt) {
 162     if (opcode == Op_VectorRearrange) {
 163       return false;
 164     }
 165     return op_sve_supported(opcode, vlen, bt);
 166   }
 167 %}
 168 
 169 definitions %{
 170   int_def SVE_COST             (200, 200);
 171 %}
 172 
 173 
 174 // All SVE instructions
 175 
 176 // vector load/store
 177 
 178 // Unpredicated vector load/store
 179 instruct loadV(vReg dst, vmemA mem) %{
 180   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() >= 16 &&
 181             n->as_LoadVector()->memory_size() == MaxVectorSize);
 182   match(Set dst (LoadVector mem));
 183   ins_cost(4 * SVE_COST);
 184   format %{ "sve_ldr $dst, $mem\t# vector (sve)" %}
 185   ins_encode %{
 186     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 187     BasicType bt = Matcher::vector_element_basic_type(this);
 188     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg, ptrue,
 189                           bt, bt, $mem->opcode(),
 190                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 191   %}
 192   ins_pipe(pipe_slow);
 193 %}
 194 
 195 instruct storeV(vReg src, vmemA mem) %{
 196   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() >= 16 &&
 197             n->as_StoreVector()->memory_size() == MaxVectorSize);
 198   match(Set mem (StoreVector mem src));
 199   ins_cost(4 * SVE_COST);
 200   format %{ "sve_str $mem, $src\t# vector (sve)" %}
 201   ins_encode %{
 202     FloatRegister src_reg = as_FloatRegister($src$$reg);
 203     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 204     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg, ptrue,
 205                           bt, bt, $mem->opcode(),
 206                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 207   %}
 208   ins_pipe(pipe_slow);
 209 %}
 210 
 211 // Load Vector (16 bits)
 212 instruct loadV2_vreg(vReg dst, vmem2 mem)
 213 %{
 214   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 2);
 215   match(Set dst (LoadVector mem));
 216   ins_cost(4 * INSN_COST);
 217   format %{ "ldrh   $dst,$mem\t# vector (16 bits)" %}
 218   ins_encode( aarch64_enc_ldrvH(dst, mem) );
 219   ins_pipe(vload_reg_mem64);
 220 %}
 221 
 222 // Store Vector (16 bits)
 223 instruct storeV2_vreg(vReg src, vmem2 mem)
 224 %{
 225   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 2);
 226   match(Set mem (StoreVector mem src));
 227   ins_cost(4 * INSN_COST);
 228   format %{ "strh   $mem,$src\t# vector (16 bits)" %}
 229   ins_encode( aarch64_enc_strvH(src, mem) );
 230   ins_pipe(vstore_reg_mem64);
 231 %}
 232 
 233 // Load Vector (32 bits)
 234 instruct loadV4_vreg(vReg dst, vmem4 mem)
 235 %{
 236   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 4);
 237   match(Set dst (LoadVector mem));
 238   ins_cost(4 * INSN_COST);
 239   format %{ "ldrs   $dst,$mem\t# vector (32 bits)" %}
 240   ins_encode( aarch64_enc_ldrvS(dst, mem) );
 241   ins_pipe(vload_reg_mem64);
 242 %}
 243 
 244 // Store Vector (32 bits)
 245 instruct storeV4_vreg(vReg src, vmem4 mem)
 246 %{
 247   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 4);
 248   match(Set mem (StoreVector mem src));
 249   ins_cost(4 * INSN_COST);
 250   format %{ "strs   $mem,$src\t# vector (32 bits)" %}
 251   ins_encode( aarch64_enc_strvS(src, mem) );
 252   ins_pipe(vstore_reg_mem64);
 253 %}
 254 
 255 // Load Vector (64 bits)
 256 instruct loadV8_vreg(vReg dst, vmem8 mem)
 257 %{
 258   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 8);
 259   match(Set dst (LoadVector mem));
 260   ins_cost(4 * INSN_COST);
 261   format %{ "ldrd   $dst,$mem\t# vector (64 bits)" %}
 262   ins_encode( aarch64_enc_ldrvD(dst, mem) );
 263   ins_pipe(vload_reg_mem64);
 264 %}
 265 
 266 // Store Vector (64 bits)
 267 instruct storeV8_vreg(vReg src, vmem8 mem)
 268 %{
 269   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 8);
 270   match(Set mem (StoreVector mem src));
 271   ins_cost(4 * INSN_COST);
 272   format %{ "strd   $mem,$src\t# vector (64 bits)" %}
 273   ins_encode( aarch64_enc_strvD(src, mem) );
 274   ins_pipe(vstore_reg_mem64);
 275 %}
 276 
 277 // Load Vector (128 bits)
 278 instruct loadV16_vreg(vReg dst, vmem16 mem)
 279 %{
 280   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 16);
 281   match(Set dst (LoadVector mem));
 282   ins_cost(4 * INSN_COST);
 283   format %{ "ldrq   $dst,$mem\t# vector (128 bits)" %}
 284   ins_encode( aarch64_enc_ldrvQ(dst, mem) );
 285   ins_pipe(vload_reg_mem128);
 286 %}
 287 
 288 // Store Vector (128 bits)
 289 instruct storeV16_vreg(vReg src, vmem16 mem)
 290 %{
 291   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 16);
 292   match(Set mem (StoreVector mem src));
 293   ins_cost(4 * INSN_COST);
 294   format %{ "strq   $mem,$src\t# vector (128 bits)" %}
 295   ins_encode( aarch64_enc_strvQ(src, mem) );
 296   ins_pipe(vstore_reg_mem128);
 297 %}
 298 
 299 // Predicated vector load/store, based on the vector length of the node.
 300 // Only load/store values in the range of the memory_size. This is needed
 301 // when the memory_size is lower than the hardware supported max vector size.
 302 // And this might happen for Vector API mask vector load/store.
 303 instruct loadV_partial(vReg dst, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
 304   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() > 16 &&
 305             n->as_LoadVector()->memory_size() < MaxVectorSize);
 306   match(Set dst (LoadVector mem));
 307   effect(TEMP pgtmp, KILL cr);
 308   ins_cost(6 * SVE_COST);
 309   format %{ "sve_ptrue $pgtmp, vector_length\n\t"
 310             "sve_ldr $dst, $pgtmp, $mem\t# load vector partial" %}
 311   ins_encode %{
 312     BasicType bt = Matcher::vector_element_basic_type(this);
 313     __ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 314                          Matcher::vector_length(this));
 315     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 316     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg,
 317                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 318                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 319   %}
 320   ins_pipe(pipe_slow);
 321 %}
 322 
 323 instruct storeV_partial(vReg src, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
 324   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() > 16 &&
 325             n->as_StoreVector()->memory_size() < MaxVectorSize);
 326   match(Set mem (StoreVector mem src));
 327   effect(TEMP pgtmp, KILL cr);
 328   ins_cost(5 * SVE_COST);
 329   format %{ "sve_ptrue $pgtmp, vector_length\n\t"
 330             "sve_str $src, $pgtmp, $mem\t# store vector partial" %}
 331   ins_encode %{
 332     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 333     __ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 334                          Matcher::vector_length(this, $src));
 335     FloatRegister src_reg = as_FloatRegister($src$$reg);
 336     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg,
 337                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 338                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 339   %}
 340   ins_pipe(pipe_slow);
 341 %}
 342 
 343 // vector load/store - predicated
 344 
 345 instruct loadV_masked(vReg dst, vmemA mem, pRegGov pg) %{
 346   predicate(UseSVE > 0);
 347   match(Set dst (LoadVectorMasked mem pg));
 348   ins_cost(4 * SVE_COST);
 349   format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated (sve)" %}
 350   ins_encode %{
 351     BasicType bt = Matcher::vector_element_basic_type(this);
 352     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
 353                           as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
 354                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 355   %}
 356   ins_pipe(pipe_slow);
 357 %}
 358 
 359 instruct storeV_masked(vReg src, vmemA mem, pRegGov pg) %{
 360   predicate(UseSVE > 0);
 361   match(Set mem (StoreVectorMasked mem (Binary src pg)));
 362   ins_cost(4 * SVE_COST);
 363   format %{ "sve_str $mem, $pg, $src\t# store vector predicated (sve)" %}
 364   ins_encode %{
 365     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 366     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
 367                           as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
 368                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 369   %}
 370   ins_pipe(pipe_slow);
 371 %}
 372 
 373 // mask logical and/or/xor
 374 
 375 instruct vmask_and(pRegGov pd, pRegGov pn, pRegGov pm) %{
 376   predicate(UseSVE > 0);
 377   match(Set pd (AndVMask pn pm));
 378   ins_cost(SVE_COST);
 379   format %{ "sve_and $pd, $pn, $pm\t# predicate (sve)" %}
 380   ins_encode %{
 381     __ sve_and(as_PRegister($pd$$reg), ptrue,
 382                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 383   %}
 384   ins_pipe(pipe_slow);
 385 %}
 386 
 387 instruct vmask_or(pRegGov pd, pRegGov pn, pRegGov pm) %{
 388   predicate(UseSVE > 0);
 389   match(Set pd (OrVMask pn pm));
 390   ins_cost(SVE_COST);
 391   format %{ "sve_orr $pd, $pn, $pm\t# predicate (sve)" %}
 392   ins_encode %{
 393     __ sve_orr(as_PRegister($pd$$reg), ptrue,
 394                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 395   %}
 396   ins_pipe(pipe_slow);
 397 %}
 398 
 399 instruct vmask_xor(pRegGov pd, pRegGov pn, pRegGov pm) %{
 400   predicate(UseSVE > 0);
 401   match(Set pd (XorVMask pn pm));
 402   ins_cost(SVE_COST);
 403   format %{ "sve_eor $pd, $pn, $pm\t# predicate (sve)" %}
 404   ins_encode %{
 405     __ sve_eor(as_PRegister($pd$$reg), ptrue,
 406                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 407   %}
 408   ins_pipe(pipe_slow);
 409 %}
 410 
 411 // mask logical and_not
 412 
 413 instruct vmask_and_notI(pRegGov pd, pRegGov pn, pRegGov pm, immI_M1 m1) %{
 414   predicate(UseSVE > 0);
 415   match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
 416   ins_cost(SVE_COST);
 417   format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) (B/H/S)" %}
 418   ins_encode %{
 419     __ sve_bic(as_PRegister($pd$$reg), ptrue,
 420                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 421   %}
 422   ins_pipe(pipe_slow);
 423 %}
 424 
 425 instruct vmask_and_notL(pRegGov pd, pRegGov pn, pRegGov pm, immL_M1 m1) %{
 426   predicate(UseSVE > 0);
 427   match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
 428   ins_cost(SVE_COST);
 429   format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) (D)" %}
 430   ins_encode %{
 431     __ sve_bic(as_PRegister($pd$$reg), ptrue,
 432                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 433   %}
 434   ins_pipe(pipe_slow);
 435 %}
 436 
 437 // vector reinterpret
 438 
 439 instruct reinterpret(vReg dst) %{
 440   predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() ==
 441                           n->in(1)->bottom_type()->is_vect()->length_in_bytes());  // src == dst
 442   match(Set dst (VectorReinterpret dst));
 443   ins_cost(0);
 444   format %{ "# reinterpret $dst\t# do nothing" %}
 445   ins_encode %{
 446     // empty
 447   %}
 448   ins_pipe(pipe_class_empty);
 449 %}
 450 
 451 instruct reinterpretResize(vReg dst, vReg src, pRegGov pgtmp, rFlagsReg cr) %{
 452   predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() !=
 453                           n->in(1)->bottom_type()->is_vect()->length_in_bytes());  // src != dst
 454   match(Set dst (VectorReinterpret src));
 455   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
 456   ins_cost(3 * SVE_COST);
 457   format %{ "reinterpretResize $dst, $src\t# vector (sve)" %}
 458   ins_encode %{
 459     uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
 460     uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
 461     uint length_in_bytes_resize = length_in_bytes_src < length_in_bytes_dst ?
 462                                   length_in_bytes_src : length_in_bytes_dst;
 463     assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
 464            "invalid vector length");
 465     __ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ B, length_in_bytes_resize);
 466     __ sve_dup(as_FloatRegister($dst$$reg), __ B, 0);
 467     __ sve_sel(as_FloatRegister($dst$$reg), __ B, as_PRegister($pgtmp$$reg),
 468                as_FloatRegister($src$$reg), as_FloatRegister($dst$$reg));
 469   %}
 470   ins_pipe(pipe_slow);
 471 %}
 472 
 473 // vector mask reinterpret
 474 
 475 instruct vmask_reinterpret_same_esize(pRegGov dst_src) %{
 476   predicate(UseSVE > 0 &&
 477             n->as_Vector()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
 478             n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
 479   match(Set dst_src (VectorReinterpret dst_src));
 480   ins_cost(0);
 481   format %{ "# vmask_reinterpret $dst_src\t# do nothing" %}
 482   ins_encode %{
 483     // empty
 484   %}
 485   ins_pipe(pipe_class_empty);
 486 %}
 487 
 488 instruct vmask_reinterpret_diff_esize(pRegGov dst, pRegGov src, vReg tmp, rFlagsReg cr) %{
 489   predicate(UseSVE > 0 &&
 490             n->as_Vector()->length() != n->in(1)->bottom_type()->is_vect()->length() &&
 491             n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
 492   match(Set dst (VectorReinterpret src));
 493   effect(TEMP tmp, KILL cr);
 494   ins_cost(2 * SVE_COST);
 495   format %{ "# vmask_reinterpret $dst, $src\t# vector (sve)" %}
 496   ins_encode %{
 497     BasicType from_bt = Matcher::vector_element_basic_type(this, $src);
 498     Assembler::SIMD_RegVariant from_size = __ elemType_to_regVariant(from_bt);
 499     BasicType to_bt = Matcher::vector_element_basic_type(this);
 500     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
 501     __ sve_cpy(as_FloatRegister($tmp$$reg), from_size, as_PRegister($src$$reg), -1, false);
 502     __ sve_cmp(Assembler::EQ, as_PRegister($dst$$reg), to_size, ptrue, as_FloatRegister($tmp$$reg), -1);
 503   %}
 504   ins_pipe(pipe_slow);
 505 %}
 506 
 507 // vector abs
 508 
 509 instruct vabsB(vReg dst, vReg src) %{
 510   predicate(UseSVE > 0 &&
 511             !n->as_Vector()->is_predicated_vector());
 512   match(Set dst (AbsVB src));
 513   ins_cost(SVE_COST);
 514   format %{ "sve_abs $dst, $src\t# vector (sve) (B)" %}
 515   ins_encode %{
 516     __ sve_abs(as_FloatRegister($dst$$reg), __ B,
 517          ptrue, as_FloatRegister($src$$reg));
 518   %}
 519   ins_pipe(pipe_slow);
 520 %}
 521 
 522 instruct vabsS(vReg dst, vReg src) %{
 523   predicate(UseSVE > 0 &&
 524             !n->as_Vector()->is_predicated_vector());
 525   match(Set dst (AbsVS src));
 526   ins_cost(SVE_COST);
 527   format %{ "sve_abs $dst, $src\t# vector (sve) (H)" %}
 528   ins_encode %{
 529     __ sve_abs(as_FloatRegister($dst$$reg), __ H,
 530          ptrue, as_FloatRegister($src$$reg));
 531   %}
 532   ins_pipe(pipe_slow);
 533 %}
 534 
 535 instruct vabsI(vReg dst, vReg src) %{
 536   predicate(UseSVE > 0 &&
 537             !n->as_Vector()->is_predicated_vector());
 538   match(Set dst (AbsVI src));
 539   ins_cost(SVE_COST);
 540   format %{ "sve_abs $dst, $src\t# vector (sve) (S)" %}
 541   ins_encode %{
 542     __ sve_abs(as_FloatRegister($dst$$reg), __ S,
 543          ptrue, as_FloatRegister($src$$reg));
 544   %}
 545   ins_pipe(pipe_slow);
 546 %}
 547 
 548 instruct vabsL(vReg dst, vReg src) %{
 549   predicate(UseSVE > 0 &&
 550             !n->as_Vector()->is_predicated_vector());
 551   match(Set dst (AbsVL src));
 552   ins_cost(SVE_COST);
 553   format %{ "sve_abs $dst, $src\t# vector (sve) (D)" %}
 554   ins_encode %{
 555     __ sve_abs(as_FloatRegister($dst$$reg), __ D,
 556          ptrue, as_FloatRegister($src$$reg));
 557   %}
 558   ins_pipe(pipe_slow);
 559 %}
 560 
 561 instruct vabsF(vReg dst, vReg src) %{
 562   predicate(UseSVE > 0 &&
 563             !n->as_Vector()->is_predicated_vector());
 564   match(Set dst (AbsVF src));
 565   ins_cost(SVE_COST);
 566   format %{ "sve_fabs $dst, $src\t# vector (sve) (S)" %}
 567   ins_encode %{
 568     __ sve_fabs(as_FloatRegister($dst$$reg), __ S,
 569          ptrue, as_FloatRegister($src$$reg));
 570   %}
 571   ins_pipe(pipe_slow);
 572 %}
 573 
 574 instruct vabsD(vReg dst, vReg src) %{
 575   predicate(UseSVE > 0 &&
 576             !n->as_Vector()->is_predicated_vector());
 577   match(Set dst (AbsVD src));
 578   ins_cost(SVE_COST);
 579   format %{ "sve_fabs $dst, $src\t# vector (sve) (D)" %}
 580   ins_encode %{
 581     __ sve_fabs(as_FloatRegister($dst$$reg), __ D,
 582          ptrue, as_FloatRegister($src$$reg));
 583   %}
 584   ins_pipe(pipe_slow);
 585 %}
 586 
 587 // vector abs - predicated
 588 
 589 instruct vabsB_masked(vReg dst_src, pRegGov pg) %{
 590   predicate(UseSVE > 0);
 591   match(Set dst_src (AbsVB dst_src pg));
 592   ins_cost(SVE_COST);
 593   format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (B)" %}
 594   ins_encode %{
 595     __ sve_abs(as_FloatRegister($dst_src$$reg), __ B,
 596             as_PRegister($pg$$reg),
 597             as_FloatRegister($dst_src$$reg));
 598   %}
 599   ins_pipe(pipe_slow);
 600 %}
 601 
 602 instruct vabsS_masked(vReg dst_src, pRegGov pg) %{
 603   predicate(UseSVE > 0);
 604   match(Set dst_src (AbsVS dst_src pg));
 605   ins_cost(SVE_COST);
 606   format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (H)" %}
 607   ins_encode %{
 608     __ sve_abs(as_FloatRegister($dst_src$$reg), __ H,
 609             as_PRegister($pg$$reg),
 610             as_FloatRegister($dst_src$$reg));
 611   %}
 612   ins_pipe(pipe_slow);
 613 %}
 614 
 615 instruct vabsI_masked(vReg dst_src, pRegGov pg) %{
 616   predicate(UseSVE > 0);
 617   match(Set dst_src (AbsVI dst_src pg));
 618   ins_cost(SVE_COST);
 619   format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
 620   ins_encode %{
 621     __ sve_abs(as_FloatRegister($dst_src$$reg), __ S,
 622             as_PRegister($pg$$reg),
 623             as_FloatRegister($dst_src$$reg));
 624   %}
 625   ins_pipe(pipe_slow);
 626 %}
 627 
 628 instruct vabsL_masked(vReg dst_src, pRegGov pg) %{
 629   predicate(UseSVE > 0);
 630   match(Set dst_src (AbsVL dst_src pg));
 631   ins_cost(SVE_COST);
 632   format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
 633   ins_encode %{
 634     __ sve_abs(as_FloatRegister($dst_src$$reg), __ D,
 635             as_PRegister($pg$$reg),
 636             as_FloatRegister($dst_src$$reg));
 637   %}
 638   ins_pipe(pipe_slow);
 639 %}
 640 
 641 instruct vabsF_masked(vReg dst_src, pRegGov pg) %{
 642   predicate(UseSVE > 0);
 643   match(Set dst_src (AbsVF dst_src pg));
 644   ins_cost(SVE_COST);
 645   format %{ "sve_fabs $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
 646   ins_encode %{
 647     __ sve_fabs(as_FloatRegister($dst_src$$reg), __ S,
 648             as_PRegister($pg$$reg),
 649             as_FloatRegister($dst_src$$reg));
 650   %}
 651   ins_pipe(pipe_slow);
 652 %}
 653 
 654 instruct vabsD_masked(vReg dst_src, pRegGov pg) %{
 655   predicate(UseSVE > 0);
 656   match(Set dst_src (AbsVD dst_src pg));
 657   ins_cost(SVE_COST);
 658   format %{ "sve_fabs $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
 659   ins_encode %{
 660     __ sve_fabs(as_FloatRegister($dst_src$$reg), __ D,
 661             as_PRegister($pg$$reg),
 662             as_FloatRegister($dst_src$$reg));
 663   %}
 664   ins_pipe(pipe_slow);
 665 %}
 666 
 667 // vector add
 668 
 669 instruct vaddB(vReg dst, vReg src1, vReg src2) %{
 670   predicate(UseSVE > 0);
 671   match(Set dst (AddVB src1 src2));
 672   ins_cost(SVE_COST);
 673   format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (B)" %}
 674   ins_encode %{
 675     __ sve_add(as_FloatRegister($dst$$reg), __ B,
 676          as_FloatRegister($src1$$reg),
 677          as_FloatRegister($src2$$reg));
 678   %}
 679   ins_pipe(pipe_slow);
 680 %}
 681 
 682 instruct vaddS(vReg dst, vReg src1, vReg src2) %{
 683   predicate(UseSVE > 0);
 684   match(Set dst (AddVS src1 src2));
 685   ins_cost(SVE_COST);
 686   format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (H)" %}
 687   ins_encode %{
 688     __ sve_add(as_FloatRegister($dst$$reg), __ H,
 689          as_FloatRegister($src1$$reg),
 690          as_FloatRegister($src2$$reg));
 691   %}
 692   ins_pipe(pipe_slow);
 693 %}
 694 
 695 instruct vaddI(vReg dst, vReg src1, vReg src2) %{
 696   predicate(UseSVE > 0);
 697   match(Set dst (AddVI src1 src2));
 698   ins_cost(SVE_COST);
 699   format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (S)" %}
 700   ins_encode %{
 701     __ sve_add(as_FloatRegister($dst$$reg), __ S,
 702          as_FloatRegister($src1$$reg),
 703          as_FloatRegister($src2$$reg));
 704   %}
 705   ins_pipe(pipe_slow);
 706 %}
 707 
 708 instruct vaddL(vReg dst, vReg src1, vReg src2) %{
 709   predicate(UseSVE > 0);
 710   match(Set dst (AddVL src1 src2));
 711   ins_cost(SVE_COST);
 712   format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (D)" %}
 713   ins_encode %{
 714     __ sve_add(as_FloatRegister($dst$$reg), __ D,
 715          as_FloatRegister($src1$$reg),
 716          as_FloatRegister($src2$$reg));
 717   %}
 718   ins_pipe(pipe_slow);
 719 %}
 720 
 721 instruct vaddF(vReg dst, vReg src1, vReg src2) %{
 722   predicate(UseSVE > 0);
 723   match(Set dst (AddVF src1 src2));
 724   ins_cost(SVE_COST);
 725   format %{ "sve_fadd $dst, $src1, $src2\t # vector (sve) (S)" %}
 726   ins_encode %{
 727     __ sve_fadd(as_FloatRegister($dst$$reg), __ S,
 728          as_FloatRegister($src1$$reg),
 729          as_FloatRegister($src2$$reg));
 730   %}
 731   ins_pipe(pipe_slow);
 732 %}
 733 
 734 instruct vaddD(vReg dst, vReg src1, vReg src2) %{
 735   predicate(UseSVE > 0);
 736   match(Set dst (AddVD src1 src2));
 737   ins_cost(SVE_COST);
 738   format %{ "sve_fadd $dst, $src1, $src2\t # vector (sve) (D)" %}
 739   ins_encode %{
 740     __ sve_fadd(as_FloatRegister($dst$$reg), __ D,
 741          as_FloatRegister($src1$$reg),
 742          as_FloatRegister($src2$$reg));
 743   %}
 744   ins_pipe(pipe_slow);
 745 %}
 746 
 747 // vector add - predicated
 748 
 749 instruct vaddB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 750   predicate(UseSVE > 0);
 751   match(Set dst_src1 (AddVB (Binary dst_src1 src2) pg));
 752   ins_cost(SVE_COST);
 753   format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
 754   ins_encode %{
 755     __ sve_add(as_FloatRegister($dst_src1$$reg), __ B,
 756             as_PRegister($pg$$reg),
 757             as_FloatRegister($src2$$reg));
 758   %}
 759   ins_pipe(pipe_slow);
 760 %}
 761 
 762 instruct vaddS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 763   predicate(UseSVE > 0);
 764   match(Set dst_src1 (AddVS (Binary dst_src1 src2) pg));
 765   ins_cost(SVE_COST);
 766   format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
 767   ins_encode %{
 768     __ sve_add(as_FloatRegister($dst_src1$$reg), __ H,
 769             as_PRegister($pg$$reg),
 770             as_FloatRegister($src2$$reg));
 771   %}
 772   ins_pipe(pipe_slow);
 773 %}
 774 
 775 instruct vaddI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 776   predicate(UseSVE > 0);
 777   match(Set dst_src1 (AddVI (Binary dst_src1 src2) pg));
 778   ins_cost(SVE_COST);
 779   format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
 780   ins_encode %{
 781     __ sve_add(as_FloatRegister($dst_src1$$reg), __ S,
 782             as_PRegister($pg$$reg),
 783             as_FloatRegister($src2$$reg));
 784   %}
 785   ins_pipe(pipe_slow);
 786 %}
 787 
 788 instruct vaddL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 789   predicate(UseSVE > 0);
 790   match(Set dst_src1 (AddVL (Binary dst_src1 src2) pg));
 791   ins_cost(SVE_COST);
 792   format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
 793   ins_encode %{
 794     __ sve_add(as_FloatRegister($dst_src1$$reg), __ D,
 795             as_PRegister($pg$$reg),
 796             as_FloatRegister($src2$$reg));
 797   %}
 798   ins_pipe(pipe_slow);
 799 %}
 800 
 801 instruct vaddF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 802   predicate(UseSVE > 0);
 803   match(Set dst_src1 (AddVF (Binary dst_src1 src2) pg));
 804   ins_cost(SVE_COST);
 805   format %{ "sve_fadd $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
 806   ins_encode %{
 807     __ sve_fadd(as_FloatRegister($dst_src1$$reg), __ S,
 808             as_PRegister($pg$$reg),
 809             as_FloatRegister($src2$$reg));
 810   %}
 811   ins_pipe(pipe_slow);
 812 %}
 813 
 814 instruct vaddD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 815   predicate(UseSVE > 0);
 816   match(Set dst_src1 (AddVD (Binary dst_src1 src2) pg));
 817   ins_cost(SVE_COST);
 818   format %{ "sve_fadd $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
 819   ins_encode %{
 820     __ sve_fadd(as_FloatRegister($dst_src1$$reg), __ D,
 821             as_PRegister($pg$$reg),
 822             as_FloatRegister($src2$$reg));
 823   %}
 824   ins_pipe(pipe_slow);
 825 %}
 826 
 827 // vector add reg imm (unpredicated)
 828 
 829 instruct vaddImmB(vReg dst_src, immBAddSubV con) %{
 830   predicate(UseSVE > 0);
 831   match(Set dst_src (AddVB dst_src (ReplicateB con)));
 832   ins_cost(SVE_COST);
 833   format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) (B)" %}
 834   ins_encode %{
 835     int32_t val = $con$$constant;
 836     if (val > 0){
 837       __ sve_add(as_FloatRegister($dst_src$$reg), __ B, val);
 838     } else if (val < 0){
 839       __ sve_sub(as_FloatRegister($dst_src$$reg), __ B, -val);
 840     }
 841   %}
 842   ins_pipe(pipe_slow);
 843 %}
 844 
 845 instruct vaddImmS(vReg dst_src, immIAddSubV con) %{
 846   predicate(UseSVE > 0);
 847   match(Set dst_src (AddVS dst_src (ReplicateS con)));
 848   ins_cost(SVE_COST);
 849   format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) (H)" %}
 850   ins_encode %{
 851     int32_t val = $con$$constant;
 852     if (val > 0){
 853       __ sve_add(as_FloatRegister($dst_src$$reg), __ H, val);
 854     } else if (val < 0){
 855       __ sve_sub(as_FloatRegister($dst_src$$reg), __ H, -val);
 856     }
 857   %}
 858   ins_pipe(pipe_slow);
 859 %}
 860 
 861 instruct vaddImmI(vReg dst_src, immIAddSubV con) %{
 862   predicate(UseSVE > 0);
 863   match(Set dst_src (AddVI dst_src (ReplicateI con)));
 864   ins_cost(SVE_COST);
 865   format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) (S)" %}
 866   ins_encode %{
 867     int32_t val = $con$$constant;
 868     if (val > 0){
 869       __ sve_add(as_FloatRegister($dst_src$$reg), __ S, val);
 870     } else if (val < 0){
 871       __ sve_sub(as_FloatRegister($dst_src$$reg), __ S, -val);
 872     }
 873   %}
 874   ins_pipe(pipe_slow);
 875 %}
 876 
 877 instruct vaddImmL(vReg dst_src, immLAddSubV con) %{
 878   predicate(UseSVE > 0);
 879   match(Set dst_src (AddVL dst_src (ReplicateL con)));
 880   ins_cost(SVE_COST);
 881   format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) (D)" %}
 882   ins_encode %{
 883     int32_t val = $con$$constant;
 884     if (val > 0){
 885       __ sve_add(as_FloatRegister($dst_src$$reg), __ D, val);
 886     } else if (val < 0){
 887       __ sve_sub(as_FloatRegister($dst_src$$reg), __ D, -val);
 888     }
 889   %}
 890   ins_pipe(pipe_slow);
 891 %}
 892 
 893 // vector binary op reg imm (unpredicated)
 894 
 895 instruct vandB(vReg dst_src, immBLog con) %{
 896   predicate(UseSVE > 0);
 897   match(Set dst_src (AndV dst_src (ReplicateB con)));
 898   ins_cost(SVE_COST);
 899   format %{ "sve_and $dst_src, $dst_src, $con\t # vector (sve) (B)" %}
 900   ins_encode %{
 901     __ sve_and(as_FloatRegister($dst_src$$reg), __ B,
 902          (uint64_t)($con$$constant));
 903   %}
 904   ins_pipe(pipe_slow);
 905 %}
 906 
 907 instruct vandH(vReg dst_src, immSLog con) %{
 908   predicate(UseSVE > 0);
 909   match(Set dst_src (AndV dst_src (ReplicateS con)));
 910   ins_cost(SVE_COST);
 911   format %{ "sve_and $dst_src, $dst_src, $con\t # vector (sve) (H)" %}
 912   ins_encode %{
 913     __ sve_and(as_FloatRegister($dst_src$$reg), __ H,
 914          (uint64_t)($con$$constant));
 915   %}
 916   ins_pipe(pipe_slow);
 917 %}
 918 
 919 instruct vandS(vReg dst_src, immILog con) %{
 920   predicate(UseSVE > 0);
 921   match(Set dst_src (AndV dst_src (ReplicateI con)));
 922   ins_cost(SVE_COST);
 923   format %{ "sve_and $dst_src, $dst_src, $con\t # vector (sve) (S)" %}
 924   ins_encode %{
 925     __ sve_and(as_FloatRegister($dst_src$$reg), __ S,
 926          (uint64_t)($con$$constant));
 927   %}
 928   ins_pipe(pipe_slow);
 929 %}
 930 
 931 instruct vandD(vReg dst_src, immLLog con) %{
 932   predicate(UseSVE > 0);
 933   match(Set dst_src (AndV dst_src (ReplicateL con)));
 934   ins_cost(SVE_COST);
 935   format %{ "sve_and $dst_src, $dst_src, $con\t # vector (sve) (D)" %}
 936   ins_encode %{
 937     __ sve_and(as_FloatRegister($dst_src$$reg), __ D,
 938          (uint64_t)($con$$constant));
 939   %}
 940   ins_pipe(pipe_slow);
 941 %}
 942 
 943 instruct vorB(vReg dst_src, immBLog con) %{
 944   predicate(UseSVE > 0);
 945   match(Set dst_src (OrV dst_src (ReplicateB con)));
 946   ins_cost(SVE_COST);
 947   format %{ "sve_orr $dst_src, $dst_src, $con\t # vector (sve) (B)" %}
 948   ins_encode %{
 949     __ sve_orr(as_FloatRegister($dst_src$$reg), __ B,
 950          (uint64_t)($con$$constant));
 951   %}
 952   ins_pipe(pipe_slow);
 953 %}
 954 
 955 instruct vorH(vReg dst_src, immSLog con) %{
 956   predicate(UseSVE > 0);
 957   match(Set dst_src (OrV dst_src (ReplicateS con)));
 958   ins_cost(SVE_COST);
 959   format %{ "sve_orr $dst_src, $dst_src, $con\t # vector (sve) (H)" %}
 960   ins_encode %{
 961     __ sve_orr(as_FloatRegister($dst_src$$reg), __ H,
 962          (uint64_t)($con$$constant));
 963   %}
 964   ins_pipe(pipe_slow);
 965 %}
 966 
 967 instruct vorS(vReg dst_src, immILog con) %{
 968   predicate(UseSVE > 0);
 969   match(Set dst_src (OrV dst_src (ReplicateI con)));
 970   ins_cost(SVE_COST);
 971   format %{ "sve_orr $dst_src, $dst_src, $con\t # vector (sve) (S)" %}
 972   ins_encode %{
 973     __ sve_orr(as_FloatRegister($dst_src$$reg), __ S,
 974          (uint64_t)($con$$constant));
 975   %}
 976   ins_pipe(pipe_slow);
 977 %}
 978 
 979 instruct vorD(vReg dst_src, immLLog con) %{
 980   predicate(UseSVE > 0);
 981   match(Set dst_src (OrV dst_src (ReplicateL con)));
 982   ins_cost(SVE_COST);
 983   format %{ "sve_orr $dst_src, $dst_src, $con\t # vector (sve) (D)" %}
 984   ins_encode %{
 985     __ sve_orr(as_FloatRegister($dst_src$$reg), __ D,
 986          (uint64_t)($con$$constant));
 987   %}
 988   ins_pipe(pipe_slow);
 989 %}
 990 
 991 instruct vxorB(vReg dst_src, immBLog con) %{
 992   predicate(UseSVE > 0);
 993   match(Set dst_src (XorV dst_src (ReplicateB con)));
 994   ins_cost(SVE_COST);
 995   format %{ "sve_eor $dst_src, $dst_src, $con\t # vector (sve) (B)" %}
 996   ins_encode %{
 997     __ sve_eor(as_FloatRegister($dst_src$$reg), __ B,
 998          (uint64_t)($con$$constant));
 999   %}
1000   ins_pipe(pipe_slow);
1001 %}
1002 
1003 instruct vxorH(vReg dst_src, immSLog con) %{
1004   predicate(UseSVE > 0);
1005   match(Set dst_src (XorV dst_src (ReplicateS con)));
1006   ins_cost(SVE_COST);
1007   format %{ "sve_eor $dst_src, $dst_src, $con\t # vector (sve) (H)" %}
1008   ins_encode %{
1009     __ sve_eor(as_FloatRegister($dst_src$$reg), __ H,
1010          (uint64_t)($con$$constant));
1011   %}
1012   ins_pipe(pipe_slow);
1013 %}
1014 
1015 instruct vxorS(vReg dst_src, immILog con) %{
1016   predicate(UseSVE > 0);
1017   match(Set dst_src (XorV dst_src (ReplicateI con)));
1018   ins_cost(SVE_COST);
1019   format %{ "sve_eor $dst_src, $dst_src, $con\t # vector (sve) (S)" %}
1020   ins_encode %{
1021     __ sve_eor(as_FloatRegister($dst_src$$reg), __ S,
1022          (uint64_t)($con$$constant));
1023   %}
1024   ins_pipe(pipe_slow);
1025 %}
1026 
1027 instruct vxorD(vReg dst_src, immLLog con) %{
1028   predicate(UseSVE > 0);
1029   match(Set dst_src (XorV dst_src (ReplicateL con)));
1030   ins_cost(SVE_COST);
1031   format %{ "sve_eor $dst_src, $dst_src, $con\t # vector (sve) (D)" %}
1032   ins_encode %{
1033     __ sve_eor(as_FloatRegister($dst_src$$reg), __ D,
1034          (uint64_t)($con$$constant));
1035   %}
1036   ins_pipe(pipe_slow);
1037 %}
1038 // vector and
1039 
1040 instruct vand(vReg dst, vReg src1, vReg src2) %{
1041   predicate(UseSVE > 0);
1042   match(Set dst (AndV src1 src2));
1043   ins_cost(SVE_COST);
1044   format %{ "sve_and  $dst, $src1, $src2\t# vector (sve)" %}
1045   ins_encode %{
1046     __ sve_and(as_FloatRegister($dst$$reg),
1047          as_FloatRegister($src1$$reg),
1048          as_FloatRegister($src2$$reg));
1049   %}
1050   ins_pipe(pipe_slow);
1051 %}
1052 
1053 // vector or
1054 
1055 instruct vor(vReg dst, vReg src1, vReg src2) %{
1056   predicate(UseSVE > 0);
1057   match(Set dst (OrV src1 src2));
1058   ins_cost(SVE_COST);
1059   format %{ "sve_orr  $dst, $src1, $src2\t# vector (sve)" %}
1060   ins_encode %{
1061     __ sve_orr(as_FloatRegister($dst$$reg),
1062          as_FloatRegister($src1$$reg),
1063          as_FloatRegister($src2$$reg));
1064   %}
1065   ins_pipe(pipe_slow);
1066 %}
1067 
1068 // vector xor
1069 
1070 instruct vxor(vReg dst, vReg src1, vReg src2) %{
1071   predicate(UseSVE > 0);
1072   match(Set dst (XorV src1 src2));
1073   ins_cost(SVE_COST);
1074   format %{ "sve_eor  $dst, $src1, $src2\t# vector (sve)" %}
1075   ins_encode %{
1076     __ sve_eor(as_FloatRegister($dst$$reg),
1077          as_FloatRegister($src1$$reg),
1078          as_FloatRegister($src2$$reg));
1079   %}
1080   ins_pipe(pipe_slow);
1081 %}
1082 
1083 // vector and - predicated
1084 
1085 instruct vand_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1086   predicate(UseSVE > 0);
1087   match(Set dst_src1 (AndV (Binary dst_src1 src2) pg));
1088   ins_cost(SVE_COST);
1089   format %{ "sve_and $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
1090   ins_encode %{
1091     BasicType bt = Matcher::vector_element_basic_type(this);
1092     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1093     __ sve_and(as_FloatRegister($dst_src1$$reg), size,
1094           as_PRegister($pg$$reg),
1095           as_FloatRegister($src2$$reg));
1096   %}
1097   ins_pipe(pipe_slow);
1098 %}
1099 
1100 // vector or - predicated
1101 
1102 instruct vor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1103   predicate(UseSVE > 0);
1104   match(Set dst_src1 (OrV (Binary dst_src1 src2) pg));
1105   ins_cost(SVE_COST);
1106   format %{ "sve_orr $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
1107   ins_encode %{
1108     BasicType bt = Matcher::vector_element_basic_type(this);
1109     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1110     __ sve_orr(as_FloatRegister($dst_src1$$reg), size,
1111           as_PRegister($pg$$reg),
1112           as_FloatRegister($src2$$reg));
1113   %}
1114   ins_pipe(pipe_slow);
1115 %}
1116 
1117 // vector xor - predicated
1118 
1119 instruct vxor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1120   predicate(UseSVE > 0);
1121   match(Set dst_src1 (XorV (Binary dst_src1 src2) pg));
1122   ins_cost(SVE_COST);
1123   format %{ "sve_eor $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
1124   ins_encode %{
1125     BasicType bt = Matcher::vector_element_basic_type(this);
1126     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1127     __ sve_eor(as_FloatRegister($dst_src1$$reg), size,
1128           as_PRegister($pg$$reg),
1129           as_FloatRegister($src2$$reg));
1130   %}
1131   ins_pipe(pipe_slow);
1132 %}
1133 
1134 // vector not
1135 
1136 instruct vnotI(vReg dst, vReg src, immI_M1 m1) %{
1137   predicate(UseSVE > 0);
1138   match(Set dst (XorV src (ReplicateB m1)));
1139   match(Set dst (XorV src (ReplicateS m1)));
1140   match(Set dst (XorV src (ReplicateI m1)));
1141   ins_cost(SVE_COST);
1142   format %{ "sve_not $dst, $src\t# vector (sve) B/H/S" %}
1143   ins_encode %{
1144     __ sve_not(as_FloatRegister($dst$$reg), __ D,
1145                ptrue, as_FloatRegister($src$$reg));
1146   %}
1147   ins_pipe(pipe_slow);
1148 %}
1149 
1150 instruct vnotL(vReg dst, vReg src, immL_M1 m1) %{
1151   predicate(UseSVE > 0);
1152   match(Set dst (XorV src (ReplicateL m1)));
1153   ins_cost(SVE_COST);
1154   format %{ "sve_not $dst, $src\t# vector (sve) D" %}
1155   ins_encode %{
1156     __ sve_not(as_FloatRegister($dst$$reg), __ D,
1157                ptrue, as_FloatRegister($src$$reg));
1158   %}
1159   ins_pipe(pipe_slow);
1160 %}
1161 
1162 // vector not - predicated
1163 
1164 instruct vnotI_masked(vReg dst, vReg src, immI_M1 m1, pRegGov pg) %{
1165   predicate(UseSVE > 0);
1166   match(Set dst (XorV (Binary src (ReplicateB m1)) pg));
1167   match(Set dst (XorV (Binary src (ReplicateS m1)) pg));
1168   match(Set dst (XorV (Binary src (ReplicateI m1)) pg));
1169   ins_cost(SVE_COST);
1170   format %{ "sve_not $dst, $pg, $src\t# vector (sve) B/H/S" %}
1171   ins_encode %{
1172     BasicType bt = Matcher::vector_element_basic_type(this);
1173     __ sve_not(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
1174                as_PRegister($pg$$reg), as_FloatRegister($src$$reg));
1175   %}
1176   ins_pipe(pipe_slow);
1177 %}
1178 
1179 instruct vnotL_masked(vReg dst, vReg src, immL_M1 m1, pRegGov pg) %{
1180   predicate(UseSVE > 0);
1181   match(Set dst (XorV (Binary src (ReplicateL m1)) pg));
1182   ins_cost(SVE_COST);
1183   format %{ "sve_not $dst, $pg, $src\t# vector (sve) D" %}
1184   ins_encode %{
1185     BasicType bt = Matcher::vector_element_basic_type(this);
1186     __ sve_not(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
1187                as_PRegister($pg$$reg), as_FloatRegister($src$$reg));
1188   %}
1189   ins_pipe(pipe_slow);
1190 %}
1191 
1192 // vector and_not
1193 
1194 instruct vand_notI(vReg dst, vReg src1, vReg src2, immI_M1 m1) %{
1195   predicate(UseSVE > 0);
1196   match(Set dst (AndV src1 (XorV src2 (ReplicateB m1))));
1197   match(Set dst (AndV src1 (XorV src2 (ReplicateS m1))));
1198   match(Set dst (AndV src1 (XorV src2 (ReplicateI m1))));
1199   ins_cost(SVE_COST);
1200   format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) B/H/S" %}
1201   ins_encode %{
1202     __ sve_bic(as_FloatRegister($dst$$reg),
1203                as_FloatRegister($src1$$reg),
1204                as_FloatRegister($src2$$reg));
1205   %}
1206   ins_pipe(pipe_slow);
1207 %}
1208 
1209 instruct vand_notL(vReg dst, vReg src1, vReg src2, immL_M1 m1) %{
1210   predicate(UseSVE > 0);
1211   match(Set dst (AndV src1 (XorV src2 (ReplicateL m1))));
1212   ins_cost(SVE_COST);
1213   format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) D" %}
1214   ins_encode %{
1215     __ sve_bic(as_FloatRegister($dst$$reg),
1216                as_FloatRegister($src1$$reg),
1217                as_FloatRegister($src2$$reg));
1218   %}
1219   ins_pipe(pipe_slow);
1220 %}
1221 
1222 // vector and_not - predicated
1223 
1224 instruct vand_notI_masked(vReg dst_src1, vReg src2, immI_M1 m1, pRegGov pg) %{
1225   predicate(UseSVE > 0);
1226   match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateB m1))) pg));
1227   match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateS m1))) pg));
1228   match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateI m1))) pg));
1229   ins_cost(SVE_COST);
1230   format %{ "sve_bic $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) B/H/S" %}
1231   ins_encode %{
1232     BasicType bt = Matcher::vector_element_basic_type(this);
1233     __ sve_bic(as_FloatRegister($dst_src1$$reg), __ elemType_to_regVariant(bt),
1234                as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1235   %}
1236   ins_pipe(pipe_slow);
1237 %}
1238 
1239 instruct vand_notL_masked(vReg dst_src1, vReg src2, immL_M1 m1, pRegGov pg) %{
1240   predicate(UseSVE > 0);
1241   match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateL m1))) pg));
1242   ins_cost(SVE_COST);
1243   format %{ "sve_bic $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) D" %}
1244   ins_encode %{
1245     BasicType bt = Matcher::vector_element_basic_type(this);
1246     __ sve_bic(as_FloatRegister($dst_src1$$reg), __ elemType_to_regVariant(bt),
1247                as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1248   %}
1249   ins_pipe(pipe_slow);
1250 %}
1251 
1252 // vector float div
1253 
1254 instruct vdivF(vReg dst_src1, vReg src2) %{
1255   predicate(UseSVE > 0);
1256   match(Set dst_src1 (DivVF dst_src1 src2));
1257   ins_cost(SVE_COST);
1258   format %{ "sve_fdiv  $dst_src1, $dst_src1, $src2\t# vector (sve) (S)" %}
1259   ins_encode %{
1260     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ S,
1261          ptrue, as_FloatRegister($src2$$reg));
1262   %}
1263   ins_pipe(pipe_slow);
1264 %}
1265 
1266 instruct vdivD(vReg dst_src1, vReg src2) %{
1267   predicate(UseSVE > 0);
1268   match(Set dst_src1 (DivVD dst_src1 src2));
1269   ins_cost(SVE_COST);
1270   format %{ "sve_fdiv  $dst_src1, $dst_src1, $src2\t# vector (sve) (D)" %}
1271   ins_encode %{
1272     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ D,
1273          ptrue, as_FloatRegister($src2$$reg));
1274   %}
1275   ins_pipe(pipe_slow);
1276 %}
1277 
1278 // vector float div - predicated
1279 
1280 instruct vfdivF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1281   predicate(UseSVE > 0);
1282   match(Set dst_src1 (DivVF (Binary dst_src1 src2) pg));
1283   ins_cost(SVE_COST);
1284   format %{ "sve_fdiv $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
1285   ins_encode %{
1286     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ S,
1287             as_PRegister($pg$$reg),
1288             as_FloatRegister($src2$$reg));
1289   %}
1290   ins_pipe(pipe_slow);
1291 %}
1292 
1293 instruct vfdivD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1294   predicate(UseSVE > 0);
1295   match(Set dst_src1 (DivVD (Binary dst_src1 src2) pg));
1296   ins_cost(SVE_COST);
1297   format %{ "sve_fdiv $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
1298   ins_encode %{
1299     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ D,
1300             as_PRegister($pg$$reg),
1301             as_FloatRegister($src2$$reg));
1302   %}
1303   ins_pipe(pipe_slow);
1304 %}
1305 
1306 // vector min/max
1307 
1308 instruct vmin(vReg dst_src1, vReg src2) %{
1309   predicate(UseSVE > 0);
1310   match(Set dst_src1 (MinV dst_src1 src2));
1311   ins_cost(SVE_COST);
1312   format %{ "sve_min $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
1313   ins_encode %{
1314     BasicType bt = Matcher::vector_element_basic_type(this);
1315     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1316     if (is_floating_point_type(bt)) {
1317       __ sve_fmin(as_FloatRegister($dst_src1$$reg), size,
1318                   ptrue, as_FloatRegister($src2$$reg));
1319     } else {
1320       assert(is_integral_type(bt), "unsupported type");
1321       __ sve_smin(as_FloatRegister($dst_src1$$reg), size,
1322                   ptrue, as_FloatRegister($src2$$reg));
1323     }
1324   %}
1325   ins_pipe(pipe_slow);
1326 %}
1327 
1328 instruct vmax(vReg dst_src1, vReg src2) %{
1329   predicate(UseSVE > 0);
1330   match(Set dst_src1 (MaxV dst_src1 src2));
1331   ins_cost(SVE_COST);
1332   format %{ "sve_max $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
1333   ins_encode %{
1334     BasicType bt = Matcher::vector_element_basic_type(this);
1335     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1336     if (is_floating_point_type(bt)) {
1337       __ sve_fmax(as_FloatRegister($dst_src1$$reg), size,
1338                   ptrue, as_FloatRegister($src2$$reg));
1339     } else {
1340       assert(is_integral_type(bt), "unsupported type");
1341       __ sve_smax(as_FloatRegister($dst_src1$$reg), size,
1342                   ptrue, as_FloatRegister($src2$$reg));
1343     }
1344   %}
1345   ins_pipe(pipe_slow);
1346 %}
1347 
1348 // vector min/max - predicated
1349 
1350 instruct vmin_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1351   predicate(UseSVE > 0);
1352   match(Set dst_src1 (MinV (Binary dst_src1 src2) pg));
1353   ins_cost(SVE_COST);
1354   format %{ "sve_min $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %}
1355   ins_encode %{
1356     BasicType bt = Matcher::vector_element_basic_type(this);
1357     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1358     if (is_floating_point_type(bt)) {
1359       __ sve_fmin(as_FloatRegister($dst_src1$$reg), size,
1360                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1361     } else {
1362       assert(is_integral_type(bt), "unsupported type");
1363       __ sve_smin(as_FloatRegister($dst_src1$$reg), size,
1364                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1365     }
1366   %}
1367   ins_pipe(pipe_slow);
1368 %}
1369 
1370 instruct vmax_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1371   predicate(UseSVE > 0);
1372   match(Set dst_src1 (MaxV (Binary dst_src1 src2) pg));
1373   ins_cost(SVE_COST);
1374   format %{ "sve_max $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %}
1375   ins_encode %{
1376     BasicType bt = Matcher::vector_element_basic_type(this);
1377     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1378     if (is_floating_point_type(bt)) {
1379       __ sve_fmax(as_FloatRegister($dst_src1$$reg), size,
1380                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1381     } else {
1382       assert(is_integral_type(bt), "unsupported type");
1383       __ sve_smax(as_FloatRegister($dst_src1$$reg), size,
1384                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1385     }
1386   %}
1387   ins_pipe(pipe_slow);
1388 %}
1389 
1390 // vector fmla
1391 
1392 // dst_src1 = dst_src1 + src2 * src3
1393 instruct vfmlaF(vReg dst_src1, vReg src2, vReg src3) %{
1394   predicate(UseFMA && UseSVE > 0);
1395   match(Set dst_src1 (FmaVF dst_src1 (Binary src2 src3)));
1396   ins_cost(SVE_COST);
1397   format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1398   ins_encode %{
1399     __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ S,
1400          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1401   %}
1402   ins_pipe(pipe_slow);
1403 %}
1404 
1405 // dst_src1 = dst_src1 + src2 * src3
1406 instruct vfmlaD(vReg dst_src1, vReg src2, vReg src3) %{
1407   predicate(UseFMA && UseSVE > 0);
1408   match(Set dst_src1 (FmaVD dst_src1 (Binary src2 src3)));
1409   ins_cost(SVE_COST);
1410   format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1411   ins_encode %{
1412     __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ D,
1413          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1414   %}
1415   ins_pipe(pipe_slow);
1416 %}
1417 
1418 // vector fmad - predicated
1419 
1420 // dst_src1 = dst_src1 * src2 + src3
1421 instruct vfmadF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1422   predicate(UseFMA && UseSVE > 0);
1423   match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary src3 pg)));
1424   ins_cost(SVE_COST);
1425   format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) (S)" %}
1426   ins_encode %{
1427     __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
1428          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1429   %}
1430   ins_pipe(pipe_slow);
1431 %}
1432 
1433 // dst_src1 = dst_src1 * src2 + src3
1434 instruct vfmadD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1435   predicate(UseFMA && UseSVE > 0);
1436   match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary src3 pg)));
1437   ins_cost(SVE_COST);
1438   format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) (D)" %}
1439   ins_encode %{
1440     __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
1441          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1442   %}
1443   ins_pipe(pipe_slow);
1444 %}
1445 
1446 // vector fmls
1447 
1448 // dst_src1 = dst_src1 + -src2 * src3
1449 // The NegVF must not be predicated.
1450 instruct vfmlsF1(vReg dst_src1, vReg src2, vReg src3) %{
1451   predicate(UseFMA && UseSVE > 0 &&
1452             !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1453   match(Set dst_src1 (FmaVF dst_src1 (Binary (NegVF src2) src3)));
1454   ins_cost(SVE_COST);
1455   format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1456   ins_encode %{
1457     __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ S,
1458          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1459   %}
1460   ins_pipe(pipe_slow);
1461 %}
1462 
1463 // dst_src1 = dst_src1 + src2 * -src3
1464 // The NegVF must not be predicated.
1465 instruct vfmlsF2(vReg dst_src1, vReg src2, vReg src3) %{
1466   predicate(UseFMA && UseSVE > 0 &&
1467             !n->in(2)->in(2)->as_Vector()->is_predicated_vector());
1468   match(Set dst_src1 (FmaVF dst_src1 (Binary src2 (NegVF src3))));
1469   ins_cost(SVE_COST);
1470   format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1471   ins_encode %{
1472     __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ S,
1473          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1474   %}
1475   ins_pipe(pipe_slow);
1476 %}
1477 
1478 // dst_src1 = dst_src1 + -src2 * src3
1479 // The NegVD must not be predicated.
1480 instruct vfmlsD1(vReg dst_src1, vReg src2, vReg src3) %{
1481   predicate(UseFMA && UseSVE > 0 &&
1482             !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1483   match(Set dst_src1 (FmaVD dst_src1 (Binary (NegVD src2) src3)));
1484   ins_cost(SVE_COST);
1485   format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1486   ins_encode %{
1487     __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ D,
1488          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1489   %}
1490   ins_pipe(pipe_slow);
1491 %}
1492 
1493 // dst_src1 = dst_src1 + src2 * -src3
1494 // The NegVD must not be predicated.
1495 instruct vfmlsD2(vReg dst_src1, vReg src2, vReg src3) %{
1496   predicate(UseFMA && UseSVE > 0 &&
1497             !n->in(2)->in(2)->as_Vector()->is_predicated_vector());
1498   match(Set dst_src1 (FmaVD dst_src1 (Binary src2 (NegVD src3))));
1499   ins_cost(SVE_COST);
1500   format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1501   ins_encode %{
1502     __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ D,
1503          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1504   %}
1505   ins_pipe(pipe_slow);
1506 %}
1507 
1508 // vector fmsb - predicated
1509 
1510 // dst_src1 = dst_src1 * -src2 + src3
1511 // The NegVF must not be predicated.
1512 instruct vfmsbF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1513   predicate(UseFMA && UseSVE > 0 &&
1514             !n->in(1)->in(2)->as_Vector()->is_predicated_vector());
1515   match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary src3 pg)));
1516   ins_cost(SVE_COST);
1517   format %{ "sve_fmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) (S)" %}
1518   ins_encode %{
1519     __ sve_fmsb(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
1520          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1521   %}
1522   ins_pipe(pipe_slow);
1523 %}
1524 
1525 // dst_src1 = dst_src1 * -src2 + src3
1526 // The NegVD must not be predicated.
1527 instruct vfmsbD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1528   predicate(UseFMA && UseSVE > 0 &&
1529             !n->in(1)->in(2)->as_Vector()->is_predicated_vector());
1530   match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary src3 pg)));
1531   ins_cost(SVE_COST);
1532   format %{ "sve_fmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) (D)" %}
1533   ins_encode %{
1534     __ sve_fmsb(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
1535          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1536   %}
1537   ins_pipe(pipe_slow);
1538 %}
1539 
1540 // vector fnmla
1541 
1542 // dst_src1 = -dst_src1 + -src2 * src3
1543 // The NegVF must not be predicated.
1544 instruct vfnmlaF1(vReg dst_src1, vReg src2, vReg src3) %{
1545   predicate(UseFMA && UseSVE > 0 &&
1546             !n->in(1)->as_Vector()->is_predicated_vector() &&
1547             !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1548   match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary (NegVF src2) src3)));
1549   ins_cost(SVE_COST);
1550   format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1551   ins_encode %{
1552     __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ S,
1553          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1554   %}
1555   ins_pipe(pipe_slow);
1556 %}
1557 
1558 // dst_src1 = -dst_src1 + src2 * -src3
1559 // The NegVF must not be predicated.
1560 instruct vfnmlaF2(vReg dst_src1, vReg src2, vReg src3) %{
1561   predicate(UseFMA && UseSVE > 0 &&
1562             !n->in(1)->as_Vector()->is_predicated_vector() &&
1563             !n->in(2)->in(2)->as_Vector()->is_predicated_vector());
1564   match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 (NegVF src3))));
1565   ins_cost(SVE_COST);
1566   format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1567   ins_encode %{
1568     __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ S,
1569          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1570   %}
1571   ins_pipe(pipe_slow);
1572 %}
1573 
1574 // dst_src1 = -dst_src1 + -src2 * src3
1575 // The NegVD must not be predicated.
1576 instruct vfnmlaD1(vReg dst_src1, vReg src2, vReg src3) %{
1577   predicate(UseFMA && UseSVE > 0 &&
1578             !n->in(1)->as_Vector()->is_predicated_vector() &&
1579             !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1580   match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary (NegVD src2) src3)));
1581   ins_cost(SVE_COST);
1582   format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1583   ins_encode %{
1584     __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ D,
1585          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1586   %}
1587   ins_pipe(pipe_slow);
1588 %}
1589 
1590 // dst_src1 = -dst_src1 + src2 * -src3
1591 // The NegVD must not be predicated.
1592 instruct vfnmlaD2(vReg dst_src1, vReg src2, vReg src3) %{
1593   predicate(UseFMA && UseSVE > 0 &&
1594             !n->in(1)->as_Vector()->is_predicated_vector() &&
1595             !n->in(2)->in(2)->as_Vector()->is_predicated_vector());
1596   match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 (NegVD src3))));
1597   ins_cost(SVE_COST);
1598   format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1599   ins_encode %{
1600     __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ D,
1601          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1602   %}
1603   ins_pipe(pipe_slow);
1604 %}
1605 
1606 // vector fnmad - predicated
1607 
1608 // dst_src1 = -src3 + dst_src1 * -src2
1609 // The NegVF must not be predicated.
1610 instruct vfnmadF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1611   predicate(UseFMA && UseSVE > 0 &&
1612             !n->in(1)->in(2)->as_Vector()->is_predicated_vector() &&
1613             !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1614   match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary (NegVF src3) pg)));
1615   ins_cost(SVE_COST);
1616   format %{ "sve_fnmad $dst_src1, $pg, $src2, $src3\t # vector (sve) (S)" %}
1617   ins_encode %{
1618     __ sve_fnmad(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
1619          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1620   %}
1621   ins_pipe(pipe_slow);
1622 %}
1623 
1624 // dst_src1 = -src3 + dst_src1 * -src2
1625 // The NegVD must not be predicated.
1626 instruct vfnmadD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1627   predicate(UseFMA && UseSVE > 0 &&
1628             !n->in(1)->in(2)->as_Vector()->is_predicated_vector() &&
1629             !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1630   match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary (NegVD src3) pg)));
1631   ins_cost(SVE_COST);
1632   format %{ "sve_fnmad $dst_src1, $pg, $src2, $src3\t # vector (sve) (D)" %}
1633   ins_encode %{
1634     __ sve_fnmad(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
1635          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1636   %}
1637   ins_pipe(pipe_slow);
1638 %}
1639 
1640 // vector fnmls
1641 
1642 // dst_src1 = -dst_src1 + src2 * src3
1643 // The NegVF must not be predicated.
1644 instruct vfnmlsF(vReg dst_src1, vReg src2, vReg src3) %{
1645   predicate(UseFMA && UseSVE > 0 &&
1646             !n->in(1)->as_Vector()->is_predicated_vector());
1647   match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 src3)));
1648   ins_cost(SVE_COST);
1649   format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1650   ins_encode %{
1651     __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ S,
1652          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1653   %}
1654   ins_pipe(pipe_slow);
1655 %}
1656 
1657 // dst_src1 = -dst_src1 + src2 * src3
1658 // The NegVD must not be predicated.
1659 instruct vfnmlsD(vReg dst_src1, vReg src2, vReg src3) %{
1660   predicate(UseFMA && UseSVE > 0 &&
1661             !n->in(1)->as_Vector()->is_predicated_vector());
1662   match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 src3)));
1663   ins_cost(SVE_COST);
1664   format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1665   ins_encode %{
1666     __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ D,
1667          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1668   %}
1669   ins_pipe(pipe_slow);
1670 %}
1671 
1672 // vector fnmsb - predicated
1673 
1674 // dst_src1 = -src3 + dst_src1 * src2
1675 // The NegVF must not be predicated.
1676 instruct vfnmsbF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1677   predicate(UseFMA && UseSVE > 0 &&
1678             !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1679   match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary (NegVF src3) pg)));
1680   ins_cost(SVE_COST);
1681   format %{ "sve_fnmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) (S)" %}
1682   ins_encode %{
1683     __ sve_fnmsb(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
1684          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1685   %}
1686   ins_pipe(pipe_slow);
1687 %}
1688 
1689 // dst_src1 = -src3 + dst_src1 * src2
1690 // The NegVD must not be predicated.
1691 instruct vfnmsbD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1692   predicate(UseFMA && UseSVE > 0 &&
1693             !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1694   match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary (NegVD src3) pg)));
1695   ins_cost(SVE_COST);
1696   format %{ "sve_fnmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) (D)" %}
1697   ins_encode %{
1698     __ sve_fnmsb(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
1699          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1700   %}
1701   ins_pipe(pipe_slow);
1702 %}
1703 
1704 // vector mla
1705 
1706 // dst_src1 = dst_src1 + src2 * src3
1707 instruct vmlaB(vReg dst_src1, vReg src2, vReg src3)
1708 %{
1709   predicate(UseSVE > 0);
1710   match(Set dst_src1 (AddVB dst_src1 (MulVB src2 src3)));
1711   ins_cost(SVE_COST);
1712   format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (B)" %}
1713   ins_encode %{
1714     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ B,
1715       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1716   %}
1717   ins_pipe(pipe_slow);
1718 %}
1719 
1720 // dst_src1 = dst_src1 + src2 * src3
1721 instruct vmlaS(vReg dst_src1, vReg src2, vReg src3)
1722 %{
1723   predicate(UseSVE > 0);
1724   match(Set dst_src1 (AddVS dst_src1 (MulVS src2 src3)));
1725   ins_cost(SVE_COST);
1726   format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (H)" %}
1727   ins_encode %{
1728     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ H,
1729       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1730   %}
1731   ins_pipe(pipe_slow);
1732 %}
1733 
1734 // dst_src1 = dst_src1 + src2 * src3
1735 instruct vmlaI(vReg dst_src1, vReg src2, vReg src3)
1736 %{
1737   predicate(UseSVE > 0);
1738   match(Set dst_src1 (AddVI dst_src1 (MulVI src2 src3)));
1739   ins_cost(SVE_COST);
1740   format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (S)" %}
1741   ins_encode %{
1742     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ S,
1743       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1744   %}
1745   ins_pipe(pipe_slow);
1746 %}
1747 
1748 // dst_src1 = dst_src1 + src2 * src3
1749 instruct vmlaL(vReg dst_src1, vReg src2, vReg src3)
1750 %{
1751   predicate(UseSVE > 0);
1752   match(Set dst_src1 (AddVL dst_src1 (MulVL src2 src3)));
1753   ins_cost(SVE_COST);
1754   format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (D)" %}
1755   ins_encode %{
1756     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ D,
1757       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1758   %}
1759   ins_pipe(pipe_slow);
1760 %}
1761 
1762 // vector mla - predicated
1763 
1764 // dst_src1 = dst_src1 + src2 * src3
1765 instruct vmlaB_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1766 %{
1767   predicate(UseSVE > 0);
1768   match(Set dst_src1 (AddVB (Binary dst_src1 (MulVB src2 src3)) pg));
1769   ins_cost(SVE_COST);
1770   format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) (B)" %}
1771   ins_encode %{
1772     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ B, as_PRegister($pg$$reg),
1773          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1774   %}
1775   ins_pipe(pipe_slow);
1776 %}
1777 
1778 // dst_src1 = dst_src1 + src2 * src3
1779 instruct vmlaS_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1780 %{
1781   predicate(UseSVE > 0);
1782   match(Set dst_src1 (AddVS (Binary dst_src1 (MulVS src2 src3)) pg));
1783   ins_cost(SVE_COST);
1784   format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) (H)" %}
1785   ins_encode %{
1786     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ H, as_PRegister($pg$$reg),
1787          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1788   %}
1789   ins_pipe(pipe_slow);
1790 %}
1791 
1792 // dst_src1 = dst_src1 + src2 * src3
1793 instruct vmlaI_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1794 %{
1795   predicate(UseSVE > 0);
1796   match(Set dst_src1 (AddVI (Binary dst_src1 (MulVI src2 src3)) pg));
1797   ins_cost(SVE_COST);
1798   format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) (S)" %}
1799   ins_encode %{
1800     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
1801          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1802   %}
1803   ins_pipe(pipe_slow);
1804 %}
1805 
1806 // dst_src1 = dst_src1 + src2 * src3
1807 instruct vmlaL_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1808 %{
1809   predicate(UseSVE > 0);
1810   match(Set dst_src1 (AddVL (Binary dst_src1 (MulVL src2 src3)) pg));
1811   ins_cost(SVE_COST);
1812   format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) (D)" %}
1813   ins_encode %{
1814     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
1815          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1816   %}
1817   ins_pipe(pipe_slow);
1818 %}
1819 
1820 // vector mls
1821 
1822 // dst_src1 = dst_src1 - src2 * src3
1823 instruct vmlsB(vReg dst_src1, vReg src2, vReg src3)
1824 %{
1825   predicate(UseSVE > 0);
1826   match(Set dst_src1 (SubVB dst_src1 (MulVB src2 src3)));
1827   ins_cost(SVE_COST);
1828   format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (B)" %}
1829   ins_encode %{
1830     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ B,
1831       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1832   %}
1833   ins_pipe(pipe_slow);
1834 %}
1835 
1836 // dst_src1 = dst_src1 - src2 * src3
1837 instruct vmlsS(vReg dst_src1, vReg src2, vReg src3)
1838 %{
1839   predicate(UseSVE > 0);
1840   match(Set dst_src1 (SubVS dst_src1 (MulVS src2 src3)));
1841   ins_cost(SVE_COST);
1842   format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (H)" %}
1843   ins_encode %{
1844     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ H,
1845       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1846   %}
1847   ins_pipe(pipe_slow);
1848 %}
1849 
1850 // dst_src1 = dst_src1 - src2 * src3
1851 instruct vmlsI(vReg dst_src1, vReg src2, vReg src3)
1852 %{
1853   predicate(UseSVE > 0);
1854   match(Set dst_src1 (SubVI dst_src1 (MulVI src2 src3)));
1855   ins_cost(SVE_COST);
1856   format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (S)" %}
1857   ins_encode %{
1858     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ S,
1859       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1860   %}
1861   ins_pipe(pipe_slow);
1862 %}
1863 
1864 // dst_src1 = dst_src1 - src2 * src3
1865 instruct vmlsL(vReg dst_src1, vReg src2, vReg src3)
1866 %{
1867   predicate(UseSVE > 0);
1868   match(Set dst_src1 (SubVL dst_src1 (MulVL src2 src3)));
1869   ins_cost(SVE_COST);
1870   format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (D)" %}
1871   ins_encode %{
1872     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ D,
1873       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1874   %}
1875   ins_pipe(pipe_slow);
1876 %}
1877 
1878 // vector mls - predicated
1879 
1880 // dst_src1 = dst_src1 - src2 * src3
1881 instruct vmlsB_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1882 %{
1883   predicate(UseSVE > 0);
1884   match(Set dst_src1 (SubVB (Binary dst_src1 (MulVB src2 src3)) pg));
1885   ins_cost(SVE_COST);
1886   format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) (B)" %}
1887   ins_encode %{
1888     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ B, as_PRegister($pg$$reg),
1889          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1890   %}
1891   ins_pipe(pipe_slow);
1892 %}
1893 
1894 // dst_src1 = dst_src1 - src2 * src3
1895 instruct vmlsS_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1896 %{
1897   predicate(UseSVE > 0);
1898   match(Set dst_src1 (SubVS (Binary dst_src1 (MulVS src2 src3)) pg));
1899   ins_cost(SVE_COST);
1900   format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) (H)" %}
1901   ins_encode %{
1902     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ H, as_PRegister($pg$$reg),
1903          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1904   %}
1905   ins_pipe(pipe_slow);
1906 %}
1907 
1908 // dst_src1 = dst_src1 - src2 * src3
1909 instruct vmlsI_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1910 %{
1911   predicate(UseSVE > 0);
1912   match(Set dst_src1 (SubVI (Binary dst_src1 (MulVI src2 src3)) pg));
1913   ins_cost(SVE_COST);
1914   format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) (S)" %}
1915   ins_encode %{
1916     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
1917          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1918   %}
1919   ins_pipe(pipe_slow);
1920 %}
1921 
1922 // dst_src1 = dst_src1 - src2 * src3
1923 instruct vmlsL_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1924 %{
1925   predicate(UseSVE > 0);
1926   match(Set dst_src1 (SubVL (Binary dst_src1 (MulVL src2 src3)) pg));
1927   ins_cost(SVE_COST);
1928   format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) (D)" %}
1929   ins_encode %{
1930     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
1931          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1932   %}
1933   ins_pipe(pipe_slow);
1934 %}
1935 
1936 // vector mul
1937 
1938 instruct vmulB(vReg dst_src1, vReg src2) %{
1939   predicate(UseSVE > 0);
1940   match(Set dst_src1 (MulVB dst_src1 src2));
1941   ins_cost(SVE_COST);
1942   format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (B)" %}
1943   ins_encode %{
1944     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ B,
1945          ptrue, as_FloatRegister($src2$$reg));
1946   %}
1947   ins_pipe(pipe_slow);
1948 %}
1949 
1950 instruct vmulS(vReg dst_src1, vReg src2) %{
1951   predicate(UseSVE > 0);
1952   match(Set dst_src1 (MulVS dst_src1 src2));
1953   ins_cost(SVE_COST);
1954   format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (H)" %}
1955   ins_encode %{
1956     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ H,
1957          ptrue, as_FloatRegister($src2$$reg));
1958   %}
1959   ins_pipe(pipe_slow);
1960 %}
1961 
1962 instruct vmulI(vReg dst_src1, vReg src2) %{
1963   predicate(UseSVE > 0);
1964   match(Set dst_src1 (MulVI dst_src1 src2));
1965   ins_cost(SVE_COST);
1966   format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (S)" %}
1967   ins_encode %{
1968     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ S,
1969          ptrue, as_FloatRegister($src2$$reg));
1970   %}
1971   ins_pipe(pipe_slow);
1972 %}
1973 
1974 instruct vmulL(vReg dst_src1, vReg src2) %{
1975   predicate(UseSVE > 0);
1976   match(Set dst_src1 (MulVL dst_src1 src2));
1977   ins_cost(SVE_COST);
1978   format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (D)" %}
1979   ins_encode %{
1980     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ D,
1981          ptrue, as_FloatRegister($src2$$reg));
1982   %}
1983   ins_pipe(pipe_slow);
1984 %}
1985 
1986 instruct vmulF(vReg dst, vReg src1, vReg src2) %{
1987   predicate(UseSVE > 0);
1988   match(Set dst (MulVF src1 src2));
1989   ins_cost(SVE_COST);
1990   format %{ "sve_fmul $dst, $src1, $src2\t # vector (sve) (S)" %}
1991   ins_encode %{
1992     __ sve_fmul(as_FloatRegister($dst$$reg), __ S,
1993          as_FloatRegister($src1$$reg),
1994          as_FloatRegister($src2$$reg));
1995   %}
1996   ins_pipe(pipe_slow);
1997 %}
1998 
1999 instruct vmulD(vReg dst, vReg src1, vReg src2) %{
2000   predicate(UseSVE > 0);
2001   match(Set dst (MulVD src1 src2));
2002   ins_cost(SVE_COST);
2003   format %{ "sve_fmul $dst, $src1, $src2\t # vector (sve) (D)" %}
2004   ins_encode %{
2005     __ sve_fmul(as_FloatRegister($dst$$reg), __ D,
2006          as_FloatRegister($src1$$reg),
2007          as_FloatRegister($src2$$reg));
2008   %}
2009   ins_pipe(pipe_slow);
2010 %}
2011 
2012 // vector mul - predicated
2013 
2014 instruct vmulB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
2015   predicate(UseSVE > 0);
2016   match(Set dst_src1 (MulVB (Binary dst_src1 src2) pg));
2017   ins_cost(SVE_COST);
2018   format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
2019   ins_encode %{
2020     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ B,
2021             as_PRegister($pg$$reg),
2022             as_FloatRegister($src2$$reg));
2023   %}
2024   ins_pipe(pipe_slow);
2025 %}
2026 
2027 instruct vmulS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
2028   predicate(UseSVE > 0);
2029   match(Set dst_src1 (MulVS (Binary dst_src1 src2) pg));
2030   ins_cost(SVE_COST);
2031   format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
2032   ins_encode %{
2033     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ H,
2034             as_PRegister($pg$$reg),
2035             as_FloatRegister($src2$$reg));
2036   %}
2037   ins_pipe(pipe_slow);
2038 %}
2039 
2040 instruct vmulI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
2041   predicate(UseSVE > 0);
2042   match(Set dst_src1 (MulVI (Binary dst_src1 src2) pg));
2043   ins_cost(SVE_COST);
2044   format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
2045   ins_encode %{
2046     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ S,
2047             as_PRegister($pg$$reg),
2048             as_FloatRegister($src2$$reg));
2049   %}
2050   ins_pipe(pipe_slow);
2051 %}
2052 
2053 instruct vmulL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
2054   predicate(UseSVE > 0);
2055   match(Set dst_src1 (MulVL (Binary dst_src1 src2) pg));
2056   ins_cost(SVE_COST);
2057   format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
2058   ins_encode %{
2059     __ sve_mul(as_FloatRegister($dst_src1$$reg), __ D,
2060             as_PRegister($pg$$reg),
2061             as_FloatRegister($src2$$reg));
2062   %}
2063   ins_pipe(pipe_slow);
2064 %}
2065 
2066 instruct vmulF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
2067   predicate(UseSVE > 0);
2068   match(Set dst_src1 (MulVF (Binary dst_src1 src2) pg));
2069   ins_cost(SVE_COST);
2070   format %{ "sve_fmul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
2071   ins_encode %{
2072     __ sve_fmul(as_FloatRegister($dst_src1$$reg), __ S,
2073             as_PRegister($pg$$reg),
2074             as_FloatRegister($src2$$reg));
2075   %}
2076   ins_pipe(pipe_slow);
2077 %}
2078 
2079 instruct vmulD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
2080   predicate(UseSVE > 0);
2081   match(Set dst_src1 (MulVD (Binary dst_src1 src2) pg));
2082   ins_cost(SVE_COST);
2083   format %{ "sve_fmul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
2084   ins_encode %{
2085     __ sve_fmul(as_FloatRegister($dst_src1$$reg), __ D,
2086             as_PRegister($pg$$reg),
2087             as_FloatRegister($src2$$reg));
2088   %}
2089   ins_pipe(pipe_slow);
2090 %}
2091 
2092 // vector neg
2093 
2094 instruct vnegI(vReg dst, vReg src) %{
2095   predicate(UseSVE > 0 &&
2096             !n->as_Vector()->is_predicated_vector());
2097   match(Set dst (NegVI src));
2098   ins_cost(SVE_COST);
2099   format %{ "sve_neg $dst, $src\t# vector (sve) (B/H/S)" %}
2100   ins_encode %{
2101     BasicType bt = Matcher::vector_element_basic_type(this);
2102     __ sve_neg(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
2103          ptrue, as_FloatRegister($src$$reg));
2104   %}
2105   ins_pipe(pipe_slow);
2106 %}
2107 
2108 instruct vnegL(vReg dst, vReg src) %{
2109   predicate(UseSVE > 0 &&
2110             !n->as_Vector()->is_predicated_vector());
2111   match(Set dst (NegVL src));
2112   ins_cost(SVE_COST);
2113   format %{ "sve_neg $dst, $src\t# vector (sve) (D)" %}
2114   ins_encode %{
2115     __ sve_neg(as_FloatRegister($dst$$reg), __ D,
2116          ptrue, as_FloatRegister($src$$reg));
2117   %}
2118   ins_pipe(pipe_slow);
2119 %}
2120 
2121 instruct vnegF(vReg dst, vReg src) %{
2122   predicate(UseSVE > 0 &&
2123             !n->as_Vector()->is_predicated_vector());
2124   match(Set dst (NegVF src));
2125   ins_cost(SVE_COST);
2126   format %{ "sve_fneg $dst, $src\t# vector (sve) (S)" %}
2127   ins_encode %{
2128     __ sve_fneg(as_FloatRegister($dst$$reg), __ S,
2129          ptrue, as_FloatRegister($src$$reg));
2130   %}
2131   ins_pipe(pipe_slow);
2132 %}
2133 
2134 instruct vnegD(vReg dst, vReg src) %{
2135   predicate(UseSVE > 0 &&
2136             !n->as_Vector()->is_predicated_vector());
2137   match(Set dst (NegVD src));
2138   ins_cost(SVE_COST);
2139   format %{ "sve_fneg $dst, $src\t# vector (sve) (D)" %}
2140   ins_encode %{
2141     __ sve_fneg(as_FloatRegister($dst$$reg), __ D,
2142          ptrue, as_FloatRegister($src$$reg));
2143   %}
2144   ins_pipe(pipe_slow);
2145 %}
2146 
2147 // vector neg - predicated
2148 
2149 instruct vnegI_masked(vReg dst_src, pRegGov pg) %{
2150   predicate(UseSVE > 0);
2151   match(Set dst_src (NegVI dst_src pg));
2152   ins_cost(SVE_COST);
2153   format %{ "sve_neg $dst_src, $pg, $dst_src\t# vector (sve) (B/H/S)" %}
2154   ins_encode %{
2155     BasicType bt = Matcher::vector_element_basic_type(this);
2156     __ sve_neg(as_FloatRegister($dst_src$$reg), __ elemType_to_regVariant(bt),
2157             as_PRegister($pg$$reg),
2158             as_FloatRegister($dst_src$$reg));
2159   %}
2160   ins_pipe(pipe_slow);
2161 %}
2162 
2163 instruct vnegL_masked(vReg dst_src, pRegGov pg) %{
2164   predicate(UseSVE > 0);
2165   match(Set dst_src (NegVL dst_src pg));
2166   ins_cost(SVE_COST);
2167   format %{ "sve_neg $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
2168   ins_encode %{
2169     __ sve_neg(as_FloatRegister($dst_src$$reg), __ D,
2170             as_PRegister($pg$$reg),
2171             as_FloatRegister($dst_src$$reg));
2172   %}
2173   ins_pipe(pipe_slow);
2174 %}
2175 
2176 instruct vnegF_masked(vReg dst_src, pRegGov pg) %{
2177   predicate(UseSVE > 0);
2178   match(Set dst_src (NegVF dst_src pg));
2179   ins_cost(SVE_COST);
2180   format %{ "sve_fneg $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
2181   ins_encode %{
2182     __ sve_fneg(as_FloatRegister($dst_src$$reg), __ S,
2183             as_PRegister($pg$$reg),
2184             as_FloatRegister($dst_src$$reg));
2185   %}
2186   ins_pipe(pipe_slow);
2187 %}
2188 
2189 instruct vnegD_masked(vReg dst_src, pRegGov pg) %{
2190   predicate(UseSVE > 0);
2191   match(Set dst_src (NegVD dst_src pg));
2192   ins_cost(SVE_COST);
2193   format %{ "sve_fneg $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
2194   ins_encode %{
2195     __ sve_fneg(as_FloatRegister($dst_src$$reg), __ D,
2196             as_PRegister($pg$$reg),
2197             as_FloatRegister($dst_src$$reg));
2198   %}
2199   ins_pipe(pipe_slow);
2200 %}
2201 
2202 // vector popcount
2203 
2204 instruct vpopcountI(vReg dst, vReg src) %{
2205   predicate(UseSVE > 0 &&
2206             !n->as_Vector()->is_predicated_vector());
2207   match(Set dst (PopCountVI src));
2208   ins_cost(SVE_COST);
2209   format %{ "sve_cnt $dst, $src\t# vector (sve) (B/H/S)" %}
2210   ins_encode %{
2211     assert(UsePopCountInstruction, "unsupported");
2212     BasicType bt = Matcher::vector_element_basic_type(this);
2213     __ sve_cnt(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
2214          ptrue, as_FloatRegister($src$$reg));
2215   %}
2216   ins_pipe(pipe_slow);
2217 %}
2218 
2219 instruct vpopcountL(vReg dst, vReg src) %{
2220   predicate(UseSVE > 0 &&
2221             !n->as_Vector()->is_predicated_vector() &&
2222             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2223   match(Set dst (PopCountVL src));
2224   ins_cost(SVE_COST);
2225   format %{ "sve_cnt $dst, $src\t# vector (sve) (D)" %}
2226   ins_encode %{
2227     assert(UsePopCountInstruction, "unsupported");
2228     __ sve_cnt(as_FloatRegister($dst$$reg), __ D,
2229          ptrue, as_FloatRegister($src$$reg));
2230   %}
2231   ins_pipe(pipe_slow);
2232 %}
2233 
2234 // If the PopCountVL is generated by auto-vectorization, the dst basic
2235 // type is T_INT. And once we have unified the type definition for
2236 // Vector API and auto-vectorization, this rule can be merged with
2237 // "vpopcountL" rule.
2238 instruct vpopcountLI(vReg dst, vReg src, vReg vtmp) %{
2239   predicate(UseSVE > 0 &&
2240             !n->as_Vector()->is_predicated_vector() &&
2241             n->bottom_type()->is_vect()->element_basic_type() == T_INT);
2242   match(Set dst (PopCountVL src));
2243   effect(TEMP_DEF dst, TEMP vtmp);
2244   ins_cost(3 * SVE_COST);
2245   format %{ "sve_cnt $dst, $src\n\t"
2246             "sve_dup $vtmp, #0\n\t"
2247             "sve_uzp1 $dst, $dst, $vtmp\t# vector (sve) (S)" %}
2248   ins_encode %{
2249     assert(UsePopCountInstruction, "unsupported");
2250     __ sve_cnt(as_FloatRegister($dst$$reg), __ D,
2251          ptrue, as_FloatRegister($src$$reg));
2252     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
2253          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($vtmp$$reg));
2254   %}
2255   ins_pipe(pipe_slow);
2256 %}
2257 
2258 // vector popcount - predicated
2259 
2260 instruct vpopcountI_masked(vReg dst_src, pRegGov pg) %{
2261   predicate(UseSVE > 0);
2262   match(Set dst_src (PopCountVI dst_src pg));
2263   ins_cost(SVE_COST);
2264   format %{ "sve_cnt $dst_src, $pg, $dst_src\t# vector (sve) (B/H/S)" %}
2265   ins_encode %{
2266     assert(UsePopCountInstruction, "unsupported");
2267     BasicType bt = Matcher::vector_element_basic_type(this);
2268     __ sve_cnt(as_FloatRegister($dst_src$$reg), __ elemType_to_regVariant(bt),
2269          as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
2270   %}
2271   ins_pipe(pipe_slow);
2272 %}
2273 
2274 instruct vpopcountL_masked(vReg dst_src, pRegGov pg) %{
2275   predicate(UseSVE > 0 &&
2276             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2277   match(Set dst_src (PopCountVL dst_src pg));
2278   ins_cost(SVE_COST);
2279   format %{ "sve_cnt $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
2280   ins_encode %{
2281     assert(UsePopCountInstruction, "unsupported");
2282     __ sve_cnt(as_FloatRegister($dst_src$$reg), __ D,
2283          as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
2284   %}
2285   ins_pipe(pipe_slow);
2286 %}
2287 
2288 // vector blend
2289 
2290 instruct vblend(vReg dst, vReg src1, vReg src2, pRegGov pg) %{
2291   predicate(UseSVE > 0);
2292   match(Set dst (VectorBlend (Binary src1 src2) pg));
2293   ins_cost(SVE_COST);
2294   format %{ "sve_sel $dst, $pg, $src2, $src1\t# vector blend (sve)" %}
2295   ins_encode %{
2296     Assembler::SIMD_RegVariant size =
2297                __ elemType_to_regVariant(Matcher::vector_element_basic_type(this));
2298     __ sve_sel(as_FloatRegister($dst$$reg), size, as_PRegister($pg$$reg),
2299                as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
2300   %}
2301   ins_pipe(pipe_slow);
2302 %}
2303 
2304 // vector store mask
2305 
2306 instruct vstoremaskB(vReg dst, pRegGov src, immI_1 size) %{
2307   predicate(UseSVE > 0);
2308   match(Set dst (VectorStoreMask src size));
2309   ins_cost(SVE_COST);
2310   format %{ "vstoremask $dst, $src\t# vector store mask (sve) (B)" %}
2311   ins_encode %{
2312     __ sve_cpy(as_FloatRegister($dst$$reg), __ B, as_PRegister($src$$reg), 1, false);
2313   %}
2314   ins_pipe(pipe_slow);
2315 %}
2316 
2317 instruct vstoremask_narrow(vReg dst, pRegGov src, vReg tmp, immI_gt_1 size) %{
2318   predicate(UseSVE > 0);
2319   match(Set dst (VectorStoreMask src size));
2320   effect(TEMP_DEF dst, TEMP tmp);
2321   ins_cost(3 * SVE_COST);
2322   format %{ "vstoremask $dst, $src\t# vector store mask (sve) (H/S/D)" %}
2323   ins_encode %{
2324     Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant((int)$size$$constant);
2325     __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($src$$reg), 1, false);
2326     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
2327                          as_FloatRegister($dst$$reg), size, as_FloatRegister($tmp$$reg));
2328   %}
2329   ins_pipe(pipe_slow);
2330 %}
2331 
2332 // Combine LoadVector+VectorLoadMask when the vector element type is not T_BYTE
2333 
2334 instruct vloadmask_loadV(pRegGov dst, indirect mem, vReg tmp, rFlagsReg cr) %{
2335   predicate(UseSVE > 0 &&
2336             n->as_Vector()->length_in_bytes() == MaxVectorSize &&
2337             type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
2338   match(Set dst (VectorLoadMask (LoadVector mem)));
2339   effect(TEMP tmp, KILL cr);
2340   ins_cost(3 * SVE_COST);
2341   format %{ "sve_ld1b $tmp, $mem\n\t"
2342             "sve_cmpne $dst, $tmp, 0\t# load vector mask (sve) (H/S/D)" %}
2343   ins_encode %{
2344     // Load mask values which are boolean type, and extend them to the
2345     // expected vector element type. Convert the vector to predicate.
2346     BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
2347     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($tmp$$reg),
2348                           ptrue, T_BOOLEAN, to_vect_bt, $mem->opcode(),
2349                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2350     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ elemType_to_regVariant(to_vect_bt),
2351                ptrue, as_FloatRegister($tmp$$reg), 0);
2352   %}
2353   ins_pipe(pipe_slow);
2354 %}
2355 
2356 instruct vloadmask_loadV_partial(pRegGov dst, indirect mem, vReg vtmp, pRegGov ptmp, rFlagsReg cr) %{
2357   predicate(UseSVE > 0 &&
2358             n->as_Vector()->length_in_bytes() > 16 &&
2359             n->as_Vector()->length_in_bytes() < MaxVectorSize &&
2360             type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
2361   match(Set dst (VectorLoadMask (LoadVector mem)));
2362   effect(TEMP vtmp, TEMP ptmp, KILL cr);
2363   ins_cost(6 * SVE_COST);
2364   format %{ "vloadmask_loadV $dst, $mem\t# load vector mask partial (sve) (H/S/D)" %}
2365   ins_encode %{
2366     // Load valid mask values which are boolean type, and extend them to the
2367     // expected vector element type. Convert the vector to predicate.
2368     BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
2369     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(to_vect_bt);
2370     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this));
2371     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($vtmp$$reg),
2372                           as_PRegister($ptmp$$reg), T_BOOLEAN, to_vect_bt, $mem->opcode(),
2373                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2374     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($vtmp$$reg), 0);
2375   %}
2376   ins_pipe(pipe_slow);
2377 %}
2378 
2379 // Combine VectorStoreMask+StoreVector when the vector element type is not T_BYTE
2380 
2381 instruct storeV_vstoremask(indirect mem, pRegGov src, vReg tmp, immI_gt_1 esize) %{
2382   predicate(UseSVE > 0 &&
2383             Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
2384   match(Set mem (StoreVector mem (VectorStoreMask src esize)));
2385   effect(TEMP tmp);
2386   ins_cost(3 * SVE_COST);
2387   format %{ "sve_cpy $tmp, $src, 1\n\t"
2388             "sve_st1b $tmp, $mem\t# store vector mask (sve) (H/S/D)" %}
2389   ins_encode %{
2390     BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
2391     assert(type2aelembytes(from_vect_bt) == (int)$esize$$constant, "unsupported type.");
2392     Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
2393     __ sve_cpy(as_FloatRegister($tmp$$reg), size, as_PRegister($src$$reg), 1, false);
2394     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($tmp$$reg),
2395                           ptrue, T_BOOLEAN, from_vect_bt, $mem->opcode(),
2396                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2397   %}
2398   ins_pipe(pipe_slow);
2399 %}
2400 
2401 instruct storeV_vstoremask_partial(indirect mem, pRegGov src, vReg vtmp,
2402                                    immI_gt_1 esize, pRegGov ptmp, rFlagsReg cr) %{
2403   predicate(UseSVE > 0 &&
2404             n->as_StoreVector()->memory_size() > 16 &&
2405             type2aelembytes(n->as_StoreVector()->vect_type()->element_basic_type()) > 1 &&
2406             Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
2407   match(Set mem (StoreVector mem (VectorStoreMask src esize)));
2408   effect(TEMP vtmp, TEMP ptmp, KILL cr);
2409   format %{ "storeV_vstoremask $src, $mem\t# store vector mask partial (sve) (H/S/D)" %}
2410   ins_cost(6 * SVE_COST);
2411   ins_encode %{
2412     // Convert the valid src predicate to vector, and store the vector
2413     // elements as boolean values.
2414     BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
2415     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(from_vect_bt);
2416     __ sve_cpy(as_FloatRegister($vtmp$$reg), size, as_PRegister($src$$reg), 1, false);
2417     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
2418     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($vtmp$$reg),
2419                           as_PRegister($ptmp$$reg), T_BOOLEAN, from_vect_bt, $mem->opcode(),
2420                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2421   %}
2422   ins_pipe(pipe_slow);
2423 %}
2424 
2425 // vector add reduction
2426 
2427 instruct reduce_addI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
2428   predicate(UseSVE > 0 &&
2429             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2430   match(Set dst (AddReductionVI src1 src2));
2431   effect(TEMP_DEF dst, TEMP tmp);
2432   ins_cost(SVE_COST);
2433   format %{ "sve_reduce_addI $dst, $src1, $src2\t# addI reduction (sve) (may extend)" %}
2434   ins_encode %{
2435     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2436     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2437                            $src1$$Register, as_FloatRegister($src2$$reg),
2438                            ptrue, as_FloatRegister($tmp$$reg));
2439   %}
2440   ins_pipe(pipe_slow);
2441 %}
2442 
2443 instruct reduce_addL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
2444   predicate(UseSVE > 0 &&
2445             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2446   match(Set dst (AddReductionVL src1 src2));
2447   effect(TEMP_DEF dst, TEMP tmp);
2448   ins_cost(SVE_COST);
2449   format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction (sve)" %}
2450   ins_encode %{
2451     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2452                            $src1$$Register, as_FloatRegister($src2$$reg),
2453                            ptrue, as_FloatRegister($tmp$$reg));
2454   %}
2455   ins_pipe(pipe_slow);
2456 %}
2457 
2458 instruct reduce_addF(vRegF src1_dst, vReg src2) %{
2459   predicate(UseSVE > 0 &&
2460             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2461   match(Set src1_dst (AddReductionVF src1_dst src2));
2462   ins_cost(SVE_COST);
2463   format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (S)" %}
2464   ins_encode %{
2465     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
2466          ptrue, as_FloatRegister($src2$$reg));
2467   %}
2468   ins_pipe(pipe_slow);
2469 %}
2470 
2471 instruct reduce_addD(vRegD src1_dst, vReg src2) %{
2472   predicate(UseSVE > 0 &&
2473             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2474   match(Set src1_dst (AddReductionVD src1_dst src2));
2475   ins_cost(SVE_COST);
2476   format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (D)" %}
2477   ins_encode %{
2478     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
2479          ptrue, as_FloatRegister($src2$$reg));
2480   %}
2481   ins_pipe(pipe_slow);
2482 %}
2483 
2484 instruct reduce_addI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2485                              pRegGov ptmp, rFlagsReg cr) %{
2486   predicate(UseSVE > 0 &&
2487             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2488   match(Set dst (AddReductionVI src1 src2));
2489   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2490   ins_cost(2 * SVE_COST);
2491   format %{ "sve_reduce_addI $dst, $src1, $src2\t# addI reduction partial (sve) (may extend)" %}
2492   ins_encode %{
2493     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2494     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2495     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
2496     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2497                            $src1$$Register, as_FloatRegister($src2$$reg),
2498                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2499   %}
2500   ins_pipe(pipe_slow);
2501 %}
2502 
2503 instruct reduce_addL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2504                              pRegGov ptmp, rFlagsReg cr) %{
2505   predicate(UseSVE > 0 &&
2506             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2507   match(Set dst (AddReductionVL src1 src2));
2508   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2509   ins_cost(2 * SVE_COST);
2510   format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction partial (sve)" %}
2511   ins_encode %{
2512     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
2513     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2514                            $src1$$Register, as_FloatRegister($src2$$reg),
2515                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2516   %}
2517   ins_pipe(pipe_slow);
2518 %}
2519 
2520 instruct reduce_addF_partial(vRegF src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
2521   predicate(UseSVE > 0 &&
2522             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2523   match(Set src1_dst (AddReductionVF src1_dst src2));
2524   ins_cost(SVE_COST);
2525   effect(TEMP ptmp, KILL cr);
2526   format %{ "sve_reduce_addF $src1_dst, $src1_dst, $src2\t# addF reduction partial (sve) (S)" %}
2527   ins_encode %{
2528     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src2));
2529     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
2530                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2531   %}
2532   ins_pipe(pipe_slow);
2533 %}
2534 
2535 instruct reduce_addD_partial(vRegD src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
2536   predicate(UseSVE > 0 &&
2537             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2538   match(Set src1_dst (AddReductionVD src1_dst src2));
2539   ins_cost(SVE_COST);
2540   effect(TEMP ptmp, KILL cr);
2541   format %{ "sve_reduce_addD $src1_dst, $src1_dst, $src2\t# addD reduction partial (sve) (D)" %}
2542   ins_encode %{
2543     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
2544     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
2545                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2546   %}
2547   ins_pipe(pipe_slow);
2548 %}
2549 
2550 // vector add reduction - predicated
2551 
2552 instruct reduce_addI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2553   predicate(UseSVE > 0);
2554   match(Set dst (AddReductionVI (Binary src1 src2) pg));
2555   effect(TEMP_DEF dst, TEMP tmp);
2556   ins_cost(SVE_COST);
2557   format %{ "sve_reduce_addI $dst, $src1, $pg, $src2\t# addI reduction predicated (sve) (may extend)" %}
2558   ins_encode %{
2559     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2560     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2561                            $src1$$Register, as_FloatRegister($src2$$reg),
2562                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2563   %}
2564   ins_pipe(pipe_slow);
2565 %}
2566 
2567 instruct reduce_addL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2568   predicate(UseSVE > 0);
2569   match(Set dst (AddReductionVL (Binary src1 src2) pg));
2570   effect(TEMP_DEF dst, TEMP tmp);
2571   ins_cost(SVE_COST);
2572   format %{ "sve_reduce_addL $dst, $src1, $pg, $src2\t# addL reduction predicated (sve)" %}
2573   ins_encode %{
2574     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2575                            $src1$$Register, as_FloatRegister($src2$$reg),
2576                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2577   %}
2578   ins_pipe(pipe_slow);
2579 %}
2580 
2581 instruct reduce_addF_masked(vRegF src1_dst, vReg src2, pRegGov pg) %{
2582   predicate(UseSVE > 0);
2583   match(Set src1_dst (AddReductionVF (Binary src1_dst src2) pg));
2584   ins_cost(SVE_COST);
2585   format %{ "sve_reduce_addF $src1_dst, $pg, $src2\t# addF reduction predicated (sve)" %}
2586   ins_encode %{
2587     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
2588                  as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
2589   %}
2590   ins_pipe(pipe_slow);
2591 %}
2592 
2593 instruct reduce_addD_masked(vRegD src1_dst, vReg src2, pRegGov pg) %{
2594   predicate(UseSVE > 0);
2595   match(Set src1_dst (AddReductionVD (Binary src1_dst src2) pg));
2596   ins_cost(SVE_COST);
2597   format %{ "sve_reduce_addD $src1_dst, $pg, $src2\t# addD reduction predicated (sve)" %}
2598   ins_encode %{
2599     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
2600                  as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
2601   %}
2602   ins_pipe(pipe_slow);
2603 %}
2604 
2605 // vector and reduction
2606 
2607 instruct reduce_andI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
2608   predicate(UseSVE > 0 &&
2609             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2610             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2611   match(Set dst (AndReductionV src1 src2));
2612   effect(TEMP_DEF dst, TEMP tmp);
2613   ins_cost(SVE_COST);
2614   format %{ "sve_reduce_andI $dst, $src1, $src2\t# andI reduction (sve) (may extend)" %}
2615   ins_encode %{
2616     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2617     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2618                            $src1$$Register, as_FloatRegister($src2$$reg),
2619                            ptrue, as_FloatRegister($tmp$$reg));
2620   %}
2621   ins_pipe(pipe_slow);
2622 %}
2623 
2624 instruct reduce_andL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
2625   predicate(UseSVE > 0 &&
2626             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2627             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2628   match(Set dst (AndReductionV src1 src2));
2629   effect(TEMP_DEF dst, TEMP tmp);
2630   ins_cost(SVE_COST);
2631   format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction (sve)" %}
2632   ins_encode %{
2633     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2634                            $src1$$Register, as_FloatRegister($src2$$reg),
2635                            ptrue, as_FloatRegister($tmp$$reg));
2636   %}
2637   ins_pipe(pipe_slow);
2638 %}
2639 
2640 instruct reduce_andI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2641                              pRegGov ptmp, rFlagsReg cr) %{
2642   predicate(UseSVE > 0 &&
2643             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2644             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2645   match(Set dst (AndReductionV src1 src2));
2646   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2647   ins_cost(2 * SVE_COST);
2648   format %{ "sve_reduce_andI $dst, $src1, $src2\t# andI reduction partial (sve) (may extend)" %}
2649   ins_encode %{
2650     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2651     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2652     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
2653     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2654                            $src1$$Register, as_FloatRegister($src2$$reg),
2655                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2656   %}
2657   ins_pipe(pipe_slow);
2658 %}
2659 
2660 instruct reduce_andL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2661                              pRegGov ptmp, rFlagsReg cr) %{
2662   predicate(UseSVE > 0 &&
2663             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2664             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2665   match(Set dst (AndReductionV src1 src2));
2666   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2667   ins_cost(2 * SVE_COST);
2668   format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction partial (sve)" %}
2669   ins_encode %{
2670     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
2671     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2672                            $src1$$Register, as_FloatRegister($src2$$reg),
2673                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2674   %}
2675   ins_pipe(pipe_slow);
2676 %}
2677 
2678 // vector and reduction - predicated
2679 
2680 instruct reduce_andI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2681   predicate(UseSVE > 0 &&
2682             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG);
2683   match(Set dst (AndReductionV (Binary src1 src2) pg));
2684   effect(TEMP_DEF dst, TEMP tmp);
2685   ins_cost(SVE_COST);
2686   format %{ "sve_reduce_andI $dst, $src1, $pg, $src2\t# andI reduction predicated (sve) (may extend)" %}
2687   ins_encode %{
2688     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2689     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2690                            $src1$$Register, as_FloatRegister($src2$$reg),
2691                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2692   %}
2693   ins_pipe(pipe_slow);
2694 %}
2695 
2696 instruct reduce_andL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2697   predicate(UseSVE > 0 &&
2698             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2699   match(Set dst (AndReductionV (Binary src1 src2) pg));
2700   effect(TEMP_DEF dst, TEMP tmp);
2701   ins_cost(SVE_COST);
2702   format %{ "sve_reduce_andL $dst, $src1, $pg, $src2\t# andL reduction predicated (sve)" %}
2703   ins_encode %{
2704     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2705                            $src1$$Register, as_FloatRegister($src2$$reg),
2706                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2707   %}
2708   ins_pipe(pipe_slow);
2709 %}
2710 
2711 // vector or reduction
2712 
2713 instruct reduce_orI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
2714   predicate(UseSVE > 0 &&
2715             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2716             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2717   match(Set dst (OrReductionV src1 src2));
2718   effect(TEMP_DEF dst, TEMP tmp);
2719   ins_cost(SVE_COST);
2720   format %{ "sve_reduce_orI $dst, $src1, $src2\t# orI reduction (sve) (may extend)" %}
2721   ins_encode %{
2722     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2723     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2724                            $src1$$Register, as_FloatRegister($src2$$reg),
2725                            ptrue, as_FloatRegister($tmp$$reg));
2726   %}
2727   ins_pipe(pipe_slow);
2728 %}
2729 
2730 instruct reduce_orL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
2731   predicate(UseSVE > 0 &&
2732             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2733             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2734   match(Set dst (OrReductionV src1 src2));
2735   effect(TEMP_DEF dst, TEMP tmp);
2736   ins_cost(SVE_COST);
2737   format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction (sve)" %}
2738   ins_encode %{
2739     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2740                            $src1$$Register, as_FloatRegister($src2$$reg),
2741                            ptrue, as_FloatRegister($tmp$$reg));
2742   %}
2743   ins_pipe(pipe_slow);
2744 %}
2745 
2746 instruct reduce_orI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2747                              pRegGov ptmp, rFlagsReg cr) %{
2748   predicate(UseSVE > 0 &&
2749             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2750             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2751   match(Set dst (OrReductionV src1 src2));
2752   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2753   ins_cost(2 * SVE_COST);
2754   format %{ "sve_reduce_orI $dst, $src1, $src2\t# orI reduction partial (sve) (may extend)" %}
2755   ins_encode %{
2756     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2757     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2758     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
2759     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2760                            $src1$$Register, as_FloatRegister($src2$$reg),
2761                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2762   %}
2763   ins_pipe(pipe_slow);
2764 %}
2765 
2766 instruct reduce_orL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2767                              pRegGov ptmp, rFlagsReg cr) %{
2768   predicate(UseSVE > 0 &&
2769             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2770             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2771   match(Set dst (OrReductionV src1 src2));
2772   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2773   ins_cost(2 * SVE_COST);
2774   format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction partial (sve)" %}
2775   ins_encode %{
2776     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
2777     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2778                            $src1$$Register, as_FloatRegister($src2$$reg),
2779                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2780   %}
2781   ins_pipe(pipe_slow);
2782 %}
2783 
2784 // vector or reduction - predicated
2785 
2786 instruct reduce_orI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2787   predicate(UseSVE > 0 &&
2788             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG);
2789   match(Set dst (OrReductionV (Binary src1 src2) pg));
2790   effect(TEMP_DEF dst, TEMP tmp);
2791   ins_cost(SVE_COST);
2792   format %{ "sve_reduce_orI $dst, $src1, $pg, $src2\t# orI reduction predicated (sve) (may extend)" %}
2793   ins_encode %{
2794     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2795     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2796                            $src1$$Register, as_FloatRegister($src2$$reg),
2797                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2798   %}
2799   ins_pipe(pipe_slow);
2800 %}
2801 
2802 instruct reduce_orL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2803   predicate(UseSVE > 0 &&
2804             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2805   match(Set dst (OrReductionV (Binary src1 src2) pg));
2806   effect(TEMP_DEF dst, TEMP tmp);
2807   ins_cost(SVE_COST);
2808   format %{ "sve_reduce_orL $dst, $src1, $pg, $src2\t# orL reduction predicated (sve)" %}
2809   ins_encode %{
2810     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2811                            $src1$$Register, as_FloatRegister($src2$$reg),
2812                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2813   %}
2814   ins_pipe(pipe_slow);
2815 %}
2816 
2817 // vector xor reduction
2818 
2819 instruct reduce_eorI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
2820   predicate(UseSVE > 0 &&
2821             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2822             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2823   match(Set dst (XorReductionV src1 src2));
2824   effect(TEMP_DEF dst, TEMP tmp);
2825   ins_cost(SVE_COST);
2826   format %{ "sve_reduce_eorI $dst, $src1, $src2\t# eorI reduction (sve) (may extend)" %}
2827   ins_encode %{
2828     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2829     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2830                            $src1$$Register, as_FloatRegister($src2$$reg),
2831                            ptrue, as_FloatRegister($tmp$$reg));
2832   %}
2833   ins_pipe(pipe_slow);
2834 %}
2835 
2836 instruct reduce_eorL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
2837   predicate(UseSVE > 0 &&
2838             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2839             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2840   match(Set dst (XorReductionV src1 src2));
2841   effect(TEMP_DEF dst, TEMP tmp);
2842   ins_cost(SVE_COST);
2843   format %{ "sve_reduce_eorL $dst, $src1, $src2\t# eorL reduction (sve)" %}
2844   ins_encode %{
2845     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2846                            $src1$$Register, as_FloatRegister($src2$$reg),
2847                            ptrue, as_FloatRegister($tmp$$reg));
2848   %}
2849   ins_pipe(pipe_slow);
2850 %}
2851 
2852 instruct reduce_eorI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2853                              pRegGov ptmp, rFlagsReg cr) %{
2854   predicate(UseSVE > 0 &&
2855             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2856             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2857   match(Set dst (XorReductionV src1 src2));
2858   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2859   ins_cost(2 * SVE_COST);
2860   format %{ "sve_reduce_eorI $dst, $src1, $src2\t# eorI reduction partial (sve) (may extend)" %}
2861   ins_encode %{
2862     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2863     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2864     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
2865     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2866                            $src1$$Register, as_FloatRegister($src2$$reg),
2867                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2868   %}
2869   ins_pipe(pipe_slow);
2870 %}
2871 
2872 instruct reduce_eorL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2873                              pRegGov ptmp, rFlagsReg cr) %{
2874   predicate(UseSVE > 0 &&
2875             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2876             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2877   match(Set dst (XorReductionV src1 src2));
2878   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2879   ins_cost(2 * SVE_COST);
2880   format %{ "sve_reduce_eorL $dst, $src1, $src2\t# eorL reduction partial (sve)" %}
2881   ins_encode %{
2882     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
2883     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2884                            $src1$$Register, as_FloatRegister($src2$$reg),
2885                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2886   %}
2887   ins_pipe(pipe_slow);
2888 %}
2889 
2890 // vector xor reduction - predicated
2891 
2892 instruct reduce_eorI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2893   predicate(UseSVE > 0 &&
2894             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG);
2895   match(Set dst (XorReductionV (Binary src1 src2) pg));
2896   effect(TEMP_DEF dst, TEMP tmp);
2897   ins_cost(SVE_COST);
2898   format %{ "sve_reduce_eorI $dst, $src1, $pg, $src2\t# eorI reduction predicated (sve) (may extend)" %}
2899   ins_encode %{
2900     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2901     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2902                            $src1$$Register, as_FloatRegister($src2$$reg),
2903                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2904   %}
2905   ins_pipe(pipe_slow);
2906 %}
2907 
2908 instruct reduce_eorL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2909   predicate(UseSVE > 0 &&
2910             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2911   match(Set dst (XorReductionV (Binary src1 src2) pg));
2912   effect(TEMP_DEF dst, TEMP tmp);
2913   ins_cost(SVE_COST);
2914   format %{ "sve_reduce_eorL $dst, $src1, $pg, $src2\t# eorL reduction predicated (sve)" %}
2915   ins_encode %{
2916     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2917                            $src1$$Register, as_FloatRegister($src2$$reg),
2918                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2919   %}
2920   ins_pipe(pipe_slow);
2921 %}
2922 
2923 // vector max reduction
2924 
2925 instruct reduce_maxI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
2926   predicate(UseSVE > 0 &&
2927             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2928             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2929             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
2930   match(Set dst (MaxReductionV src1 src2));
2931   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2932   ins_cost(SVE_COST);
2933   format %{ "sve_reduce_maxI $dst, $src1, $src2\t# maxI reduction (sve)" %}
2934   ins_encode %{
2935     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2936     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2937                            $src1$$Register, as_FloatRegister($src2$$reg),
2938                            ptrue, as_FloatRegister($tmp$$reg));
2939   %}
2940   ins_pipe(pipe_slow);
2941 %}
2942 
2943 instruct reduce_maxL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
2944   predicate(UseSVE > 0 &&
2945             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2946             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2947   match(Set dst (MaxReductionV src1 src2));
2948   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2949   ins_cost(SVE_COST);
2950   format %{ "sve_reduce_maxL $dst, $src1, $src2\t# maxL reduction (sve)" %}
2951   ins_encode %{
2952     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2953                            $src1$$Register, as_FloatRegister($src2$$reg),
2954                            ptrue, as_FloatRegister($tmp$$reg));
2955   %}
2956   ins_pipe(pipe_slow);
2957 %}
2958 
2959 instruct reduce_maxI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2960                              pRegGov ptmp, rFlagsReg cr) %{
2961   predicate(UseSVE > 0 &&
2962             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2963             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2964             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
2965   match(Set dst (MaxReductionV src1 src2));
2966   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2967   ins_cost(2 * SVE_COST);
2968   format %{ "sve_reduce_maxI $dst, $src1, $src2\t# maxI reduction partial (sve)" %}
2969   ins_encode %{
2970     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2971     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2972     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
2973     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2974                            $src1$$Register, as_FloatRegister($src2$$reg),
2975                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2976   %}
2977   ins_pipe(pipe_slow);
2978 %}
2979 
2980 instruct reduce_maxL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2981                              pRegGov ptmp, rFlagsReg cr) %{
2982   predicate(UseSVE > 0 &&
2983             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2984             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2985   match(Set dst (MaxReductionV src1 src2));
2986   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2987   ins_cost(2 * SVE_COST);
2988   format %{ "sve_reduce_maxL $dst, $src1, $src2\t# maxL reduction  partial (sve)" %}
2989   ins_encode %{
2990     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
2991     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2992                            $src1$$Register, as_FloatRegister($src2$$reg),
2993                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2994   %}
2995   ins_pipe(pipe_slow);
2996 %}
2997 
2998 instruct reduce_maxF(vRegF dst, vRegF src1, vReg src2) %{
2999   predicate(UseSVE > 0 &&
3000             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3001             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3002   match(Set dst (MaxReductionV src1 src2));
3003   ins_cost(INSN_COST);
3004   effect(TEMP_DEF dst);
3005   format %{ "sve_reduce_maxF $dst, $src1, $src2\t# maxF reduction (sve)" %}
3006   ins_encode %{
3007     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src2$$reg));
3008     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3009   %}
3010   ins_pipe(pipe_slow);
3011 %}
3012 
3013 instruct reduce_maxF_partial(vRegF dst, vRegF src1, vReg src2,
3014                              pRegGov ptmp, rFlagsReg cr) %{
3015   predicate(UseSVE > 0 &&
3016             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3017             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3018   match(Set dst (MaxReductionV src1 src2));
3019   ins_cost(INSN_COST);
3020   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3021   format %{ "sve_reduce_maxF $dst, $src1, $src2\t# maxF reduction partial (sve)" %}
3022   ins_encode %{
3023     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src2));
3024     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
3025     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3026   %}
3027   ins_pipe(pipe_slow);
3028 %}
3029 
3030 instruct reduce_maxD(vRegD dst, vRegD src1, vReg src2) %{
3031   predicate(UseSVE > 0 &&
3032             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3033             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3034   match(Set dst (MaxReductionV src1 src2));
3035   ins_cost(INSN_COST);
3036   effect(TEMP_DEF dst);
3037   format %{ "sve_reduce_maxD $dst, $src1, $src2\t# maxD reduction (sve)" %}
3038   ins_encode %{
3039     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
3040     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3041   %}
3042   ins_pipe(pipe_slow);
3043 %}
3044 
3045 instruct reduce_maxD_partial(vRegD dst, vRegD src1, vReg src2,
3046                              pRegGov ptmp, rFlagsReg cr) %{
3047   predicate(UseSVE > 0 &&
3048             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3049             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3050   match(Set dst (MaxReductionV src1 src2));
3051   ins_cost(INSN_COST);
3052   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3053   format %{ "sve_reduce_maxD $dst, $src1, $src2\t# maxD reduction partial (sve)" %}
3054   ins_encode %{
3055     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
3056     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
3057     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3058   %}
3059   ins_pipe(pipe_slow);
3060 %}
3061 
3062 // vector max reduction - predicated
3063 
3064 instruct reduce_maxI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp,
3065                            pRegGov pg, rFlagsReg cr) %{
3066   predicate(UseSVE > 0 &&
3067             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
3068             is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
3069   match(Set dst (MaxReductionV (Binary src1 src2) pg));
3070   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3071   ins_cost(SVE_COST);
3072   format %{ "sve_reduce_maxI $dst, $src1, $pg, $src2\t# maxI reduction predicated (sve)" %}
3073   ins_encode %{
3074     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
3075     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
3076                            $src1$$Register, as_FloatRegister($src2$$reg),
3077                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
3078   %}
3079   ins_pipe(pipe_slow);
3080 %}
3081 
3082 instruct reduce_maxL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp,
3083                           pRegGov pg, rFlagsReg cr) %{
3084   predicate(UseSVE > 0 &&
3085             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3086   match(Set dst (MaxReductionV (Binary src1 src2) pg));
3087   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3088   ins_cost(SVE_COST);
3089   format %{ "sve_reduce_maxL $dst, $src1, $pg, $src2\t# maxL reduction predicated (sve)" %}
3090   ins_encode %{
3091     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
3092                            $src1$$Register, as_FloatRegister($src2$$reg),
3093                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
3094   %}
3095   ins_pipe(pipe_slow);
3096 %}
3097 
3098 instruct reduce_maxF_masked(vRegF dst, vRegF src1, vReg src2, pRegGov pg) %{
3099   predicate(UseSVE > 0 &&
3100             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
3101   match(Set dst (MaxReductionV (Binary src1 src2) pg));
3102   ins_cost(SVE_COST);
3103   effect(TEMP_DEF dst);
3104   format %{ "sve_reduce_maxF $dst, $src1, $pg, $src2\t# maxF reduction predicated (sve)" %}
3105   ins_encode %{
3106     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
3107     __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3108   %}
3109   ins_pipe(pipe_slow);
3110 %}
3111 
3112 instruct reduce_maxD_masked(vRegD dst, vRegD src1, vReg src2, pRegGov pg) %{
3113   predicate(UseSVE > 0 &&
3114             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
3115   match(Set dst (MaxReductionV (Binary src1 src2) pg));
3116   ins_cost(SVE_COST);
3117   effect(TEMP_DEF dst);
3118   format %{ "sve_reduce_maxD $dst, $src1, $pg, $src2\t# maxD reduction predicated (sve)" %}
3119   ins_encode %{
3120     __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
3121     __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3122   %}
3123   ins_pipe(pipe_slow);
3124 %}
3125 
3126 // vector min reduction
3127 
3128 instruct reduce_minI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
3129   predicate(UseSVE > 0 &&
3130             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
3131             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
3132             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
3133   match(Set dst (MinReductionV src1 src2));
3134   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3135   ins_cost(SVE_COST);
3136   format %{ "sve_reduce_minI $dst, $src1, $src2\t# minI reduction (sve)" %}
3137   ins_encode %{
3138     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
3139     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
3140                            $src1$$Register, as_FloatRegister($src2$$reg),
3141                            ptrue, as_FloatRegister($tmp$$reg));
3142   %}
3143   ins_pipe(pipe_slow);
3144 %}
3145 
3146 instruct reduce_minL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
3147   predicate(UseSVE > 0 &&
3148             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
3149             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3150   match(Set dst (MinReductionV src1 src2));
3151   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3152   ins_cost(SVE_COST);
3153   format %{ "sve_reduce_minL $dst, $src1, $src2\t# minL reduction (sve)" %}
3154   ins_encode %{
3155     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
3156                            $src1$$Register, as_FloatRegister($src2$$reg),
3157                            ptrue, as_FloatRegister($tmp$$reg));
3158   %}
3159   ins_pipe(pipe_slow);
3160 %}
3161 
3162 instruct reduce_minI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
3163                              pRegGov ptmp, rFlagsReg cr) %{
3164   predicate(UseSVE > 0 &&
3165             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
3166             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
3167             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
3168   match(Set dst (MinReductionV src1 src2));
3169   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
3170   ins_cost(2 * SVE_COST);
3171   format %{ "sve_reduce_minI $dst, $src1, $src2\t# minI reduction partial (sve)" %}
3172   ins_encode %{
3173     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
3174     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
3175     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
3176     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
3177                            $src1$$Register, as_FloatRegister($src2$$reg),
3178                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
3179   %}
3180   ins_pipe(pipe_slow);
3181 %}
3182 
3183 instruct reduce_minL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
3184                              pRegGov ptmp, rFlagsReg cr) %{
3185   predicate(UseSVE > 0 &&
3186             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
3187             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3188   match(Set dst (MinReductionV src1 src2));
3189   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
3190   ins_cost(2 * SVE_COST);
3191   format %{ "sve_reduce_minL $dst, $src1, $src2\t# minL reduction  partial (sve)" %}
3192   ins_encode %{
3193     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
3194     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
3195                            $src1$$Register, as_FloatRegister($src2$$reg),
3196                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
3197   %}
3198   ins_pipe(pipe_slow);
3199 %}
3200 
3201 instruct reduce_minF(vRegF dst, vRegF src1, vReg src2) %{
3202   predicate(UseSVE > 0 &&
3203             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3204             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3205   match(Set dst (MinReductionV src1 src2));
3206   ins_cost(INSN_COST);
3207   effect(TEMP_DEF dst);
3208   format %{ "sve_reduce_minF $dst, $src1, $src2\t# minF reduction (sve)" %}
3209   ins_encode %{
3210     __ sve_fminv(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src2$$reg));
3211     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3212   %}
3213   ins_pipe(pipe_slow);
3214 %}
3215 
3216 instruct reduce_minF_partial(vRegF dst, vRegF src1, vReg src2,
3217                              pRegGov ptmp, rFlagsReg cr) %{
3218   predicate(UseSVE > 0 &&
3219             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3220             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3221   match(Set dst (MinReductionV src1 src2));
3222   ins_cost(INSN_COST);
3223   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3224   format %{ "sve_reduce_minF $dst, $src1, $src2\t# minF reduction partial (sve)" %}
3225   ins_encode %{
3226     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src2));
3227     __ sve_fminv(as_FloatRegister($dst$$reg), __ S, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
3228     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3229   %}
3230   ins_pipe(pipe_slow);
3231 %}
3232 
3233 instruct reduce_minD(vRegD dst, vRegD src1, vReg src2) %{
3234   predicate(UseSVE > 0 &&
3235             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3236             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3237   match(Set dst (MinReductionV src1 src2));
3238   ins_cost(INSN_COST);
3239   effect(TEMP_DEF dst);
3240   format %{ "sve_reduce_minD $dst, $src1, $src2\t# minD reduction (sve)" %}
3241   ins_encode %{
3242     __ sve_fminv(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
3243     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3244   %}
3245   ins_pipe(pipe_slow);
3246 %}
3247 
3248 instruct reduce_minD_partial(vRegD dst, vRegD src1, vReg src2,
3249                              pRegGov ptmp, rFlagsReg cr) %{
3250   predicate(UseSVE > 0 &&
3251             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3252             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3253   match(Set dst (MinReductionV src1 src2));
3254   ins_cost(INSN_COST);
3255   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3256   format %{ "sve_reduce_minD $dst, $src1, $src2\t# minD reduction partial (sve)" %}
3257   ins_encode %{
3258     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
3259     __ sve_fminv(as_FloatRegister($dst$$reg), __ D, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
3260     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3261   %}
3262   ins_pipe(pipe_slow);
3263 %}
3264 
3265 // vector min reduction - predicated
3266 
3267 instruct reduce_minI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp,
3268                            pRegGov pg, rFlagsReg cr) %{
3269   predicate(UseSVE > 0 &&
3270             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
3271             is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
3272   match(Set dst (MinReductionV (Binary src1 src2) pg));
3273   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3274   ins_cost(SVE_COST);
3275   format %{ "sve_reduce_minI $dst, $src1, $pg, $src2\t# minI reduction predicated (sve)" %}
3276   ins_encode %{
3277     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
3278     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
3279                            $src1$$Register, as_FloatRegister($src2$$reg),
3280                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
3281   %}
3282   ins_pipe(pipe_slow);
3283 %}
3284 
3285 instruct reduce_minL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp,
3286                           pRegGov pg, rFlagsReg cr) %{
3287   predicate(UseSVE > 0 &&
3288             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3289   match(Set dst (MinReductionV (Binary src1 src2) pg));
3290   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3291   ins_cost(SVE_COST);
3292   format %{ "sve_reduce_minL $dst, $src1, $pg, $src2\t# minL reduction predicated (sve)" %}
3293   ins_encode %{
3294     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
3295                            $src1$$Register, as_FloatRegister($src2$$reg),
3296                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
3297   %}
3298   ins_pipe(pipe_slow);
3299 %}
3300 
3301 instruct reduce_minF_masked(vRegF dst, vRegF src1, vReg src2, pRegGov pg) %{
3302   predicate(UseSVE > 0 &&
3303             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
3304   match(Set dst (MinReductionV (Binary src1 src2) pg));
3305   ins_cost(SVE_COST);
3306   effect(TEMP_DEF dst);
3307   format %{ "sve_reduce_minF $dst, $src1, $pg, $src2\t# minF reduction predicated (sve)" %}
3308   ins_encode %{
3309     __ sve_fminv(as_FloatRegister($dst$$reg), __ S, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
3310     __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3311   %}
3312   ins_pipe(pipe_slow);
3313 %}
3314 
3315 instruct reduce_minD_masked(vRegD dst, vRegD src1, vReg src2, pRegGov pg) %{
3316   predicate(UseSVE > 0 &&
3317             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
3318   match(Set dst (MinReductionV (Binary src1 src2) pg));
3319   ins_cost(SVE_COST);
3320   effect(TEMP_DEF dst);
3321   format %{ "sve_reduce_minD $dst, $src1, $pg, $src2\t# minD reduction predicated (sve)" %}
3322   ins_encode %{
3323     __ sve_fminv(as_FloatRegister($dst$$reg), __ D, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
3324     __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3325   %}
3326   ins_pipe(pipe_slow);
3327 %}
3328 
3329 // vector Math.rint, floor, ceil
3330 
3331 instruct vroundD(vReg dst, vReg src, immI rmode) %{
3332   predicate(UseSVE > 0 &&
3333             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
3334   match(Set dst (RoundDoubleModeV src rmode));
3335   format %{ "sve_frint $dst, $src, $rmode\t# vector (sve) (D)" %}
3336   ins_encode %{
3337     switch ($rmode$$constant) {
3338       case RoundDoubleModeNode::rmode_rint:
3339         __ sve_frintn(as_FloatRegister($dst$$reg), __ D,
3340              ptrue, as_FloatRegister($src$$reg));
3341         break;
3342       case RoundDoubleModeNode::rmode_floor:
3343         __ sve_frintm(as_FloatRegister($dst$$reg), __ D,
3344              ptrue, as_FloatRegister($src$$reg));
3345         break;
3346       case RoundDoubleModeNode::rmode_ceil:
3347         __ sve_frintp(as_FloatRegister($dst$$reg), __ D,
3348              ptrue, as_FloatRegister($src$$reg));
3349         break;
3350     }
3351   %}
3352   ins_pipe(pipe_slow);
3353 %}
3354 
3355 instruct vroundFtoI(vReg dst, vReg src, vReg tmp1, vReg tmp2, vReg tmp3, pRegGov ptmp)
3356 %{
3357   predicate(UseSVE > 0);
3358   match(Set dst (RoundVF src));
3359   effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP ptmp);
3360   format %{ "sve_vround  $dst, S, $src\t# round F to I vector" %}
3361   ins_encode %{
3362     BasicType bt = Matcher::vector_element_basic_type(this);
3363     int vlen = Matcher::vector_length_in_bytes(this);
3364     if (vlen > 16) {
3365       __ vector_round_sve(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3366                           as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg),
3367                           as_PRegister($ptmp$$reg), __ S);
3368     } else {
3369       __ vector_round_neon(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3370                            as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg),
3371                            as_FloatRegister($tmp3$$reg),
3372                            __ esize2arrangement(type2aelembytes(bt),
3373                               /*isQ*/ vlen == 16));
3374     }
3375   %}
3376   ins_pipe(pipe_class_default);
3377 %}
3378 
3379 instruct vroundDtoL(vReg dst, vReg src, vReg tmp1, vReg tmp2, vReg tmp3, pRegGov ptmp)
3380 %{
3381   predicate(UseSVE > 0);
3382   match(Set dst (RoundVD src));
3383   effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP ptmp);
3384   format %{ "sve_vround  $dst, D, $src\t# round D to L vector" %}
3385   ins_encode %{
3386     BasicType bt = Matcher::vector_element_basic_type(this);
3387     int vlen = Matcher::vector_length_in_bytes(this);
3388     if (vlen > 16) {
3389       __ vector_round_sve(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3390                           as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg),
3391                           as_PRegister($ptmp$$reg), __ D);
3392     } else {
3393       __ vector_round_neon(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3394                            as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg),
3395                            as_FloatRegister($tmp3$$reg),
3396                            __ esize2arrangement(type2aelembytes(bt),
3397                               /*isQ*/ vlen == 16));
3398     }
3399   %}
3400   ins_pipe(pipe_class_default);
3401 %}
3402 
3403 // vector replicate
3404 
3405 instruct replicateB(vReg dst, iRegIorL2I src) %{
3406   predicate(UseSVE > 0);
3407   match(Set dst (ReplicateB src));
3408   ins_cost(SVE_COST);
3409   format %{ "sve_dup  $dst, $src\t# vector (sve) (B)" %}
3410   ins_encode %{
3411     __ sve_dup(as_FloatRegister($dst$$reg), __ B, as_Register($src$$reg));
3412   %}
3413   ins_pipe(pipe_slow);
3414 %}
3415 
3416 instruct replicateS(vReg dst, iRegIorL2I src) %{
3417   predicate(UseSVE > 0);
3418   match(Set dst (ReplicateS src));
3419   ins_cost(SVE_COST);
3420   format %{ "sve_dup  $dst, $src\t# vector (sve) (H)" %}
3421   ins_encode %{
3422     __ sve_dup(as_FloatRegister($dst$$reg), __ H, as_Register($src$$reg));
3423   %}
3424   ins_pipe(pipe_slow);
3425 %}
3426 
3427 instruct replicateI(vReg dst, iRegIorL2I src) %{
3428   predicate(UseSVE > 0);
3429   match(Set dst (ReplicateI src));
3430   ins_cost(SVE_COST);
3431   format %{ "sve_dup  $dst, $src\t# vector (sve) (S)" %}
3432   ins_encode %{
3433     __ sve_dup(as_FloatRegister($dst$$reg), __ S, as_Register($src$$reg));
3434   %}
3435   ins_pipe(pipe_slow);
3436 %}
3437 
3438 instruct replicateL(vReg dst, iRegL src) %{
3439   predicate(UseSVE > 0);
3440   match(Set dst (ReplicateL src));
3441   ins_cost(SVE_COST);
3442   format %{ "sve_dup  $dst, $src\t# vector (sve) (D)" %}
3443   ins_encode %{
3444     __ sve_dup(as_FloatRegister($dst$$reg), __ D, as_Register($src$$reg));
3445   %}
3446   ins_pipe(pipe_slow);
3447 %}
3448 
3449 instruct replicateB_imm8(vReg dst, immI8 con) %{
3450   predicate(UseSVE > 0);
3451   match(Set dst (ReplicateB con));
3452   ins_cost(SVE_COST);
3453   format %{ "sve_dup  $dst, $con\t# vector (sve) (B)" %}
3454   ins_encode %{
3455     __ sve_dup(as_FloatRegister($dst$$reg), __ B, $con$$constant);
3456   %}
3457   ins_pipe(pipe_slow);
3458 %}
3459 
3460 instruct replicateS_imm8(vReg dst, immI8_shift8 con) %{
3461   predicate(UseSVE > 0);
3462   match(Set dst (ReplicateS con));
3463   ins_cost(SVE_COST);
3464   format %{ "sve_dup  $dst, $con\t# vector (sve) (H)" %}
3465   ins_encode %{
3466     __ sve_dup(as_FloatRegister($dst$$reg), __ H, $con$$constant);
3467   %}
3468   ins_pipe(pipe_slow);
3469 %}
3470 
3471 instruct replicateI_imm8(vReg dst, immI8_shift8 con) %{
3472   predicate(UseSVE > 0);
3473   match(Set dst (ReplicateI con));
3474   ins_cost(SVE_COST);
3475   format %{ "sve_dup  $dst, $con\t# vector (sve) (S)" %}
3476   ins_encode %{
3477     __ sve_dup(as_FloatRegister($dst$$reg), __ S, $con$$constant);
3478   %}
3479   ins_pipe(pipe_slow);
3480 %}
3481 
3482 instruct replicateL_imm8(vReg dst, immL8_shift8 con) %{
3483   predicate(UseSVE > 0);
3484   match(Set dst (ReplicateL con));
3485   ins_cost(SVE_COST);
3486   format %{ "sve_dup  $dst, $con\t# vector (sve) (D)" %}
3487   ins_encode %{
3488     __ sve_dup(as_FloatRegister($dst$$reg), __ D, $con$$constant);
3489   %}
3490   ins_pipe(pipe_slow);
3491 %}
3492 
3493 instruct replicateF(vReg dst, vRegF src) %{
3494   predicate(UseSVE > 0);
3495   match(Set dst (ReplicateF src));
3496   ins_cost(SVE_COST);
3497   format %{ "sve_cpy  $dst, $src\t# vector (sve) (S)" %}
3498   ins_encode %{
3499     __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
3500          ptrue, as_FloatRegister($src$$reg));
3501   %}
3502   ins_pipe(pipe_slow);
3503 %}
3504 
3505 instruct replicateD(vReg dst, vRegD src) %{
3506   predicate(UseSVE > 0);
3507   match(Set dst (ReplicateD src));
3508   ins_cost(SVE_COST);
3509   format %{ "sve_cpy  $dst, $src\t# vector (sve) (D)" %}
3510   ins_encode %{
3511     __ sve_cpy(as_FloatRegister($dst$$reg), __ D,
3512          ptrue, as_FloatRegister($src$$reg));
3513   %}
3514   ins_pipe(pipe_slow);
3515 %}
3516 
3517 // vector shift
3518 
3519 instruct vasrB(vReg dst, vReg shift) %{
3520   predicate(UseSVE > 0);
3521   match(Set dst (RShiftVB dst shift));
3522   ins_cost(SVE_COST);
3523   format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (B)" %}
3524   ins_encode %{
3525     __ sve_asr(as_FloatRegister($dst$$reg), __ B,
3526          ptrue, as_FloatRegister($shift$$reg));
3527   %}
3528   ins_pipe(pipe_slow);
3529 %}
3530 
3531 instruct vasrS(vReg dst, vReg shift) %{
3532   predicate(UseSVE > 0);
3533   match(Set dst (RShiftVS dst shift));
3534   ins_cost(SVE_COST);
3535   format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (H)" %}
3536   ins_encode %{
3537     __ sve_asr(as_FloatRegister($dst$$reg), __ H,
3538          ptrue, as_FloatRegister($shift$$reg));
3539   %}
3540   ins_pipe(pipe_slow);
3541 %}
3542 
3543 instruct vasrI(vReg dst, vReg shift) %{
3544   predicate(UseSVE > 0);
3545   match(Set dst (RShiftVI dst shift));
3546   ins_cost(SVE_COST);
3547   format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (S)" %}
3548   ins_encode %{
3549     __ sve_asr(as_FloatRegister($dst$$reg), __ S,
3550          ptrue, as_FloatRegister($shift$$reg));
3551   %}
3552   ins_pipe(pipe_slow);
3553 %}
3554 
3555 instruct vasrL(vReg dst, vReg shift) %{
3556   predicate(UseSVE > 0);
3557   match(Set dst (RShiftVL dst shift));
3558   ins_cost(SVE_COST);
3559   format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (D)" %}
3560   ins_encode %{
3561     __ sve_asr(as_FloatRegister($dst$$reg), __ D,
3562          ptrue, as_FloatRegister($shift$$reg));
3563   %}
3564   ins_pipe(pipe_slow);
3565 %}
3566 
3567 instruct vlslB(vReg dst, vReg shift) %{
3568   predicate(UseSVE > 0);
3569   match(Set dst (LShiftVB dst shift));
3570   ins_cost(SVE_COST);
3571   format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (B)" %}
3572   ins_encode %{
3573     __ sve_lsl(as_FloatRegister($dst$$reg), __ B,
3574          ptrue, as_FloatRegister($shift$$reg));
3575   %}
3576   ins_pipe(pipe_slow);
3577 %}
3578 
3579 instruct vlslS(vReg dst, vReg shift) %{
3580   predicate(UseSVE > 0);
3581   match(Set dst (LShiftVS dst shift));
3582   ins_cost(SVE_COST);
3583   format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (H)" %}
3584   ins_encode %{
3585     __ sve_lsl(as_FloatRegister($dst$$reg), __ H,
3586          ptrue, as_FloatRegister($shift$$reg));
3587   %}
3588   ins_pipe(pipe_slow);
3589 %}
3590 
3591 instruct vlslI(vReg dst, vReg shift) %{
3592   predicate(UseSVE > 0);
3593   match(Set dst (LShiftVI dst shift));
3594   ins_cost(SVE_COST);
3595   format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (S)" %}
3596   ins_encode %{
3597     __ sve_lsl(as_FloatRegister($dst$$reg), __ S,
3598          ptrue, as_FloatRegister($shift$$reg));
3599   %}
3600   ins_pipe(pipe_slow);
3601 %}
3602 
3603 instruct vlslL(vReg dst, vReg shift) %{
3604   predicate(UseSVE > 0);
3605   match(Set dst (LShiftVL dst shift));
3606   ins_cost(SVE_COST);
3607   format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (D)" %}
3608   ins_encode %{
3609     __ sve_lsl(as_FloatRegister($dst$$reg), __ D,
3610          ptrue, as_FloatRegister($shift$$reg));
3611   %}
3612   ins_pipe(pipe_slow);
3613 %}
3614 
3615 instruct vlsrB(vReg dst, vReg shift) %{
3616   predicate(UseSVE > 0);
3617   match(Set dst (URShiftVB dst shift));
3618   ins_cost(SVE_COST);
3619   format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (B)" %}
3620   ins_encode %{
3621     __ sve_lsr(as_FloatRegister($dst$$reg), __ B,
3622          ptrue, as_FloatRegister($shift$$reg));
3623   %}
3624   ins_pipe(pipe_slow);
3625 %}
3626 
3627 instruct vlsrS(vReg dst, vReg shift) %{
3628   predicate(UseSVE > 0);
3629   match(Set dst (URShiftVS dst shift));
3630   ins_cost(SVE_COST);
3631   format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (H)" %}
3632   ins_encode %{
3633     __ sve_lsr(as_FloatRegister($dst$$reg), __ H,
3634          ptrue, as_FloatRegister($shift$$reg));
3635   %}
3636   ins_pipe(pipe_slow);
3637 %}
3638 
3639 instruct vlsrI(vReg dst, vReg shift) %{
3640   predicate(UseSVE > 0);
3641   match(Set dst (URShiftVI dst shift));
3642   ins_cost(SVE_COST);
3643   format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (S)" %}
3644   ins_encode %{
3645     __ sve_lsr(as_FloatRegister($dst$$reg), __ S,
3646          ptrue, as_FloatRegister($shift$$reg));
3647   %}
3648   ins_pipe(pipe_slow);
3649 %}
3650 
3651 instruct vlsrL(vReg dst, vReg shift) %{
3652   predicate(UseSVE > 0);
3653   match(Set dst (URShiftVL dst shift));
3654   ins_cost(SVE_COST);
3655   format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (D)" %}
3656   ins_encode %{
3657     __ sve_lsr(as_FloatRegister($dst$$reg), __ D,
3658          ptrue, as_FloatRegister($shift$$reg));
3659   %}
3660   ins_pipe(pipe_slow);
3661 %}
3662 
3663 instruct vasrB_imm(vReg dst, vReg src, immI shift) %{
3664   predicate(UseSVE > 0);
3665   match(Set dst (RShiftVB src (RShiftCntV shift)));
3666   ins_cost(SVE_COST);
3667   format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (B)" %}
3668   ins_encode %{
3669     int con = (int)$shift$$constant;
3670     if (con == 0) {
3671       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3672            as_FloatRegister($src$$reg));
3673       return;
3674     }
3675     if (con >= 8) con = 7;
3676     __ sve_asr(as_FloatRegister($dst$$reg), __ B,
3677          as_FloatRegister($src$$reg), con);
3678   %}
3679   ins_pipe(pipe_slow);
3680 %}
3681 
3682 instruct vasrS_imm(vReg dst, vReg src, immI shift) %{
3683   predicate(UseSVE > 0);
3684   match(Set dst (RShiftVS src (RShiftCntV shift)));
3685   ins_cost(SVE_COST);
3686   format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (H)" %}
3687   ins_encode %{
3688     int con = (int)$shift$$constant;
3689     if (con == 0) {
3690       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3691            as_FloatRegister($src$$reg));
3692       return;
3693     }
3694     if (con >= 16) con = 15;
3695     __ sve_asr(as_FloatRegister($dst$$reg), __ H,
3696          as_FloatRegister($src$$reg), con);
3697   %}
3698   ins_pipe(pipe_slow);
3699 %}
3700 
3701 instruct vasrI_imm(vReg dst, vReg src, immI shift) %{
3702   predicate(UseSVE > 0);
3703   match(Set dst (RShiftVI src (RShiftCntV shift)));
3704   ins_cost(SVE_COST);
3705   format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (S)" %}
3706   ins_encode %{
3707     int con = (int)$shift$$constant;
3708     if (con == 0) {
3709       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3710            as_FloatRegister($src$$reg));
3711       return;
3712     }
3713     __ sve_asr(as_FloatRegister($dst$$reg), __ S,
3714          as_FloatRegister($src$$reg), con);
3715   %}
3716   ins_pipe(pipe_slow);
3717 %}
3718 
3719 instruct vasrL_imm(vReg dst, vReg src, immI shift) %{
3720   predicate(UseSVE > 0);
3721   match(Set dst (RShiftVL src (RShiftCntV shift)));
3722   ins_cost(SVE_COST);
3723   format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (D)" %}
3724   ins_encode %{
3725     int con = (int)$shift$$constant;
3726     if (con == 0) {
3727       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3728            as_FloatRegister($src$$reg));
3729       return;
3730     }
3731     __ sve_asr(as_FloatRegister($dst$$reg), __ D,
3732          as_FloatRegister($src$$reg), con);
3733   %}
3734   ins_pipe(pipe_slow);
3735 %}
3736 
3737 instruct vlsrB_imm(vReg dst, vReg src, immI shift) %{
3738   predicate(UseSVE > 0);
3739   match(Set dst (URShiftVB src (RShiftCntV shift)));
3740   ins_cost(SVE_COST);
3741   format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (B)" %}
3742   ins_encode %{
3743     int con = (int)$shift$$constant;
3744     if (con == 0) {
3745       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3746            as_FloatRegister($src$$reg));
3747       return;
3748     }
3749     if (con >= 8) {
3750       __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3751            as_FloatRegister($src$$reg));
3752       return;
3753     }
3754     __ sve_lsr(as_FloatRegister($dst$$reg), __ B,
3755          as_FloatRegister($src$$reg), con);
3756   %}
3757   ins_pipe(pipe_slow);
3758 %}
3759 
3760 instruct vlsrS_imm(vReg dst, vReg src, immI shift) %{
3761   predicate(UseSVE > 0);
3762   match(Set dst (URShiftVS src (RShiftCntV shift)));
3763   ins_cost(SVE_COST);
3764   format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (H)" %}
3765   ins_encode %{
3766     int con = (int)$shift$$constant;
3767     if (con == 0) {
3768       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3769            as_FloatRegister($src$$reg));
3770       return;
3771     }
3772     if (con >= 16) {
3773       __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3774            as_FloatRegister($src$$reg));
3775       return;
3776     }
3777     __ sve_lsr(as_FloatRegister($dst$$reg), __ H,
3778          as_FloatRegister($src$$reg), con);
3779   %}
3780   ins_pipe(pipe_slow);
3781 %}
3782 
3783 instruct vlsrI_imm(vReg dst, vReg src, immI shift) %{
3784   predicate(UseSVE > 0);
3785   match(Set dst (URShiftVI src (RShiftCntV shift)));
3786   ins_cost(SVE_COST);
3787   format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (S)" %}
3788   ins_encode %{
3789     int con = (int)$shift$$constant;
3790     if (con == 0) {
3791       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3792            as_FloatRegister($src$$reg));
3793       return;
3794     }
3795     __ sve_lsr(as_FloatRegister($dst$$reg), __ S,
3796          as_FloatRegister($src$$reg), con);
3797   %}
3798   ins_pipe(pipe_slow);
3799 %}
3800 
3801 instruct vlsrL_imm(vReg dst, vReg src, immI shift) %{
3802   predicate(UseSVE > 0);
3803   match(Set dst (URShiftVL src (RShiftCntV shift)));
3804   ins_cost(SVE_COST);
3805   format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (D)" %}
3806   ins_encode %{
3807     int con = (int)$shift$$constant;
3808     if (con == 0) {
3809       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3810            as_FloatRegister($src$$reg));
3811       return;
3812     }
3813     __ sve_lsr(as_FloatRegister($dst$$reg), __ D,
3814          as_FloatRegister($src$$reg), con);
3815   %}
3816   ins_pipe(pipe_slow);
3817 %}
3818 
3819 instruct vlslB_imm(vReg dst, vReg src, immI shift) %{
3820   predicate(UseSVE > 0);
3821   match(Set dst (LShiftVB src (LShiftCntV shift)));
3822   ins_cost(SVE_COST);
3823   format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (B)" %}
3824   ins_encode %{
3825     int con = (int)$shift$$constant;
3826     if (con >= 8) {
3827       __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3828            as_FloatRegister($src$$reg));
3829       return;
3830     }
3831     __ sve_lsl(as_FloatRegister($dst$$reg), __ B,
3832          as_FloatRegister($src$$reg), con);
3833   %}
3834   ins_pipe(pipe_slow);
3835 %}
3836 
3837 instruct vlslS_imm(vReg dst, vReg src, immI shift) %{
3838   predicate(UseSVE > 0);
3839   match(Set dst (LShiftVS src (LShiftCntV shift)));
3840   ins_cost(SVE_COST);
3841   format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (H)" %}
3842   ins_encode %{
3843     int con = (int)$shift$$constant;
3844     if (con >= 16) {
3845       __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3846            as_FloatRegister($src$$reg));
3847       return;
3848     }
3849     __ sve_lsl(as_FloatRegister($dst$$reg), __ H,
3850          as_FloatRegister($src$$reg), con);
3851   %}
3852   ins_pipe(pipe_slow);
3853 %}
3854 
3855 instruct vlslI_imm(vReg dst, vReg src, immI shift) %{
3856   predicate(UseSVE > 0);
3857   match(Set dst (LShiftVI src (LShiftCntV shift)));
3858   ins_cost(SVE_COST);
3859   format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (S)" %}
3860   ins_encode %{
3861     int con = (int)$shift$$constant;
3862     __ sve_lsl(as_FloatRegister($dst$$reg), __ S,
3863          as_FloatRegister($src$$reg), con);
3864   %}
3865   ins_pipe(pipe_slow);
3866 %}
3867 
3868 instruct vlslL_imm(vReg dst, vReg src, immI shift) %{
3869   predicate(UseSVE > 0);
3870   match(Set dst (LShiftVL src (LShiftCntV shift)));
3871   ins_cost(SVE_COST);
3872   format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (D)" %}
3873   ins_encode %{
3874     int con = (int)$shift$$constant;
3875     __ sve_lsl(as_FloatRegister($dst$$reg), __ D,
3876          as_FloatRegister($src$$reg), con);
3877   %}
3878   ins_pipe(pipe_slow);
3879 %}
3880 
3881 instruct vshiftcntB(vReg dst, iRegIorL2I cnt) %{
3882   predicate(UseSVE > 0 &&
3883             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE));
3884   match(Set dst (LShiftCntV cnt));
3885   match(Set dst (RShiftCntV cnt));
3886   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (B)" %}
3887   ins_encode %{
3888     __ sve_dup(as_FloatRegister($dst$$reg), __ B, as_Register($cnt$$reg));
3889   %}
3890   ins_pipe(pipe_slow);
3891 %}
3892 
3893 instruct vshiftcntS(vReg dst, iRegIorL2I cnt) %{
3894   predicate(UseSVE > 0 &&
3895             (n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
3896             (n->bottom_type()->is_vect()->element_basic_type() == T_CHAR)));
3897   match(Set dst (LShiftCntV cnt));
3898   match(Set dst (RShiftCntV cnt));
3899   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (H)" %}
3900   ins_encode %{
3901     __ sve_dup(as_FloatRegister($dst$$reg), __ H, as_Register($cnt$$reg));
3902   %}
3903   ins_pipe(pipe_slow);
3904 %}
3905 
3906 instruct vshiftcntI(vReg dst, iRegIorL2I cnt) %{
3907   predicate(UseSVE > 0 &&
3908             (n->bottom_type()->is_vect()->element_basic_type() == T_INT));
3909   match(Set dst (LShiftCntV cnt));
3910   match(Set dst (RShiftCntV cnt));
3911   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (S)" %}
3912   ins_encode %{
3913     __ sve_dup(as_FloatRegister($dst$$reg), __ S, as_Register($cnt$$reg));
3914   %}
3915   ins_pipe(pipe_slow);
3916 %}
3917 
3918 instruct vshiftcntL(vReg dst, iRegIorL2I cnt) %{
3919   predicate(UseSVE > 0 &&
3920             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
3921   match(Set dst (LShiftCntV cnt));
3922   match(Set dst (RShiftCntV cnt));
3923   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (D)" %}
3924   ins_encode %{
3925     __ sve_dup(as_FloatRegister($dst$$reg), __ D, as_Register($cnt$$reg));
3926   %}
3927   ins_pipe(pipe_slow);
3928 %}
3929 
3930 // vector shift - predicated
3931 
3932 instruct vasrB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3933   predicate(UseSVE > 0);
3934   match(Set dst_src1 (RShiftVB (Binary dst_src1 src2) pg));
3935   ins_cost(SVE_COST);
3936   format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
3937   ins_encode %{
3938     __ sve_asr(as_FloatRegister($dst_src1$$reg), __ B,
3939             as_PRegister($pg$$reg),
3940             as_FloatRegister($src2$$reg));
3941   %}
3942   ins_pipe(pipe_slow);
3943 %}
3944 
3945 instruct vasrS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3946   predicate(UseSVE > 0);
3947   match(Set dst_src1 (RShiftVS (Binary dst_src1 src2) pg));
3948   ins_cost(SVE_COST);
3949   format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
3950   ins_encode %{
3951     __ sve_asr(as_FloatRegister($dst_src1$$reg), __ H,
3952             as_PRegister($pg$$reg),
3953             as_FloatRegister($src2$$reg));
3954   %}
3955   ins_pipe(pipe_slow);
3956 %}
3957 
3958 instruct vasrI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3959   predicate(UseSVE > 0);
3960   match(Set dst_src1 (RShiftVI (Binary dst_src1 src2) pg));
3961   ins_cost(SVE_COST);
3962   format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
3963   ins_encode %{
3964     __ sve_asr(as_FloatRegister($dst_src1$$reg), __ S,
3965             as_PRegister($pg$$reg),
3966             as_FloatRegister($src2$$reg));
3967   %}
3968   ins_pipe(pipe_slow);
3969 %}
3970 
3971 instruct vasrL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3972   predicate(UseSVE > 0);
3973   match(Set dst_src1 (RShiftVL (Binary dst_src1 src2) pg));
3974   ins_cost(SVE_COST);
3975   format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
3976   ins_encode %{
3977     __ sve_asr(as_FloatRegister($dst_src1$$reg), __ D,
3978             as_PRegister($pg$$reg),
3979             as_FloatRegister($src2$$reg));
3980   %}
3981   ins_pipe(pipe_slow);
3982 %}
3983 
3984 instruct vlslB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3985   predicate(UseSVE > 0);
3986   match(Set dst_src1 (LShiftVB (Binary dst_src1 src2) pg));
3987   ins_cost(SVE_COST);
3988   format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
3989   ins_encode %{
3990     __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ B,
3991             as_PRegister($pg$$reg),
3992             as_FloatRegister($src2$$reg));
3993   %}
3994   ins_pipe(pipe_slow);
3995 %}
3996 
3997 instruct vlslS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3998   predicate(UseSVE > 0);
3999   match(Set dst_src1 (LShiftVS (Binary dst_src1 src2) pg));
4000   ins_cost(SVE_COST);
4001   format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
4002   ins_encode %{
4003     __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ H,
4004             as_PRegister($pg$$reg),
4005             as_FloatRegister($src2$$reg));
4006   %}
4007   ins_pipe(pipe_slow);
4008 %}
4009 
4010 instruct vlslI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4011   predicate(UseSVE > 0);
4012   match(Set dst_src1 (LShiftVI (Binary dst_src1 src2) pg));
4013   ins_cost(SVE_COST);
4014   format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
4015   ins_encode %{
4016     __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ S,
4017             as_PRegister($pg$$reg),
4018             as_FloatRegister($src2$$reg));
4019   %}
4020   ins_pipe(pipe_slow);
4021 %}
4022 
4023 instruct vlslL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4024   predicate(UseSVE > 0);
4025   match(Set dst_src1 (LShiftVL (Binary dst_src1 src2) pg));
4026   ins_cost(SVE_COST);
4027   format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
4028   ins_encode %{
4029     __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ D,
4030             as_PRegister($pg$$reg),
4031             as_FloatRegister($src2$$reg));
4032   %}
4033   ins_pipe(pipe_slow);
4034 %}
4035 
4036 instruct vlsrB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4037   predicate(UseSVE > 0);
4038   match(Set dst_src1 (URShiftVB (Binary dst_src1 src2) pg));
4039   ins_cost(SVE_COST);
4040   format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
4041   ins_encode %{
4042     __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ B,
4043             as_PRegister($pg$$reg),
4044             as_FloatRegister($src2$$reg));
4045   %}
4046   ins_pipe(pipe_slow);
4047 %}
4048 
4049 instruct vlsrS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4050   predicate(UseSVE > 0);
4051   match(Set dst_src1 (URShiftVS (Binary dst_src1 src2) pg));
4052   ins_cost(SVE_COST);
4053   format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
4054   ins_encode %{
4055     __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ H,
4056             as_PRegister($pg$$reg),
4057             as_FloatRegister($src2$$reg));
4058   %}
4059   ins_pipe(pipe_slow);
4060 %}
4061 
4062 instruct vlsrI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4063   predicate(UseSVE > 0);
4064   match(Set dst_src1 (URShiftVI (Binary dst_src1 src2) pg));
4065   ins_cost(SVE_COST);
4066   format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
4067   ins_encode %{
4068     __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ S,
4069             as_PRegister($pg$$reg),
4070             as_FloatRegister($src2$$reg));
4071   %}
4072   ins_pipe(pipe_slow);
4073 %}
4074 
4075 instruct vlsrL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4076   predicate(UseSVE > 0);
4077   match(Set dst_src1 (URShiftVL (Binary dst_src1 src2) pg));
4078   ins_cost(SVE_COST);
4079   format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
4080   ins_encode %{
4081     __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ D,
4082             as_PRegister($pg$$reg),
4083             as_FloatRegister($src2$$reg));
4084   %}
4085   ins_pipe(pipe_slow);
4086 %}
4087 
4088 instruct vasrB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4089   predicate(UseSVE > 0);
4090   match(Set dst_src (RShiftVB (Binary dst_src (RShiftCntV shift)) pg));
4091   ins_cost(SVE_COST);
4092   format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %}
4093   ins_encode %{
4094     int con = (int)$shift$$constant;
4095     assert(con > 0 && con < 8, "invalid shift immediate");
4096     __ sve_asr(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con);
4097   %}
4098   ins_pipe(pipe_slow);
4099 %}
4100 
4101 instruct vasrS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4102   predicate(UseSVE > 0);
4103   match(Set dst_src (RShiftVS (Binary dst_src (RShiftCntV shift)) pg));
4104   ins_cost(SVE_COST);
4105   format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %}
4106   ins_encode %{
4107     int con = (int)$shift$$constant;
4108     assert(con > 0 && con < 16, "invalid shift immediate");
4109     __ sve_asr(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con);
4110   %}
4111   ins_pipe(pipe_slow);
4112 %}
4113 
4114 instruct vasrI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4115   predicate(UseSVE > 0);
4116   match(Set dst_src (RShiftVI (Binary dst_src (RShiftCntV shift)) pg));
4117   ins_cost(SVE_COST);
4118   format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %}
4119   ins_encode %{
4120     int con = (int)$shift$$constant;
4121     assert(con > 0 && con < 32, "invalid shift immediate");
4122     __ sve_asr(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con);
4123   %}
4124   ins_pipe(pipe_slow);
4125 %}
4126 
4127 instruct vasrL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4128   predicate(UseSVE > 0);
4129   match(Set dst_src (RShiftVL (Binary dst_src (RShiftCntV shift)) pg));
4130   ins_cost(SVE_COST);
4131   format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %}
4132   ins_encode %{
4133     int con = (int)$shift$$constant;
4134     assert(con > 0 && con < 64, "invalid shift immediate");
4135     __ sve_asr(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con);
4136   %}
4137   ins_pipe(pipe_slow);
4138 %}
4139 
4140 instruct vlsrB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4141   predicate(UseSVE > 0);
4142   match(Set dst_src (URShiftVB (Binary dst_src (RShiftCntV shift)) pg));
4143   ins_cost(SVE_COST);
4144   format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %}
4145   ins_encode %{
4146     int con = (int)$shift$$constant;
4147     assert(con > 0 && con < 8, "invalid shift immediate");
4148     __ sve_lsr(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con);
4149   %}
4150   ins_pipe(pipe_slow);
4151 %}
4152 
4153 instruct vlsrS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4154   predicate(UseSVE > 0);
4155   match(Set dst_src (URShiftVS (Binary dst_src (RShiftCntV shift)) pg));
4156   ins_cost(SVE_COST);
4157   format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %}
4158   ins_encode %{
4159     int con = (int)$shift$$constant;
4160     assert(con > 0 && con < 16, "invalid shift immediate");
4161     __ sve_lsr(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con);
4162   %}
4163   ins_pipe(pipe_slow);
4164 %}
4165 
4166 instruct vlsrI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4167   predicate(UseSVE > 0);
4168   match(Set dst_src (URShiftVI (Binary dst_src (RShiftCntV shift)) pg));
4169   ins_cost(SVE_COST);
4170   format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %}
4171   ins_encode %{
4172     int con = (int)$shift$$constant;
4173     assert(con > 0 && con < 32, "invalid shift immediate");
4174     __ sve_lsr(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con);
4175   %}
4176   ins_pipe(pipe_slow);
4177 %}
4178 
4179 instruct vlsrL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4180   predicate(UseSVE > 0);
4181   match(Set dst_src (URShiftVL (Binary dst_src (RShiftCntV shift)) pg));
4182   ins_cost(SVE_COST);
4183   format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %}
4184   ins_encode %{
4185     int con = (int)$shift$$constant;
4186     assert(con > 0 && con < 64, "invalid shift immediate");
4187     __ sve_lsr(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con);
4188   %}
4189   ins_pipe(pipe_slow);
4190 %}
4191 
4192 instruct vlslB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4193   predicate(UseSVE > 0);
4194   match(Set dst_src (LShiftVB (Binary dst_src (LShiftCntV shift)) pg));
4195   ins_cost(SVE_COST);
4196   format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %}
4197   ins_encode %{
4198     int con = (int)$shift$$constant;
4199     assert(con >= 0 && con < 8, "invalid shift immediate");
4200     __ sve_lsl(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con);
4201   %}
4202   ins_pipe(pipe_slow);
4203 %}
4204 
4205 instruct vlslS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4206   predicate(UseSVE > 0);
4207   match(Set dst_src (LShiftVS (Binary dst_src (LShiftCntV shift)) pg));
4208   ins_cost(SVE_COST);
4209   format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %}
4210   ins_encode %{
4211     int con = (int)$shift$$constant;
4212     assert(con >= 0 && con < 16, "invalid shift immediate");
4213     __ sve_lsl(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con);
4214   %}
4215   ins_pipe(pipe_slow);
4216 %}
4217 
4218 instruct vlslI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4219   predicate(UseSVE > 0);
4220   match(Set dst_src (LShiftVI (Binary dst_src (LShiftCntV shift)) pg));
4221   ins_cost(SVE_COST);
4222   format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %}
4223   ins_encode %{
4224     int con = (int)$shift$$constant;
4225     assert(con >= 0 && con < 32, "invalid shift immediate");
4226     __ sve_lsl(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con);
4227   %}
4228   ins_pipe(pipe_slow);
4229 %}
4230 
4231 instruct vlslL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4232   predicate(UseSVE > 0);
4233   match(Set dst_src (LShiftVL (Binary dst_src (LShiftCntV shift)) pg));
4234   ins_cost(SVE_COST);
4235   format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %}
4236   ins_encode %{
4237     int con = (int)$shift$$constant;
4238     assert(con >= 0 && con < 64, "invalid shift immediate");
4239     __ sve_lsl(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con);
4240   %}
4241   ins_pipe(pipe_slow);
4242 %}
4243 
4244 // vector sqrt
4245 
4246 instruct vsqrtF(vReg dst, vReg src) %{
4247   predicate(UseSVE > 0 &&
4248             !n->as_Vector()->is_predicated_vector());
4249   match(Set dst (SqrtVF src));
4250   ins_cost(SVE_COST);
4251   format %{ "sve_fsqrt $dst, $src\t# vector (sve) (S)" %}
4252   ins_encode %{
4253     __ sve_fsqrt(as_FloatRegister($dst$$reg), __ S,
4254          ptrue, as_FloatRegister($src$$reg));
4255   %}
4256   ins_pipe(pipe_slow);
4257 %}
4258 
4259 instruct vsqrtD(vReg dst, vReg src) %{
4260   predicate(UseSVE > 0 &&
4261             !n->as_Vector()->is_predicated_vector());
4262   match(Set dst (SqrtVD src));
4263   ins_cost(SVE_COST);
4264   format %{ "sve_fsqrt $dst, $src\t# vector (sve) (D)" %}
4265   ins_encode %{
4266     __ sve_fsqrt(as_FloatRegister($dst$$reg), __ D,
4267          ptrue, as_FloatRegister($src$$reg));
4268   %}
4269   ins_pipe(pipe_slow);
4270 %}
4271 
4272 // vector sqrt - predicated
4273 
4274 instruct vsqrtF_masked(vReg dst_src, pRegGov pg) %{
4275   predicate(UseSVE > 0);
4276   match(Set dst_src (SqrtVF dst_src pg));
4277   ins_cost(SVE_COST);
4278   format %{ "sve_fsqrt $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
4279   ins_encode %{
4280     __ sve_fsqrt(as_FloatRegister($dst_src$$reg), __ S,
4281             as_PRegister($pg$$reg),
4282             as_FloatRegister($dst_src$$reg));
4283   %}
4284   ins_pipe(pipe_slow);
4285 %}
4286 
4287 instruct vsqrtD_masked(vReg dst_src, pRegGov pg) %{
4288   predicate(UseSVE > 0);
4289   match(Set dst_src (SqrtVD dst_src pg));
4290   ins_cost(SVE_COST);
4291   format %{ "sve_fsqrt $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
4292   ins_encode %{
4293     __ sve_fsqrt(as_FloatRegister($dst_src$$reg), __ D,
4294             as_PRegister($pg$$reg),
4295             as_FloatRegister($dst_src$$reg));
4296   %}
4297   ins_pipe(pipe_slow);
4298 %}
4299 
4300 // vector sub
4301 
4302 instruct vsubB(vReg dst, vReg src1, vReg src2) %{
4303   predicate(UseSVE > 0);
4304   match(Set dst (SubVB src1 src2));
4305   ins_cost(SVE_COST);
4306   format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (B)" %}
4307   ins_encode %{
4308     __ sve_sub(as_FloatRegister($dst$$reg), __ B,
4309          as_FloatRegister($src1$$reg),
4310          as_FloatRegister($src2$$reg));
4311   %}
4312   ins_pipe(pipe_slow);
4313 %}
4314 
4315 instruct vsubS(vReg dst, vReg src1, vReg src2) %{
4316   predicate(UseSVE > 0);
4317   match(Set dst (SubVS src1 src2));
4318   ins_cost(SVE_COST);
4319   format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (H)" %}
4320   ins_encode %{
4321     __ sve_sub(as_FloatRegister($dst$$reg), __ H,
4322          as_FloatRegister($src1$$reg),
4323          as_FloatRegister($src2$$reg));
4324   %}
4325   ins_pipe(pipe_slow);
4326 %}
4327 
4328 instruct vsubI(vReg dst, vReg src1, vReg src2) %{
4329   predicate(UseSVE > 0);
4330   match(Set dst (SubVI src1 src2));
4331   ins_cost(SVE_COST);
4332   format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (S)" %}
4333   ins_encode %{
4334     __ sve_sub(as_FloatRegister($dst$$reg), __ S,
4335          as_FloatRegister($src1$$reg),
4336          as_FloatRegister($src2$$reg));
4337   %}
4338   ins_pipe(pipe_slow);
4339 %}
4340 
4341 instruct vsubL(vReg dst, vReg src1, vReg src2) %{
4342   predicate(UseSVE > 0);
4343   match(Set dst (SubVL src1 src2));
4344   ins_cost(SVE_COST);
4345   format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (D)" %}
4346   ins_encode %{
4347     __ sve_sub(as_FloatRegister($dst$$reg), __ D,
4348          as_FloatRegister($src1$$reg),
4349          as_FloatRegister($src2$$reg));
4350   %}
4351   ins_pipe(pipe_slow);
4352 %}
4353 
4354 instruct vsubF(vReg dst, vReg src1, vReg src2) %{
4355   predicate(UseSVE > 0);
4356   match(Set dst (SubVF src1 src2));
4357   ins_cost(SVE_COST);
4358   format %{ "sve_fsub $dst, $src1, $src2\t # vector (sve) (S)" %}
4359   ins_encode %{
4360     __ sve_fsub(as_FloatRegister($dst$$reg), __ S,
4361          as_FloatRegister($src1$$reg),
4362          as_FloatRegister($src2$$reg));
4363   %}
4364   ins_pipe(pipe_slow);
4365 %}
4366 
4367 instruct vsubD(vReg dst, vReg src1, vReg src2) %{
4368   predicate(UseSVE > 0);
4369   match(Set dst (SubVD src1 src2));
4370   ins_cost(SVE_COST);
4371   format %{ "sve_fsub $dst, $src1, $src2\t # vector (sve) (D)" %}
4372   ins_encode %{
4373     __ sve_fsub(as_FloatRegister($dst$$reg), __ D,
4374          as_FloatRegister($src1$$reg),
4375          as_FloatRegister($src2$$reg));
4376   %}
4377   ins_pipe(pipe_slow);
4378 %}
4379 
4380 // vector sub - predicated
4381 
4382 instruct vsubB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4383   predicate(UseSVE > 0);
4384   match(Set dst_src1 (SubVB (Binary dst_src1 src2) pg));
4385   ins_cost(SVE_COST);
4386   format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
4387   ins_encode %{
4388     __ sve_sub(as_FloatRegister($dst_src1$$reg), __ B,
4389             as_PRegister($pg$$reg),
4390             as_FloatRegister($src2$$reg));
4391   %}
4392   ins_pipe(pipe_slow);
4393 %}
4394 
4395 instruct vsubS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4396   predicate(UseSVE > 0);
4397   match(Set dst_src1 (SubVS (Binary dst_src1 src2) pg));
4398   ins_cost(SVE_COST);
4399   format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
4400   ins_encode %{
4401     __ sve_sub(as_FloatRegister($dst_src1$$reg), __ H,
4402             as_PRegister($pg$$reg),
4403             as_FloatRegister($src2$$reg));
4404   %}
4405   ins_pipe(pipe_slow);
4406 %}
4407 
4408 instruct vsubI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4409   predicate(UseSVE > 0);
4410   match(Set dst_src1 (SubVI (Binary dst_src1 src2) pg));
4411   ins_cost(SVE_COST);
4412   format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
4413   ins_encode %{
4414     __ sve_sub(as_FloatRegister($dst_src1$$reg), __ S,
4415             as_PRegister($pg$$reg),
4416             as_FloatRegister($src2$$reg));
4417   %}
4418   ins_pipe(pipe_slow);
4419 %}
4420 
4421 instruct vsubL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4422   predicate(UseSVE > 0);
4423   match(Set dst_src1 (SubVL (Binary dst_src1 src2) pg));
4424   ins_cost(SVE_COST);
4425   format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
4426   ins_encode %{
4427     __ sve_sub(as_FloatRegister($dst_src1$$reg), __ D,
4428             as_PRegister($pg$$reg),
4429             as_FloatRegister($src2$$reg));
4430   %}
4431   ins_pipe(pipe_slow);
4432 %}
4433 
4434 instruct vsubF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4435   predicate(UseSVE > 0);
4436   match(Set dst_src1 (SubVF (Binary dst_src1 src2) pg));
4437   ins_cost(SVE_COST);
4438   format %{ "sve_fsub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
4439   ins_encode %{
4440     __ sve_fsub(as_FloatRegister($dst_src1$$reg), __ S,
4441             as_PRegister($pg$$reg),
4442             as_FloatRegister($src2$$reg));
4443   %}
4444   ins_pipe(pipe_slow);
4445 %}
4446 
4447 instruct vsubD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4448   predicate(UseSVE > 0);
4449   match(Set dst_src1 (SubVD (Binary dst_src1 src2) pg));
4450   ins_cost(SVE_COST);
4451   format %{ "sve_fsub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
4452   ins_encode %{
4453     __ sve_fsub(as_FloatRegister($dst_src1$$reg), __ D,
4454             as_PRegister($pg$$reg),
4455             as_FloatRegister($src2$$reg));
4456   %}
4457   ins_pipe(pipe_slow);
4458 %}
4459 
4460 // ------------------------------ Vector mask cast --------------------------
4461 
4462 instruct vmaskcast(pRegGov dst_src) %{
4463   predicate(UseSVE > 0 &&
4464             n->bottom_type()->is_vect()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
4465             n->bottom_type()->is_vect()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
4466   match(Set dst_src (VectorMaskCast dst_src));
4467   ins_cost(0);
4468   format %{ "vmaskcast $dst_src\t# empty (sve)" %}
4469   ins_encode %{
4470     // empty
4471   %}
4472   ins_pipe(pipe_class_empty);
4473 %}
4474 
4475 instruct vmaskcast_extend(pRegGov dst, pReg src)
4476 %{
4477   predicate(UseSVE > 0 &&
4478             (Matcher::vector_length_in_bytes(n) == 2 * Matcher::vector_length_in_bytes(n->in(1)) ||
4479              Matcher::vector_length_in_bytes(n) == 4 * Matcher::vector_length_in_bytes(n->in(1)) ||
4480              Matcher::vector_length_in_bytes(n) == 8 * Matcher::vector_length_in_bytes(n->in(1))));
4481   match(Set dst (VectorMaskCast src));
4482   ins_cost(SVE_COST * 3);
4483   format %{ "sve_vmaskcast_extend  $dst, $src\t# extend predicate $src" %}
4484   ins_encode %{
4485     __ sve_vmaskcast_extend(as_PRegister($dst$$reg), as_PRegister($src$$reg),
4486                             Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
4487   %}
4488   ins_pipe(pipe_slow);
4489 %}
4490 
4491 instruct vmaskcast_narrow(pRegGov dst, pReg src)
4492 %{
4493   predicate(UseSVE > 0 &&
4494             (Matcher::vector_length_in_bytes(n) * 2 == Matcher::vector_length_in_bytes(n->in(1)) ||
4495              Matcher::vector_length_in_bytes(n) * 4 == Matcher::vector_length_in_bytes(n->in(1)) ||
4496              Matcher::vector_length_in_bytes(n) * 8 == Matcher::vector_length_in_bytes(n->in(1))));
4497   match(Set dst (VectorMaskCast src));
4498   ins_cost(SVE_COST * 3);
4499   format %{ "sve_vmaskcast_narrow  $dst, $src\t# narrow predicate $src" %}
4500   ins_encode %{
4501     __ sve_vmaskcast_narrow(as_PRegister($dst$$reg), as_PRegister($src$$reg),
4502                             Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
4503   %}
4504   ins_pipe(pipe_slow);
4505 %}
4506 
4507 // ------------------------------ Vector cast -------------------------------
4508 
4509 instruct vcvtBtoX_extend(vReg dst, vReg src)
4510 %{
4511   predicate(UseSVE > 0);
4512   match(Set dst (VectorCastB2X src));
4513   ins_cost(2 * SVE_COST);
4514   format %{ "sve_vectorcast_b2x  $dst, $src\t# convert B to X vector (extend)" %}
4515   ins_encode %{
4516     BasicType to_bt = Matcher::vector_element_basic_type(this);
4517     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4518     __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ B);
4519     if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
4520       __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
4521     }
4522   %}
4523   ins_pipe(pipe_slow);
4524 %}
4525 
4526 instruct vcvtStoB(vReg dst, vReg src, vReg tmp)
4527 %{
4528   predicate(UseSVE > 0 &&
4529             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
4530   match(Set dst (VectorCastS2X src));
4531   effect(TEMP tmp);
4532   ins_cost(2 * SVE_COST);
4533   format %{ "sve_vectorcast_s2b  $dst, $src\t# convert H to B vector" %}
4534   ins_encode %{
4535     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
4536                          as_FloatRegister($src$$reg), __ H, as_FloatRegister($tmp$$reg));
4537   %}
4538   ins_pipe(pipe_slow);
4539 %}
4540 
4541 instruct vcvtStoX_extend(vReg dst, vReg src)
4542 %{
4543   predicate(UseSVE > 0 &&
4544             type2aelembytes(Matcher::vector_element_basic_type(n)) > 2);
4545   match(Set dst (VectorCastS2X src));
4546   ins_cost(2 * SVE_COST);
4547   format %{ "sve_vectorcast_s2x  $dst, $src\t# convert H to X vector (extend)" %}
4548   ins_encode %{
4549     BasicType to_bt = Matcher::vector_element_basic_type(this);
4550     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4551     __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ H);
4552     if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
4553       __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
4554     }
4555   %}
4556   ins_pipe(pipe_slow);
4557 %}
4558 
4559 instruct vcvtItoB(vReg dst, vReg src, vReg tmp)
4560 %{
4561   predicate(UseSVE > 0 &&
4562             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
4563   match(Set dst (VectorCastI2X src));
4564   effect(TEMP_DEF dst, TEMP tmp);
4565   ins_cost(3 * SVE_COST);
4566   format %{ "sve_vectorcast_i2b  $dst, $src\t# convert I to B vector" %}
4567   ins_encode %{
4568     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
4569                          as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
4570   %}
4571   ins_pipe(pipe_slow);
4572 %}
4573 
4574 instruct vcvtItoS(vReg dst, vReg src, vReg tmp)
4575 %{
4576   predicate(UseSVE > 0 &&
4577             n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
4578   match(Set dst (VectorCastI2X src));
4579   effect(TEMP tmp);
4580   ins_cost(2 * SVE_COST);
4581   format %{ "sve_vectorcast_i2s $dst, $src\t# convert I to H vector" %}
4582   ins_encode %{
4583     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ H,
4584                          as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
4585   %}
4586   ins_pipe(pipe_slow);
4587 %}
4588 
4589 instruct vcvtItoL(vReg dst, vReg src)
4590 %{
4591   predicate(UseSVE > 0 &&
4592             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
4593   match(Set dst (VectorCastI2X src));
4594   ins_cost(SVE_COST);
4595   format %{ "sve_vectorcast_i2l  $dst, $src\t# convert I to L vector" %}
4596   ins_encode %{
4597     __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
4598   %}
4599   ins_pipe(pipe_slow);
4600 %}
4601 
4602 instruct vcvtItoF(vReg dst, vReg src)
4603 %{
4604   predicate(UseSVE > 0 &&
4605             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4606   match(Set dst (VectorCastI2X src));
4607   ins_cost(SVE_COST);
4608   format %{ "sve_vectorcast_i2f  $dst, $src\t# convert I to F vector" %}
4609   ins_encode %{
4610     __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
4611   %}
4612   ins_pipe(pipe_slow);
4613 %}
4614 
4615 instruct vcvtItoD(vReg dst, vReg src)
4616 %{
4617   predicate(UseSVE > 0 &&
4618             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
4619   match(Set dst (VectorCastI2X src));
4620   ins_cost(2 * SVE_COST);
4621   format %{ "sve_vectorcast_i2d  $dst, $src\t# convert I to D vector" %}
4622   ins_encode %{
4623     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
4624     __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ D);
4625   %}
4626   ins_pipe(pipe_slow);
4627 %}
4628 
4629 instruct vcvtLtoX_narrow(vReg dst, vReg src, vReg tmp)
4630 %{
4631   predicate(UseSVE > 0 && is_integral_type(Matcher::vector_element_basic_type(n)));
4632   match(Set dst (VectorCastL2X src));
4633   effect(TEMP_DEF dst, TEMP tmp);
4634   ins_cost(2 * SVE_COST);
4635   format %{ "sve_vectorcast_l2x  $dst, $src\t# convert L to B/H/S vector (narrow)" %}
4636   ins_encode %{
4637     BasicType to_bt = Matcher::vector_element_basic_type(this);
4638     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4639     __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
4640                          as_FloatRegister($src$$reg), __ D, as_FloatRegister($tmp$$reg));
4641   %}
4642   ins_pipe(pipe_slow);
4643 %}
4644 
4645 instruct vcvtLtoF(vReg dst, vReg src, vReg tmp)
4646 %{
4647   predicate(UseSVE > 0 &&
4648             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4649   match(Set dst (VectorCastL2X src));
4650   effect(TEMP_DEF dst, TEMP tmp);
4651   ins_cost(3 * SVE_COST);
4652   format %{ "sve_vectorcast_l2f  $dst, $src\t# convert L to F vector" %}
4653   ins_encode %{
4654     __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
4655     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
4656                          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
4657 
4658   %}
4659   ins_pipe(pipe_slow);
4660 %}
4661 
4662 instruct vcvtLtoD(vReg dst, vReg src)
4663 %{
4664   predicate(UseSVE > 0 &&
4665             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
4666   match(Set dst (VectorCastL2X src));
4667   ins_cost(SVE_COST);
4668   format %{ "sve_vectorcast_l2d  $dst, $src\t# convert L to D vector" %}
4669   ins_encode %{
4670     __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
4671   %}
4672   ins_pipe(pipe_slow);
4673 %}
4674 
4675 instruct vcvtFtoX_narrow(vReg dst, vReg src, vReg tmp)
4676 %{
4677   predicate(UseSVE > 0 &&
4678             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4679              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT));
4680   match(Set dst (VectorCastF2X src));
4681   effect(TEMP_DEF dst, TEMP tmp);
4682   ins_cost(3 * SVE_COST);
4683   format %{ "sve_vectorcast_f2x  $dst, $src\t# convert F to B/H vector" %}
4684   ins_encode %{
4685     BasicType to_bt = Matcher::vector_element_basic_type(this);
4686     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4687     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
4688     __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
4689                          as_FloatRegister($dst$$reg), __ S, as_FloatRegister($tmp$$reg));
4690   %}
4691   ins_pipe(pipe_slow);
4692 %}
4693 
4694 instruct vcvtFtoI(vReg dst, vReg src)
4695 %{
4696   predicate(UseSVE > 0 &&
4697             (n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4698   match(Set dst (VectorCastF2X src));
4699   ins_cost(SVE_COST);
4700   format %{ "sve_vectorcast_f2x  $dst, $src\t# convert F to I vector" %}
4701   ins_encode %{
4702     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
4703   %}
4704   ins_pipe(pipe_slow);
4705 %}
4706 
4707 instruct vcvtFtoL(vReg dst, vReg src)
4708 %{
4709   predicate(UseSVE > 0 &&
4710             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
4711   match(Set dst (VectorCastF2X src));
4712   ins_cost(SVE_COST * 2);
4713   format %{ "sve_vectorcast_f2x  $dst, $src\t# convert F to L vector" %}
4714   ins_encode %{
4715     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
4716     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);
4717   %}
4718   ins_pipe(pipe_slow);
4719 %}
4720 
4721 instruct vcvtFtoD(vReg dst, vReg src)
4722 %{
4723   predicate(UseSVE > 0 &&
4724             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
4725   match(Set dst (VectorCastF2X src));
4726   ins_cost(2 * SVE_COST);
4727   format %{ "sve_vectorcast_f2d  $dst, $dst\t# convert F to D vector" %}
4728   ins_encode %{
4729     __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
4730     __ sve_fcvt(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);
4731   %}
4732   ins_pipe(pipe_slow);
4733 %}
4734 
4735 instruct vcvtDtoX_narrow(vReg dst, vReg src, vReg tmp)
4736 %{
4737   predicate(UseSVE > 0 &&
4738             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4739              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
4740              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4741   match(Set dst (VectorCastD2X src));
4742   effect(TEMP_DEF dst, TEMP tmp);
4743   ins_cost(3 * SVE_COST);
4744   format %{ "sve_vectorcast_d2x  $dst, $src\t# convert D to X vector (narrow)" %}
4745   ins_encode %{
4746     BasicType to_bt = Matcher::vector_element_basic_type(this);
4747     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4748     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
4749     __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
4750                          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
4751   %}
4752   ins_pipe(pipe_slow);
4753 %}
4754 
4755 instruct vcvtDtoL(vReg dst, vReg src)
4756 %{
4757   predicate(UseSVE > 0 &&
4758             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
4759   match(Set dst (VectorCastD2X src));
4760   ins_cost(SVE_COST);
4761   format %{ "sve_vectorcast_d2l  $dst, $src\t# convert D to L vector" %}
4762   ins_encode %{
4763     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
4764   %}
4765   ins_pipe(pipe_slow);
4766 %}
4767 
4768 instruct vcvtDtoF(vReg dst, vReg src, vReg tmp)
4769 %{
4770   predicate(UseSVE > 0 &&
4771             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4772   match(Set dst (VectorCastD2X src));
4773   effect(TEMP_DEF dst, TEMP tmp);
4774   ins_cost(3 * SVE_COST);
4775   format %{ "sve_vectorcast_d2f  $dst, S, $dst\t# convert D to F vector" %}
4776   ins_encode %{
4777     __ sve_fcvt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
4778     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
4779                          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
4780   %}
4781   ins_pipe(pipe_slow);
4782 %}
4783 
4784 // ------------------------------ Vector extract ---------------------------------
4785 
4786 instruct extractB(iRegINoSp dst, vReg src, immI idx, vReg vtmp)
4787 %{
4788   predicate(UseSVE > 0 && n->in(2)->get_int() >= 16);
4789   match(Set dst (ExtractB src idx));
4790   effect(TEMP vtmp);
4791   ins_cost(2 * SVE_COST);
4792   format %{ "sve_extract_integral $dst, B, $src, $idx\t# extract from vector(B)" %}
4793   ins_encode %{
4794     __ sve_extract_integral(as_Register($dst$$reg), __ B, as_FloatRegister($src$$reg),
4795                             (int)($idx$$constant), /* is_signed */ true, as_FloatRegister($vtmp$$reg));
4796   %}
4797   ins_pipe(pipe_slow);
4798 %}
4799 
4800 instruct extractS(iRegINoSp dst, vReg src, immI idx, vReg vtmp)
4801 %{
4802   predicate(UseSVE > 0 && n->in(2)->get_int() >= 8);
4803   match(Set dst (ExtractS src idx));
4804   effect(TEMP vtmp);
4805   ins_cost(2 * SVE_COST);
4806   format %{ "sve_extract_integral $dst, H, $src, $idx\t# extract from vector(S)" %}
4807   ins_encode %{
4808     __ sve_extract_integral(as_Register($dst$$reg), __ H, as_FloatRegister($src$$reg),
4809                             (int)($idx$$constant), /* is_signed */ true, as_FloatRegister($vtmp$$reg));
4810   %}
4811   ins_pipe(pipe_slow);
4812 %}
4813 
4814 instruct extractI(iRegINoSp dst, vReg src, immI idx, vReg vtmp)
4815 %{
4816   predicate(UseSVE > 0 && n->in(2)->get_int() >= 4);
4817   match(Set dst (ExtractI src idx));
4818   effect(TEMP vtmp);
4819   ins_cost(2 * SVE_COST);
4820   format %{ "sve_extract_integral $dst, S, $src, $idx\t# extract from vector(I)" %}
4821   ins_encode %{
4822     __ sve_extract_integral(as_Register($dst$$reg), __ S, as_FloatRegister($src$$reg),
4823                             (int)($idx$$constant), /* is_signed */ true, as_FloatRegister($vtmp$$reg));
4824   %}
4825   ins_pipe(pipe_slow);
4826 %}
4827 
4828 instruct extractL(iRegLNoSp dst, vReg src, immI idx, vReg vtmp)
4829 %{
4830   predicate(UseSVE > 0 && n->in(2)->get_int() >= 2);
4831   match(Set dst (ExtractL src idx));
4832   effect(TEMP vtmp);
4833   ins_cost(2 * SVE_COST);
4834   format %{ "sve_extract_integral $dst, D, $src, $idx\t# extract from vector(L)" %}
4835   ins_encode %{
4836     __ sve_extract_integral(as_Register($dst$$reg), __ D, as_FloatRegister($src$$reg),
4837                             (int)($idx$$constant), /* is_signed */ false, as_FloatRegister($vtmp$$reg));
4838   %}
4839   ins_pipe(pipe_slow);
4840 %}
4841 
4842 instruct extractB_LT16(iRegINoSp dst, vReg src, immI idx)
4843 %{
4844   predicate(UseSVE > 0 && n->in(2)->get_int() < 16);
4845   match(Set dst (ExtractB src idx));
4846   ins_cost(INSN_COST);
4847   format %{ "smov $dst, B, $src, $idx\t# extract from vector(B)" %}
4848   ins_encode %{
4849     __ smov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ B, $idx$$constant);
4850   %}
4851   ins_pipe(pipe_class_default);
4852 %}
4853 
4854 instruct extractS_LT8(iRegINoSp dst, vReg src, immI idx)
4855 %{
4856   predicate(UseSVE > 0 && n->in(2)->get_int() < 8);
4857   match(Set dst (ExtractS src idx));
4858   ins_cost(INSN_COST);
4859   format %{ "smov $dst, H, $src, $idx\t# extract from vector(S)" %}
4860   ins_encode %{
4861     __ smov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ H, $idx$$constant);
4862   %}
4863   ins_pipe(pipe_class_default);
4864 %}
4865 
4866 instruct extractI_LT4(iRegINoSp dst, vReg src, immI idx)
4867 %{
4868   predicate(UseSVE > 0 && n->in(2)->get_int() < 4);
4869   match(Set dst (ExtractI src idx));
4870   ins_cost(INSN_COST);
4871   format %{ "smov $dst, S, $src, $idx\t# extract from vector(I)" %}
4872   ins_encode %{
4873     __ smov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ S, $idx$$constant);
4874   %}
4875   ins_pipe(pipe_class_default);
4876 %}
4877 
4878 instruct extractL_LT2(iRegLNoSp dst, vReg src, immI idx)
4879 %{
4880   predicate(UseSVE > 0 && n->in(2)->get_int() < 2);
4881   match(Set dst (ExtractL src idx));
4882   ins_cost(INSN_COST);
4883   format %{ "umov $dst, D, $src, $idx\t# extract from vector(L)" %}
4884   ins_encode %{
4885     __ umov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ D, $idx$$constant);
4886   %}
4887   ins_pipe(pipe_class_default);
4888 %}
4889 
4890 instruct extractF(vRegF dst, vReg src, immI idx)
4891 %{
4892   predicate(UseSVE > 0);
4893   match(Set dst (ExtractF src idx));
4894   ins_cost(2 * SVE_COST);
4895   format %{ "sve_extract_f $dst, S, $src, $idx\t# extract from vector(F)" %}
4896   ins_encode %{
4897     if ((as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg)) && ($idx$$constant == 0)) {
4898       /* empty */
4899     } else if ($idx$$constant == 0) {
4900       __ fmovs(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
4901     } else if ($idx$$constant < 4) {
4902       __ ins(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($src$$reg), 0, (int)($idx$$constant));
4903     } else {
4904       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4905       __ sve_ext(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), $idx$$constant << 2);
4906     }
4907   %}
4908   ins_pipe(pipe_slow);
4909 %}
4910 
4911 instruct extractD(vRegD dst, vReg src, immI idx)
4912 %{
4913   predicate(UseSVE > 0);
4914   match(Set dst (ExtractD src idx));
4915   ins_cost(2 * SVE_COST);
4916   format %{ "sve_extract_d $dst, D, $src, $idx\t# extract from vector(D)" %}
4917   ins_encode %{
4918     if ((as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg)) && ($idx$$constant == 0)) {
4919       /* empty */
4920     } else if ($idx$$constant == 0) {
4921       __ fmovd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
4922     } else if ($idx$$constant == 1) {
4923       __ ins(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), 0, 1);
4924     } else {
4925       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4926       __ sve_ext(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), $idx$$constant << 3);
4927     }
4928   %}
4929   ins_pipe(pipe_slow);
4930 %}
4931 
4932 // ------------------------------- VectorTest ----------------------------------
4933 
4934 instruct vtest_alltrue(iRegINoSp dst, pRegGov src1, pRegGov src2, pReg ptmp, rFlagsReg cr)
4935 %{
4936   predicate(UseSVE > 0 &&
4937             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
4938   match(Set dst (VectorTest src1 src2));
4939   effect(TEMP ptmp, KILL cr);
4940   ins_cost(SVE_COST);
4941   format %{ "sve_eors $ptmp, $src1, $src2\t# $src2 is all true mask\n"
4942             "csetw $dst, EQ\t# VectorTest (sve) - alltrue" %}
4943   ins_encode %{
4944     __ sve_eors(as_PRegister($ptmp$$reg), ptrue,
4945                 as_PRegister($src1$$reg), as_PRegister($src2$$reg));
4946     __ csetw(as_Register($dst$$reg), Assembler::EQ);
4947   %}
4948   ins_pipe(pipe_slow);
4949 %}
4950 
4951 instruct vtest_anytrue(iRegINoSp dst, pRegGov src1, pRegGov src2, rFlagsReg cr)
4952 %{
4953   predicate(UseSVE > 0 &&
4954             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
4955   match(Set dst (VectorTest src1 src2));
4956   effect(KILL cr);
4957   ins_cost(SVE_COST);
4958   format %{ "sve_ptest $src1\n\t"
4959             "csetw $dst, NE\t# VectorTest (sve) - anytrue" %}
4960   ins_encode %{
4961     // "src2" is not used for sve.
4962     __ sve_ptest(ptrue, as_PRegister($src1$$reg));
4963     __ csetw(as_Register($dst$$reg), Assembler::NE);
4964   %}
4965   ins_pipe(pipe_slow);
4966 %}
4967 // ------------------------------ Vector insert ---------------------------------
4968 
4969 instruct insertI_le128bits(vReg dst, vReg src, iRegIorL2I val, immI idx) %{
4970   predicate(UseSVE > 0 &&
4971             (Matcher::vector_length_in_bytes(n) == 8 || Matcher::vector_length_in_bytes(n) == 16) &&
4972             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4973              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
4974              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4975   match(Set dst (VectorInsert (Binary src val) idx));
4976   ins_cost(2 * INSN_COST);
4977   format %{ "orr    $dst, T8/16B, $src, $src\n\t"
4978             "mov    $dst, B/H/S, $idx, $val\t# insertI into vector(64/128bits)" %}
4979   ins_encode %{
4980     BasicType bt = Matcher::vector_element_basic_type(this);
4981     if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
4982       __ orr(as_FloatRegister($dst$$reg), Matcher::vector_length_in_bytes(this) == 8 ? __ T8B : __ T16B,
4983              as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4984     }
4985     __ mov(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(Matcher::vector_element_basic_type(this)),
4986            $idx$$constant, $val$$Register);
4987   %}
4988   ins_pipe(pipe_slow);
4989 %}
4990 
4991 instruct insertI_small_index(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg vtmp, pRegGov pgtmp, rFlagsReg cr) %{
4992   predicate(UseSVE > 0 && n->in(2)->get_int() < 32 &&
4993             Matcher::vector_length_in_bytes(n) > 16 &&
4994             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4995              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
4996              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4997   match(Set dst (VectorInsert (Binary src val) idx));
4998   effect(TEMP vtmp, TEMP pgtmp, KILL cr);
4999   ins_cost(4 * SVE_COST);
5000   format %{ "sve_index $vtmp, -16, 1\t# (B/H/S)\n\t"
5001             "sve_cmpeq $pgtmp, $vtmp, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
5002             "sve_orr $dst, $src, $src\n\t"
5003             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
5004   ins_encode %{
5005     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5006     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5007     __ block_comment("insert into vector (B/H/S) {");
5008       __ sve_index(as_FloatRegister($vtmp$$reg), size, -16, 1);
5009       __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
5010                  as_FloatRegister($vtmp$$reg), (int)($idx$$constant) - 16);
5011       if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
5012         __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
5013       }
5014       __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), $val$$Register);
5015     __ block_comment("} insert into vector (B/H/S)");
5016   %}
5017   ins_pipe(pipe_slow);
5018 %}
5019 
5020 instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg vtmp1, vReg vtmp2, pRegGov pgtmp, rFlagsReg cr) %{
5021   predicate(UseSVE > 0 && n->in(2)->get_int() >= 32 &&
5022             Matcher::vector_length_in_bytes(n) > 16 &&
5023             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
5024              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
5025              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
5026   match(Set dst (VectorInsert (Binary src val) idx));
5027   effect(TEMP vtmp1, TEMP vtmp2, TEMP pgtmp, KILL cr);
5028   ins_cost(5 * SVE_COST);
5029   format %{ "sve_index $vtmp1, 0, 1\t# (B/H/S)\n\t"
5030             "sve_dup $vtmp2, $idx\t# (B/H/S)\n\t"
5031             "sve_cmpeq $pgtmp, $vtmp1, $vtmp2\n\t"
5032             "sve_orr $dst, $src, $src\n\t"
5033             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
5034   ins_encode %{
5035     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5036     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5037     __ block_comment("insert into vector (B/H/S) {");
5038       __ sve_index(as_FloatRegister($vtmp1$$reg), size, 0, 1);
5039       __ sve_dup(as_FloatRegister($vtmp2$$reg), size, (int)($idx$$constant));
5040       __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
5041                  as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg));
5042       if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
5043         __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
5044       }
5045       __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), $val$$Register);
5046     __ block_comment("} insert into vector (B/H/S)");
5047   %}
5048   ins_pipe(pipe_slow);
5049 %}
5050 
5051 instruct insertL_128bits(vReg dst, vReg src, iRegL val, immI idx) %{
5052   predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) == 16 &&
5053             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
5054   match(Set dst (VectorInsert (Binary src val) idx));
5055   ins_cost(2 * INSN_COST);
5056   format %{ "orr    $dst, T16B, $src, $src\n\t"
5057             "mov    $dst, D, $idx, $val\t# insertL into vector(128bits)" %}
5058   ins_encode %{
5059     BasicType bt = Matcher::vector_element_basic_type(this);
5060     if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
5061       __ orr(as_FloatRegister($dst$$reg), __ T16B,
5062              as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
5063     }
5064     __ mov(as_FloatRegister($dst$$reg), __ D, $idx$$constant, $val$$Register);
5065   %}
5066   ins_pipe(pipe_slow);
5067 %}
5068 
5069 instruct insertL(vReg dst, vReg src, iRegL val, immI idx, vReg vtmp, pRegGov pgtmp, rFlagsReg cr) %{
5070   predicate(UseSVE > 0 &&
5071             Matcher::vector_length_in_bytes(n) > 16 &&
5072             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
5073   match(Set dst (VectorInsert (Binary src val) idx));
5074   effect(TEMP vtmp, TEMP pgtmp, KILL cr);
5075   ins_cost(4 * SVE_COST);
5076   format %{ "sve_index $vtmp, D, -16, 1\n\t"
5077             "sve_cmpeq $pgtmp, $vtmp, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
5078             "sve_orr $dst, $src, $src\n\t"
5079             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (L)" %}
5080   ins_encode %{
5081     __ block_comment("insert into vector (L) {");
5082       __ sve_index(as_FloatRegister($vtmp$$reg), __ D, -16, 1);
5083       __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
5084                  as_FloatRegister($vtmp$$reg), (int)($idx$$constant) - 16);
5085       if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
5086         __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
5087       }
5088       __ sve_cpy(as_FloatRegister($dst$$reg), __ D,
5089                  as_PRegister($pgtmp$$reg), $val$$Register);
5090     __ block_comment("} insert into vector (L)");
5091   %}
5092   ins_pipe(pipe_slow);
5093 %}
5094 
5095 instruct insertF_le128bits(vReg dst, vReg src, vRegF val, immI idx) %{
5096   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
5097             (Matcher::vector_length_in_bytes(n) == 8 || Matcher::vector_length_in_bytes(n) == 16));
5098   match(Set dst (VectorInsert (Binary src val) idx));
5099   ins_cost(2 * INSN_COST);
5100   effect(TEMP_DEF dst);
5101   format %{ "orr    $dst, T8/16B, $src, $src\n\t"
5102             "ins    $dst, S, $val, $idx, 0\t# insertF into vector(64/128bits)" %}
5103   ins_encode %{
5104     __ orr(as_FloatRegister($dst$$reg), Matcher::vector_length_in_bytes(this) == 8 ? __ T8B : __ T16B,
5105            as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
5106     __ ins(as_FloatRegister($dst$$reg), __ S,
5107            as_FloatRegister($val$$reg), $idx$$constant, 0);
5108   %}
5109   ins_pipe(pipe_slow);
5110 %}
5111 
5112 instruct insertF_small_index(vReg dst, vReg src, vRegF val, immI idx, pRegGov pgtmp, rFlagsReg cr) %{
5113   predicate(UseSVE > 0 && n->in(2)->get_int() < 32 &&
5114             Matcher::vector_length_in_bytes(n) > 16 &&
5115             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
5116   match(Set dst (VectorInsert (Binary src val) idx));
5117   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
5118   ins_cost(4 * SVE_COST);
5119   format %{ "sve_index $dst, S, -16, 1\n\t"
5120             "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
5121             "sve_orr $dst, $src, $src\n\t"
5122             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
5123   ins_encode %{
5124     __ block_comment("insert into vector (F) {");
5125       __ sve_index(as_FloatRegister($dst$$reg), __ S, -16, 1);
5126       __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
5127                  as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
5128       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
5129       __ sve_cpy(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
5130     __ block_comment("} insert into vector (F)");
5131   %}
5132   ins_pipe(pipe_slow);
5133 %}
5134 
5135 instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr) %{
5136   predicate(UseSVE > 0 && n->in(2)->get_int() >= 32 &&
5137             Matcher::vector_length_in_bytes(n) > 16 &&
5138             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
5139   match(Set dst (VectorInsert (Binary src val) idx));
5140   effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
5141   ins_cost(5 * SVE_COST);
5142   format %{ "sve_index $tmp1, S, 0, 1\n\t"
5143             "sve_dup $dst, S, $idx\n\t"
5144             "sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
5145             "sve_orr $dst, $src, $src\n\t"
5146             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
5147   ins_encode %{
5148     __ block_comment("insert into vector (F) {");
5149       __ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
5150       __ sve_dup(as_FloatRegister($dst$$reg), __ S, (int)($idx$$constant));
5151       __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
5152                  as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
5153       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
5154                  as_FloatRegister($src$$reg));
5155       __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
5156                  as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
5157     __ block_comment("} insert into vector (F)");
5158   %}
5159   ins_pipe(pipe_slow);
5160 %}
5161 
5162 instruct insertD_128bits(vReg dst, vReg src, vRegD val, immI idx) %{
5163   predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) == 16 &&
5164             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
5165   match(Set dst (VectorInsert (Binary src val) idx));
5166   ins_cost(2 * INSN_COST);
5167   effect(TEMP_DEF dst);
5168   format %{ "orr    $dst, T16B, $src, $src\n\t"
5169             "ins    $dst, D, $val, $idx, 0\t# insertD into vector(128bits)" %}
5170   ins_encode %{
5171     __ orr(as_FloatRegister($dst$$reg), __ T16B,
5172            as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
5173     __ ins(as_FloatRegister($dst$$reg), __ D,
5174            as_FloatRegister($val$$reg), $idx$$constant, 0);
5175   %}
5176   ins_pipe(pipe_slow);
5177 %}
5178 
5179 instruct insertD(vReg dst, vReg src, vRegD val, immI idx, pRegGov pgtmp, rFlagsReg cr) %{
5180   predicate(UseSVE > 0 &&
5181             Matcher::vector_length_in_bytes(n) > 16 &&
5182             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
5183   match(Set dst (VectorInsert (Binary src val) idx));
5184   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
5185   ins_cost(4 * SVE_COST);
5186   format %{ "sve_index $dst, D, -16, 1\n\t"
5187             "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
5188             "sve_orr $dst, $src, $src\n\t"
5189             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (D)" %}
5190   ins_encode %{
5191     __ block_comment("insert into vector (D) {");
5192       __ sve_index(as_FloatRegister($dst$$reg), __ D, -16, 1);
5193       __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
5194                  as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
5195       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
5196                  as_FloatRegister($src$$reg));
5197       __ sve_cpy(as_FloatRegister($dst$$reg), __ D,
5198                  as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
5199     __ block_comment("} insert into vector (D)");
5200   %}
5201   ins_pipe(pipe_slow);
5202 %}
5203 
5204 // ------------------------------ Vector shuffle -------------------------------
5205 
5206 instruct loadshuffle(vReg dst, vReg src) %{
5207   predicate(UseSVE > 0);
5208   match(Set dst (VectorLoadShuffle src));
5209   ins_cost(SVE_COST);
5210   format %{ "sve_loadshuffle $dst, $src\t# vector load shuffle (B/H/S/D)" %}
5211   ins_encode %{
5212     BasicType bt = Matcher::vector_element_basic_type(this);
5213     if (bt == T_BYTE) {
5214       if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
5215         __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
5216                    as_FloatRegister($src$$reg));
5217       }
5218     } else {
5219       __ sve_vector_extend(as_FloatRegister($dst$$reg),  __ elemType_to_regVariant(bt),
5220                            as_FloatRegister($src$$reg), __ B);
5221     }
5222   %}
5223   ins_pipe(pipe_slow);
5224 %}
5225 
5226 // ------------------------------ Vector rearrange -------------------------------
5227 
5228 instruct rearrange(vReg dst, vReg src, vReg shuffle)
5229 %{
5230   predicate(UseSVE > 0);
5231   match(Set dst (VectorRearrange src shuffle));
5232   ins_cost(SVE_COST);
5233   format %{ "sve_tbl $dst, $src, $shuffle\t# vector rearrange" %}
5234   ins_encode %{
5235     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5236     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5237     __ sve_tbl(as_FloatRegister($dst$$reg), size,
5238                as_FloatRegister($src$$reg), as_FloatRegister($shuffle$$reg));
5239   %}
5240   ins_pipe(pipe_slow);
5241 %}
5242 
5243 // ------------------------------ Vector Load Gather ---------------------------------
5244 
5245 instruct gatherI(vReg dst, indirect mem, vReg idx) %{
5246   predicate(UseSVE > 0 &&
5247             n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
5248             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5249              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5250   match(Set dst (LoadVectorGather mem idx));
5251   ins_cost(SVE_COST);
5252   format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (S)" %}
5253   ins_encode %{
5254     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), ptrue,
5255                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5256   %}
5257   ins_pipe(pipe_slow);
5258 %}
5259 
5260 instruct gatherL(vReg dst, indirect mem, vReg idx) %{
5261   predicate(UseSVE > 0 &&
5262             n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
5263             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5264              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5265   match(Set dst (LoadVectorGather mem idx));
5266   ins_cost(2 * SVE_COST);
5267   format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (D)" %}
5268   ins_encode %{
5269     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5270     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), ptrue, as_Register($mem$$base),
5271                        as_FloatRegister($idx$$reg));
5272   %}
5273   ins_pipe(pipe_slow);
5274 %}
5275 
5276 // ------------------------------ Vector Load Gather Partial-------------------------------
5277 
5278 instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5279   predicate(UseSVE > 0 &&
5280             n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
5281             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5282              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5283   match(Set dst (LoadVectorGather mem idx));
5284   effect(TEMP ptmp, KILL cr);
5285   ins_cost(2 * SVE_COST + INSN_COST);
5286   format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (S)" %}
5287   ins_encode %{
5288     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this));
5289     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
5290                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5291   %}
5292   ins_pipe(pipe_slow);
5293 %}
5294 
5295 instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5296   predicate(UseSVE > 0 &&
5297             n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
5298             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5299              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5300   match(Set dst (LoadVectorGather mem idx));
5301   effect(TEMP ptmp, KILL cr);
5302   ins_cost(3 * SVE_COST + INSN_COST);
5303   format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (D)" %}
5304   ins_encode %{
5305     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this));
5306     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5307     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
5308                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5309   %}
5310   ins_pipe(pipe_slow);
5311 %}
5312 
5313 // ------------------------------ Vector Load Gather Predicated -------------------------------
5314 
5315 instruct gatherI_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
5316   predicate(UseSVE > 0 &&
5317             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5318              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5319   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
5320   ins_cost(SVE_COST);
5321   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (S)" %}
5322   ins_encode %{
5323     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
5324                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5325   %}
5326   ins_pipe(pipe_slow);
5327 %}
5328 
5329 instruct gatherL_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
5330   predicate(UseSVE > 0 &&
5331             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5332              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5333   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
5334   ins_cost(2 * SVE_COST);
5335   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (D)" %}
5336   ins_encode %{
5337     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5338     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
5339                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
5340   %}
5341   ins_pipe(pipe_slow);
5342 %}
5343 
5344 // ------------------------------ Vector Store Scatter -------------------------------
5345 
5346 instruct scatterI(indirect mem, vReg src, vReg idx) %{
5347   predicate(UseSVE > 0 &&
5348             n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
5349             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5350              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5351   match(Set mem (StoreVectorScatter mem (Binary src idx)));
5352   ins_cost(SVE_COST);
5353   format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (S)" %}
5354   ins_encode %{
5355     __ sve_st1w_scatter(as_FloatRegister($src$$reg), ptrue,
5356                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5357   %}
5358   ins_pipe(pipe_slow);
5359 %}
5360 
5361 instruct scatterL(indirect mem, vReg src, vReg idx) %{
5362   predicate(UseSVE > 0 &&
5363             n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
5364             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5365              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5366   match(Set mem (StoreVectorScatter mem (Binary src idx)));
5367   ins_cost(2 * SVE_COST);
5368   format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (D)" %}
5369   ins_encode %{
5370     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5371     __ sve_st1d_scatter(as_FloatRegister($src$$reg), ptrue,
5372                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5373   %}
5374   ins_pipe(pipe_slow);
5375 %}
5376 
5377 // ------------------------------ Vector Store Scatter Partial -------------------------------
5378 
5379 instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5380   predicate(UseSVE > 0 &&
5381             n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
5382             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5383              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5384   match(Set mem (StoreVectorScatter mem (Binary src idx)));
5385   effect(TEMP ptmp, KILL cr);
5386   ins_cost(2 * SVE_COST + INSN_COST);
5387   format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (S)" %}
5388   ins_encode %{
5389     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src));
5390     __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
5391                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5392   %}
5393   ins_pipe(pipe_slow);
5394 %}
5395 
5396 instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5397   predicate(UseSVE > 0 &&
5398             n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
5399             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5400              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5401   match(Set mem (StoreVectorScatter mem (Binary src idx)));
5402   effect(TEMP ptmp, KILL cr);
5403   ins_cost(3 * SVE_COST + INSN_COST);
5404   format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (D)" %}
5405   ins_encode %{
5406     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src));
5407     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5408     __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
5409                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5410   %}
5411   ins_pipe(pipe_slow);
5412 %}
5413 
5414 // ------------------------------ Vector Store Scatter Predicated -------------------------------
5415 
5416 instruct scatterI_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
5417   predicate(UseSVE > 0 &&
5418             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5419              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5420   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
5421   ins_cost(SVE_COST);
5422   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicate (S)" %}
5423   ins_encode %{
5424     __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
5425                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5426   %}
5427   ins_pipe(pipe_slow);
5428 %}
5429 
5430 instruct scatterL_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
5431   predicate(UseSVE > 0 &&
5432             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5433              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5434   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
5435   ins_cost(2 * SVE_COST);
5436   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated (D)" %}
5437   ins_encode %{
5438     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5439     __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
5440                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
5441   %}
5442   ins_pipe(pipe_slow);
5443 %}
5444 
5445 // ------------------------------ Vector Load Const -------------------------------
5446 
5447 instruct loadconB(vReg dst, immI0 src) %{
5448   predicate(UseSVE > 0 &&
5449             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
5450   match(Set dst (VectorLoadConst src));
5451   ins_cost(SVE_COST);
5452   format %{ "sve_index $dst, 0, 1\t# generate iota indices" %}
5453   ins_encode %{
5454     __ sve_index(as_FloatRegister($dst$$reg), __ B, 0, 1);
5455   %}
5456   ins_pipe(pipe_slow);
5457 %}
5458 
5459 // Intrisics for String.indexOf(char)
5460 
5461 
5462 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
5463                                   iRegI_R0 result, vReg ztmp1, vReg ztmp2,
5464                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
5465 %{
5466   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
5467   predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
5468   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
5469 
5470   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
5471 
5472   ins_encode %{
5473     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
5474                                as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
5475                                as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), true /* isL */);
5476   %}
5477   ins_pipe(pipe_class_memory);
5478 %}
5479 
5480 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
5481                                   iRegI_R0 result, vReg ztmp1, vReg ztmp2,
5482                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
5483 %{
5484   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
5485   predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
5486   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
5487 
5488   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
5489 
5490   ins_encode %{
5491     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
5492                                as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
5493                                as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), false /* isL */);
5494   %}
5495   ins_pipe(pipe_class_memory);
5496 %}
5497 
5498 // ---------------------------- Vector mask reductions ---------------------------
5499 instruct vmask_truecount(iRegINoSp dst, pReg src) %{
5500   predicate(UseSVE > 0);
5501   match(Set dst (VectorMaskTrueCount src));
5502   ins_cost(SVE_COST);
5503   format %{ "vmask_truecount $dst, $src\t# vector mask truecount (sve)" %}
5504   ins_encode %{
5505     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5506     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5507     __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($src$$reg));
5508   %}
5509   ins_pipe(pipe_slow);
5510 %}
5511 
5512 // Return the index of the first mask lane that is set, or vector length if none of
5513 // them are set.
5514 instruct vmask_firsttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
5515   predicate(UseSVE > 0);
5516   match(Set dst (VectorMaskFirstTrue src));
5517   effect(TEMP ptmp);
5518   ins_cost(3 * SVE_COST);
5519   format %{ "vmask_firsttrue $dst, $src\t# vector mask firsttrue (sve)" %}
5520   ins_encode %{
5521     uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
5522     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5523     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5524     // When the input predicate is all-false, the result should be the vector length
5525     // instead of max vector register size.
5526     if (length_in_bytes == MaxVectorSize) {
5527       __ sve_brkb(as_PRegister($ptmp$$reg), ptrue, as_PRegister($src$$reg), false);
5528     } else {
5529       __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
5530       __ sve_brkb(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg), as_PRegister($src$$reg), false);
5531     }
5532     __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($ptmp$$reg));
5533   %}
5534   ins_pipe(pipe_slow);
5535 %}
5536 
5537 instruct vmask_lasttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
5538   predicate(UseSVE > 0);
5539   match(Set dst (VectorMaskLastTrue src));
5540   effect(TEMP ptmp);
5541   ins_cost(3 * SVE_COST);
5542   format %{ "vmask_lasttrue $dst, $src\t# vector mask lasttrue (sve)" %}
5543   ins_encode %{
5544     BasicType bt = Matcher::vector_element_basic_type(this, $src);
5545     __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($src$$reg), as_PRegister($ptmp$$reg));
5546   %}
5547   ins_pipe(pipe_slow);
5548 %}
5549 
5550 instruct vmask_tolong(iRegLNoSp dst, pReg src, vReg vtmp1, vReg vtmp2) %{
5551   predicate(UseSVE > 0 &&
5552             n->in(1)->bottom_type()->is_vect()->length() <= 64);
5553   match(Set dst (VectorMaskToLong src));
5554   effect(TEMP vtmp1, TEMP vtmp2);
5555   ins_cost(13 * SVE_COST);
5556   format %{ "vmask_tolong $dst, $src\t# vector mask tolong (sve)" %}
5557   ins_encode %{
5558     __ sve_vmask_tolong(as_Register($dst$$reg), as_PRegister($src$$reg),
5559                         Matcher::vector_element_basic_type(this, $src),
5560                         Matcher::vector_length(this, $src),
5561                         as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg));
5562   %}
5563   ins_pipe(pipe_slow);
5564 %}
5565 
5566 // ---------------------------- Vector mask generation ---------------------------
5567 // The rules below set predicate registers. They can guarantee the high bits of dst
5568 // are cleared with zero when the vector length is less than the full size of
5569 // hardware vector register width.
5570 
5571 
5572 // maskAll (full or partial predicate size)
5573 
5574 instruct vmaskAll_immI(pRegGov dst, immI src) %{
5575   predicate(UseSVE > 0);
5576   match(Set dst (MaskAll src));
5577   ins_cost(SVE_COST);
5578   format %{ "sve_ptrue_lanecnt/sve_pfalse $dst\t# mask all (sve) (B/H/S)" %}
5579   ins_encode %{
5580     int con = (int)$src$$constant;
5581     if (con == 0) {
5582       __ sve_pfalse(as_PRegister($dst$$reg));
5583     } else {
5584       assert(con == -1, "invalid constant value for mask");
5585       BasicType bt = Matcher::vector_element_basic_type(this);
5586       __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
5587                            Matcher::vector_length(this));
5588     }
5589   %}
5590   ins_pipe(pipe_slow);
5591 %}
5592 
5593 instruct vmaskAllI(pRegGov dst, iRegIorL2I src, vReg tmp, rFlagsReg cr) %{
5594   predicate(UseSVE > 0);
5595   match(Set dst (MaskAll src));
5596   effect(TEMP tmp, KILL cr);
5597   ins_cost(3 * SVE_COST);
5598   format %{ "sve_dup $tmp, $src\n\t"
5599             "sve_ptrue $dst, vector_length\n\t"
5600             "sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) (B/H/S)" %}
5601   ins_encode %{
5602     BasicType bt = Matcher::vector_element_basic_type(this);
5603     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5604     uint length_in_bytes = Matcher::vector_length_in_bytes(this);
5605     __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
5606     if (length_in_bytes < MaxVectorSize) {
5607       __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), size, Matcher::vector_length(this));
5608       __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size,
5609                  as_PRegister($dst$$reg), as_FloatRegister($tmp$$reg), 0);
5610     } else {
5611       __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
5612     }
5613   %}
5614   ins_pipe(pipe_slow);
5615 %}
5616 
5617 instruct vmaskAll_immL(pRegGov dst, immL src) %{
5618   predicate(UseSVE > 0);
5619   match(Set dst (MaskAll src));
5620   ins_cost(SVE_COST);
5621   format %{ "sve_ptrue_lanecnt/sve_pfalse $dst\t# mask all (sve) (D)" %}
5622   ins_encode %{
5623     long con = (long)$src$$constant;
5624     if (con == 0) {
5625       __ sve_pfalse(as_PRegister($dst$$reg));
5626     } else {
5627       assert(con == -1, "invalid constant value for mask");
5628       BasicType bt = Matcher::vector_element_basic_type(this);
5629       __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
5630                            Matcher::vector_length(this));
5631     }
5632   %}
5633   ins_pipe(pipe_slow);
5634 %}
5635 
5636 instruct vmaskAllL(pRegGov dst, iRegL src, vReg tmp, rFlagsReg cr) %{
5637   predicate(UseSVE > 0);
5638   match(Set dst (MaskAll src));
5639   effect(TEMP tmp, KILL cr);
5640   ins_cost(3 * SVE_COST);
5641   format %{ "sve_dup $tmp, $src\n\t"
5642             "sve_ptrue $dst, vector_length\n\t"
5643             "sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) (D)" %}
5644   ins_encode %{
5645     BasicType bt = Matcher::vector_element_basic_type(this);
5646     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5647     uint length_in_bytes = Matcher::vector_length_in_bytes(this);
5648     __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
5649     if (length_in_bytes < MaxVectorSize) {
5650       __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), size, Matcher::vector_length(this));
5651       __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size,
5652                  as_PRegister($dst$$reg), as_FloatRegister($tmp$$reg), 0);
5653     } else {
5654       __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
5655     }
5656   %}
5657   ins_pipe(pipe_slow);
5658 %}
5659 
5660 // vector mask compare
5661 
5662 instruct vmaskcmp(pRegGov dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %{
5663   predicate(UseSVE > 0);
5664   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
5665   effect(KILL cr);
5666   ins_cost(2 * SVE_COST);
5667   format %{ "sve_cmp $dst, $src1, $src2\t# vector mask cmp (sve)" %}
5668   ins_encode %{
5669     uint length_in_bytes = Matcher::vector_length_in_bytes(this);
5670     BasicType bt = Matcher::vector_element_basic_type(this);
5671     if (length_in_bytes == MaxVectorSize) {
5672       __ sve_compare(as_PRegister($dst$$reg), bt, ptrue, as_FloatRegister($src1$$reg),
5673                      as_FloatRegister($src2$$reg), (int)$cond$$constant);
5674     } else {
5675       __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
5676                            Matcher::vector_length(this));
5677       __ sve_compare(as_PRegister($dst$$reg), bt, as_PRegister($dst$$reg), as_FloatRegister($src1$$reg),
5678                      as_FloatRegister($src2$$reg), (int)$cond$$constant);
5679     }
5680   %}
5681   ins_pipe(pipe_slow);
5682 %}
5683 
5684 instruct vmaskcmp_masked(pRegGov dst, vReg src1, vReg src2, immI cond, pRegGov pg, rFlagsReg cr) %{
5685   predicate(UseSVE > 0);
5686   match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond pg)));
5687   effect(KILL cr);
5688   ins_cost(SVE_COST);
5689   format %{ "sve_cmp $dst, $pg, $src1, $src2\t# vector mask cmp (sve)" %}
5690   ins_encode %{
5691     BasicType bt = Matcher::vector_element_basic_type(this);
5692     __ sve_compare(as_PRegister($dst$$reg), bt, as_PRegister($pg$$reg), as_FloatRegister($src1$$reg),
5693                    as_FloatRegister($src2$$reg), (int)$cond$$constant);
5694   %}
5695   ins_pipe(pipe_slow);
5696 %}
5697 
5698 // vector load mask
5699 
5700 instruct vloadmaskB(pRegGov dst, vReg src, rFlagsReg cr) %{
5701   predicate(UseSVE > 0 &&
5702             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
5703   match(Set dst (VectorLoadMask src));
5704   effect(KILL cr);
5705   ins_cost(SVE_COST);
5706   format %{ "vloadmaskB $dst, $src\t# vector load mask (sve) (B)" %}
5707   ins_encode %{
5708     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ B,
5709                ptrue, as_FloatRegister($src$$reg), 0);
5710   %}
5711   ins_pipe(pipe_slow);
5712 %}
5713 
5714 instruct vloadmask_extend(pRegGov dst, vReg src, vReg tmp, rFlagsReg cr) %{
5715   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() != T_BYTE);
5716   match(Set dst (VectorLoadMask src));
5717   effect(TEMP tmp, KILL cr);
5718   ins_cost(3 * SVE_COST);
5719   format %{ "vloadmask $dst, $src\t# vector load mask (sve) (H/S/D)" %}
5720   ins_encode %{
5721     BasicType bt = Matcher::vector_element_basic_type(this);
5722     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5723     __ sve_vector_extend(as_FloatRegister($tmp$$reg), size, as_FloatRegister($src$$reg), __ B);
5724     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
5725   %}
5726   ins_pipe(pipe_slow);
5727 %}
5728 
5729 // ---------------------------- Compress/Expand Operations ---------------------------
5730 
5731 instruct mcompress(pReg dst, pReg pg, rFlagsReg cr) %{
5732   predicate(UseSVE > 0);
5733   match(Set dst (CompressM pg));
5734   effect(KILL cr);
5735   ins_cost(2 * SVE_COST);
5736   format %{ "sve_cntp rscratch1, $pg\n\t"
5737             "sve_whilelo $dst, zr, rscratch1\t# mask compress (B/H/S/D)" %}
5738   ins_encode %{
5739     BasicType bt = Matcher::vector_element_basic_type(this);
5740     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5741     __ sve_cntp(rscratch1, size, ptrue, as_PRegister($pg$$reg));
5742     __ sve_whilelo(as_PRegister($dst$$reg), size, zr, rscratch1);
5743   %}
5744   ins_pipe(pipe_slow);
5745 %}
5746 
5747 instruct vcompress(vReg dst, vReg src, pRegGov pg) %{
5748   predicate(UseSVE > 0 &&
5749             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5750              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT ||
5751              n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5752              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5753   match(Set dst (CompressV src pg));
5754   ins_cost(SVE_COST);
5755   format %{ "sve_compact $dst, $src, $pg\t# vector compress (S/D)" %}
5756   ins_encode %{
5757     BasicType bt = Matcher::vector_element_basic_type(this);
5758     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5759     __ sve_compact(as_FloatRegister($dst$$reg), size, as_FloatRegister($src$$reg), as_PRegister($pg$$reg));
5760   %}
5761   ins_pipe(pipe_slow);
5762 %}
5763 
5764 instruct vcompressB(vReg dst, vReg src, pReg pg, vReg vtmp1, vReg vtmp2, vReg vtmp3, vReg vtmp4,
5765                     pReg ptmp, pRegGov pgtmp) %{
5766   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
5767   effect(TEMP_DEF dst, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP ptmp, TEMP pgtmp);
5768   match(Set dst (CompressV src pg));
5769   ins_cost(13 * SVE_COST);
5770   format %{ "sve_compact $dst, $src, $pg\t# vector compress (B)" %}
5771   ins_encode %{
5772     __ sve_compress_byte(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
5773                          as_FloatRegister($vtmp1$$reg),as_FloatRegister($vtmp2$$reg),
5774                          as_FloatRegister($vtmp3$$reg),as_FloatRegister($vtmp4$$reg),
5775                          as_PRegister($ptmp$$reg), as_PRegister($pgtmp$$reg));
5776   %}
5777   ins_pipe(pipe_slow);
5778 %}
5779 
5780 instruct vcompressS(vReg dst, vReg src, pReg pg, vReg vtmp1, vReg vtmp2, pRegGov pgtmp) %{
5781   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
5782   effect(TEMP_DEF dst, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp);
5783   match(Set dst (CompressV src pg));
5784   ins_cost(38 * SVE_COST);
5785   format %{ "sve_compact $dst, $src, $pg\t# vector compress (H)" %}
5786   ins_encode %{
5787     __ sve_compress_short(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
5788                           as_FloatRegister($vtmp1$$reg),as_FloatRegister($vtmp2$$reg), as_PRegister($pgtmp$$reg));
5789   %}
5790   ins_pipe(pipe_slow);
5791 %}
5792 
5793 instruct vexpand(vReg dst, vReg src, pRegGov pg) %{
5794   match(Set dst (ExpandV src pg));
5795   effect(TEMP_DEF dst);
5796   ins_cost(4 * SVE_COST);
5797   format %{ "sve_dup $dst, S/D, 0\n\t"
5798             "sve_histcnt $dst, S/D, $pg, $dst, $dst\n\t"
5799             "sve_sub $dst, S/D, 1\n\t"
5800             "sve_tbl $dst, S/D, $src, $dst\t# vector expand (S/D)" %}
5801   ins_encode %{
5802     // Example input:   src   = 1 2 3 4 5 6 7 8
5803     //                  pg    = 1 0 0 1 1 0 1 1
5804     // Expected result: dst   = 4 0 0 5 6 0 7 8
5805 
5806     // The basic idea is to use TBL which can shuffle the elements in the given
5807     // vector flexibly. HISTCNT + SUB is used to generate the second source input
5808     // for TBL whose value is used to select the indexed element from src vector.
5809 
5810     BasicType bt = Matcher::vector_element_basic_type(this);
5811     assert(UseSVE == 2 && !is_subword_type(bt), "unsupported");
5812     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5813     // dst = 0 0 0 0 0 0 0 0
5814     __ sve_dup(as_FloatRegister($dst$$reg), size, 0);
5815     // dst = 5 0 0 4 3 0 2 1
5816     __ sve_histcnt(as_FloatRegister($dst$$reg), size, as_PRegister($pg$$reg),
5817                    as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg));
5818     // dst = 4 -1 -1 3 2 -1 1 0
5819     __ sve_sub(as_FloatRegister($dst$$reg), size, 1);
5820     // dst = 4 0 0 5 6 0 7 8
5821     __ sve_tbl(as_FloatRegister($dst$$reg), size, as_FloatRegister($src$$reg),
5822                as_FloatRegister($dst$$reg));
5823   %}
5824   ins_pipe(pipe_slow);
5825 %}
5826 
5827 instruct vmask_gen(pRegGov pg, iRegL len, rFlagsReg cr) %{
5828   predicate(UseSVE > 0);
5829   match(Set pg (VectorMaskGen len));
5830   effect(KILL cr);
5831   ins_cost(SVE_COST);
5832   format %{ "sve_whilelo $pg, zr, $len\t # sve" %}
5833   ins_encode %{
5834     BasicType bt = Matcher::vector_element_basic_type(this);
5835     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5836     __ sve_whilelo(as_PRegister($pg$$reg), size, zr, as_Register($len$$reg));
5837   %}
5838   ins_pipe(pipe_slow);
5839 %}
5840 
5841 // ------------------------------ CountLeadingZerosV ------------------------------
5842 
5843 instruct vcountLeadingZeros(vReg dst, vReg src) %{
5844   predicate(UseSVE > 0 &&
5845             !n->as_Vector()->is_predicated_vector());
5846   match(Set dst (CountLeadingZerosV src));
5847   ins_cost(SVE_COST);
5848   format %{ "sve_clz $dst, $src\t# vector (sve)" %}
5849   ins_encode %{
5850     BasicType bt = Matcher::vector_element_basic_type(this);
5851     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5852     __ sve_clz(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($src$$reg));
5853   %}
5854   ins_pipe(pipe_slow);
5855 %}
5856 
5857 // The dst and src should use the same register to make sure the
5858 // inactive lanes in dst save the same elements as src.
5859 instruct vcountLeadingZeros_masked(vReg dst_src, pRegGov pg) %{
5860   predicate(UseSVE > 0);
5861   match(Set dst_src (CountLeadingZerosV dst_src pg));
5862   ins_cost(SVE_COST);
5863   format %{ "sve_clz $dst_src, $pg, $dst_src\t# vector (sve)" %}
5864   ins_encode %{
5865     BasicType bt = Matcher::vector_element_basic_type(this);
5866     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5867     __ sve_clz(as_FloatRegister($dst_src$$reg), size,
5868         as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
5869   %}
5870   ins_pipe(pipe_slow);
5871 %}
5872 
5873 // ------------------------------ CountTrailingZerosV -----------------------------
5874 
5875 instruct vcountTrailingZeros(vReg dst, vReg src) %{
5876   predicate(UseSVE > 0 &&
5877             !n->as_Vector()->is_predicated_vector());
5878   match(Set dst (CountTrailingZerosV src));
5879   ins_cost(2 * SVE_COST);
5880   format %{ "sve_rbit $dst, $src\n\t"
5881             "sve_clz  $dst, $dst\t# vector (sve)" %}
5882   ins_encode %{
5883     BasicType bt = Matcher::vector_element_basic_type(this);
5884     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5885     __ sve_rbit(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($src$$reg));
5886     __ sve_clz(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($dst$$reg));
5887   %}
5888   ins_pipe(pipe_slow);
5889 %}
5890 
5891 // The dst and src should use the same register to make sure the
5892 // inactive lanes in dst save the same elements as src.
5893 instruct vcountTrailingZeros_masked(vReg dst_src, pRegGov pg) %{
5894   predicate(UseSVE > 0);
5895   match(Set dst_src (CountTrailingZerosV dst_src pg));
5896   ins_cost(2 * SVE_COST);
5897   format %{ "sve_rbit $dst_src, $pg, $dst_src\n\t"
5898             "sve_clz  $dst_src, $pg, $dst_src\t# vector (sve)" %}
5899   ins_encode %{
5900     BasicType bt = Matcher::vector_element_basic_type(this);
5901     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5902     __ sve_rbit(as_FloatRegister($dst_src$$reg), size,
5903         as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
5904     __ sve_clz(as_FloatRegister($dst_src$$reg), size,
5905         as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
5906   %}
5907   ins_pipe(pipe_slow);
5908 %}
5909 
5910 // ---------------------------------- ReverseV ------------------------------------
5911 
5912 instruct vreverse(vReg dst, vReg src) %{
5913   predicate(UseSVE > 0 &&
5914             !n->as_Vector()->is_predicated_vector());
5915   match(Set dst (ReverseV src));
5916   ins_cost(SVE_COST);
5917   format %{ "sve_rbit $dst, $src\t# vector (sve)" %}
5918   ins_encode %{
5919     BasicType bt = Matcher::vector_element_basic_type(this);
5920     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5921     __ sve_rbit(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($src$$reg));
5922   %}
5923   ins_pipe(pipe_slow);
5924 %}
5925 
5926 // The dst and src should use the same register to make sure the
5927 // inactive lanes in dst save the same elements as src.
5928 instruct vreverse_masked(vReg dst_src, pRegGov pg) %{
5929   predicate(UseSVE > 0);
5930   match(Set dst_src (ReverseV dst_src pg));
5931   ins_cost(SVE_COST);
5932   format %{ "sve_rbit $dst_src, $pg, $dst_src\t# vector (sve)" %}
5933   ins_encode %{
5934     BasicType bt = Matcher::vector_element_basic_type(this);
5935     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5936     __ sve_rbit(as_FloatRegister($dst_src$$reg), size,
5937         as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
5938   %}
5939   ins_pipe(pipe_slow);
5940 %}
5941 
5942 // -------------------------------- ReverseBytesV ---------------------------------
5943 
5944 instruct vreverseBytes(vReg dst, vReg src) %{
5945   predicate(UseSVE > 0 &&
5946             !n->as_Vector()->is_predicated_vector());
5947   match(Set dst (ReverseBytesV src));
5948   ins_cost(SVE_COST);
5949   format %{ "sve_revb $dst, $src\t# vector (sve)" %}
5950   ins_encode %{
5951     BasicType bt = Matcher::vector_element_basic_type(this);
5952     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5953     if (bt == T_BYTE) {
5954       if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
5955         __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
5956       }
5957     } else {
5958       __ sve_revb(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($src$$reg));
5959     }
5960   %}
5961   ins_pipe(pipe_slow);
5962 %}
5963 
5964 // The dst and src should use the same register to make sure the
5965 // inactive lanes in dst save the same elements as src.
5966 instruct vreverseBytes_masked(vReg dst_src, pRegGov pg) %{
5967   predicate(UseSVE > 0);
5968   match(Set dst_src (ReverseBytesV dst_src pg));
5969   ins_cost(SVE_COST);
5970   format %{ "sve_revb $dst_src, $pg, $dst_src\t# vector (sve)" %}
5971   ins_encode %{
5972     BasicType bt = Matcher::vector_element_basic_type(this);
5973     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5974     if (bt == T_BYTE) {
5975       // do nothing
5976     } else {
5977       __ sve_revb(as_FloatRegister($dst_src$$reg), size,
5978           as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
5979     }
5980   %}
5981   ins_pipe(pipe_slow);
5982 %}
5983