1 //
   2 // Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
   3 // Copyright (c) 2020, 2022, Arm Limited. All rights reserved.
   4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5 //
   6 // This code is free software; you can redistribute it and/or modify it
   7 // under the terms of the GNU General Public License version 2 only, as
   8 // published by the Free Software Foundation.
   9 //
  10 // This code is distributed in the hope that it will be useful, but WITHOUT
  11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 // version 2 for more details (a copy is included in the LICENSE file that
  14 // accompanied this code).
  15 //
  16 // You should have received a copy of the GNU General Public License version
  17 // 2 along with this work; if not, write to the Free Software Foundation,
  18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21 // or visit www.oracle.com if you need additional information or have any
  22 // questions.
  23 //
  24 //
  25 
  26 dnl Generate the warning
  27 // This file is automatically generated by running "m4 aarch64_sve_ad.m4". Do not edit ----
  28 dnl
  29 
  30 // AArch64 SVE Architecture Description File
  31 
  32 dnl
  33 define(`TYPE2DATATYPE',
  34 `ifelse($1, `B', `BYTE',
  35         $1, `S', `SHORT',
  36         $1, `I', `INT',
  37         $1, `L', `LONG',
  38         $1, `F', `FLOAT',
  39         $1, `D', `DOUBLE',
  40         `error($1)')')dnl
  41 dnl
  42 dnl OPERAND_VMEMORYA_IMMEDIATE_OFFSET($1,            $2,       $3       $4   )
  43 dnl OPERAND_VMEMORYA_IMMEDIATE_OFFSET(imm_type_abbr, imm_type, imm_len, scale)
  44 define(`OPERAND_VMEMORYA_IMMEDIATE_OFFSET', `
  45 operand vmemA_imm$1Offset$3()
  46 %{
  47   // (esize / msize) = $4
  48   predicate(Address::offset_ok_for_sve_immed(n->get_$2(), $3,
  49             Matcher::scalable_vector_reg_size(T_BYTE)ifelse($4, `1', `', ` / $4')));
  50   match(Con$1);
  51 
  52   op_cost(0);
  53   format %{ %}
  54   interface(CONST_INTER);
  55 %}')dnl
  56 
  57 // 4 bit signed offset -- for predicated load/store
  58 OPERAND_VMEMORYA_IMMEDIATE_OFFSET(I, int,  4, 1)
  59 OPERAND_VMEMORYA_IMMEDIATE_OFFSET(L, long, 4, 1)
  60 dnl
  61 dnl OPERAND_VMEMORYA_INDIRECT_OFFSET($1,            $2     )
  62 dnl OPERAND_VMEMORYA_INDIRECT_OFFSET(imm_type_abbr, imm_len)
  63 define(`OPERAND_VMEMORYA_INDIRECT_OFFSET', `
  64 operand vmemA_indOff$1$2$3(iRegP reg, vmemA_imm$1Offset$2 off)
  65 %{
  66   constraint(ALLOC_IN_RC(ptr_reg));
  67   match(AddP reg off);
  68   op_cost(0);
  69   format %{ "[$reg, $off]" %}
  70   interface(MEMORY_INTER) %{
  71     base($reg);
  72     `index'(0xffffffff);
  73     scale(0x0);
  74     disp($off);
  75   %}
  76 %}')dnl
  77 OPERAND_VMEMORYA_INDIRECT_OFFSET(I, 4)
  78 OPERAND_VMEMORYA_INDIRECT_OFFSET(L, 4)
  79 
  80 // The indOff of vmemA is valid only when the vector element (load to/store from)
  81 // size equals to memory element (load from/store to) size.
  82 opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);
  83 
  84 source_hpp %{
  85   bool op_sve_supported(int opcode, int vlen, BasicType bt);
  86   bool masked_op_sve_supported(int opcode, int vlen, BasicType bt);
  87 %}
  88 
  89 source %{
  90 
  91   typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
  92                                                              PRegister Pg, const Address &adr);
  93 
  94   // Predicated load/store, with optional ptrue to all elements of given predicate register.
  95   static void loadStoreA_predicated(C2_MacroAssembler masm, bool is_store, FloatRegister reg,
  96                                     PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
  97                                     int opcode, Register base, int index, int size, int disp) {
  98     sve_mem_insn_predicate insn;
  99     int mesize = type2aelembytes(mem_elem_bt);
 100     if (index == -1) {
 101       assert(size == 0, "unsupported address mode: scale size = %d", size);
 102       switch(mesize) {
 103       case 1:
 104         insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
 105         break;
 106       case 2:
 107         insn = is_store ? &C2_MacroAssembler::sve_st1h : &C2_MacroAssembler::sve_ld1h;
 108         break;
 109       case 4:
 110         insn = is_store ? &C2_MacroAssembler::sve_st1w : &C2_MacroAssembler::sve_ld1w;
 111         break;
 112       case 8:
 113         insn = is_store ? &C2_MacroAssembler::sve_st1d : &C2_MacroAssembler::sve_ld1d;
 114         break;
 115       default:
 116         assert(false, "unsupported");
 117         ShouldNotReachHere();
 118       }
 119       int imm4 = disp / mesize / Matcher::scalable_vector_reg_size(vector_elem_bt);
 120       (masm.*insn)(reg, Assembler::elemType_to_regVariant(vector_elem_bt), pg, Address(base, imm4));
 121     } else {
 122       assert(false, "unimplemented");
 123       ShouldNotReachHere();
 124     }
 125   }
 126 
 127   bool op_sve_supported(int opcode, int vlen, BasicType bt) {
 128     int length_in_bytes = vlen * type2aelembytes(bt);
 129     switch (opcode) {
 130       case Op_MulAddVS2VI:
 131       // No multiply reduction instructions
 132       case Op_MulReductionVD:
 133       case Op_MulReductionVF:
 134       case Op_MulReductionVI:
 135       case Op_MulReductionVL:
 136       // Others
 137       case Op_ExtractC:
 138       case Op_ExtractUB:
 139         return false;
 140       // Vector API specific
 141       case Op_VectorLoadShuffle:
 142       case Op_VectorRearrange:
 143         return vlen >= 4 && length_in_bytes <= MaxVectorSize;
 144       case Op_LoadVector:
 145       case Op_StoreVector:
 146         return Matcher::vector_size_supported(bt, vlen);
 147       default:
 148         break;
 149     }
 150     // By default, we only support vector operations with no less than 8 bytes and 2 elements.
 151     return 8 <= length_in_bytes && length_in_bytes <= MaxVectorSize && vlen >= 2;
 152   }
 153 
 154   bool masked_op_sve_supported(int opcode, int vlen, BasicType bt) {
 155     if (opcode == Op_VectorRearrange) {
 156       return false;
 157     }
 158     return op_sve_supported(opcode, vlen, bt);
 159   }
 160 %}
 161 
 162 definitions %{
 163   int_def SVE_COST             (200, 200);
 164 %}
 165 
 166 dnl
 167 dnl ELEMENT_SHORT_CHART($1, $2)
 168 dnl ELEMENT_SHORT_CHART(etype, node)
 169 define(`ELEMENT_SHORT_CHAR',`ifelse(`$1', `T_SHORT',
 170   `($2->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
 171             ($2->bottom_type()->is_vect()->element_basic_type() == T_CHAR))',
 172    `($2->bottom_type()->is_vect()->element_basic_type() == $1)')')dnl
 173 dnl
 174 
 175 // All SVE instructions
 176 
 177 // vector load/store
 178 
 179 // Unpredicated vector load/store
 180 instruct loadV(vReg dst, vmemA mem) %{
 181   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() >= 16 &&
 182             n->as_LoadVector()->memory_size() == MaxVectorSize);
 183   match(Set dst (LoadVector mem));
 184   ins_cost(4 * SVE_COST);
 185   format %{ "sve_ldr $dst, $mem\t# vector (sve)" %}
 186   ins_encode %{
 187     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 188     BasicType bt = Matcher::vector_element_basic_type(this);
 189     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg, ptrue,
 190                           bt, bt, $mem->opcode(),
 191                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 192   %}
 193   ins_pipe(pipe_slow);
 194 %}
 195 
 196 instruct storeV(vReg src, vmemA mem) %{
 197   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() >= 16 &&
 198             n->as_StoreVector()->memory_size() == MaxVectorSize);
 199   match(Set mem (StoreVector mem src));
 200   ins_cost(4 * SVE_COST);
 201   format %{ "sve_str $mem, $src\t# vector (sve)" %}
 202   ins_encode %{
 203     FloatRegister src_reg = as_FloatRegister($src$$reg);
 204     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 205     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg, ptrue,
 206                           bt, bt, $mem->opcode(),
 207                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 208   %}
 209   ins_pipe(pipe_slow);
 210 %}dnl
 211 
 212 dnl
 213 define(`VLoadStore', `
 214 // ifelse(load, $3, Load, Store) Vector ($6 bits)
 215 instruct $3V$4_vreg`'(vReg $7, vmem$4 mem)
 216 %{
 217   predicate(UseSVE > 0 && `n->as_'ifelse(load, $3, Load, Store)Vector()->memory_size() == $4);
 218   match(Set ifelse(load, $3, dst (LoadVector mem), mem (StoreVector mem src)));
 219   ins_cost(4 * INSN_COST);
 220   format %{ "$1   ifelse(load, $3, `$dst,$mem', `$mem,$src')\t# vector ($6 bits)" %}
 221   ins_encode( `aarch64_enc_'ifelse(load, $3, ldr, str)v$2($7, mem) );
 222   ins_pipe(v$3`_reg_mem'ifelse(eval($4 * 8), 128, 128, 64));
 223 %}')dnl
 224 dnl        $1    $2 $3     $4  $5 $6   $7
 225 VLoadStore(ldrh, H, load,  2,  D, 16,  dst)
 226 VLoadStore(strh, H, store, 2,  D, 16,  src)
 227 VLoadStore(ldrs, S, load,  4,  D, 32,  dst)
 228 VLoadStore(strs, S, store, 4,  D, 32,  src)
 229 VLoadStore(ldrd, D, load,  8,  D, 64,  dst)
 230 VLoadStore(strd, D, store, 8,  D, 64,  src)
 231 VLoadStore(ldrq, Q, load, 16,  X, 128, dst)
 232 VLoadStore(strq, Q, store, 16, X, 128, src)
 233 
 234 // Predicated vector load/store, based on the vector length of the node.
 235 // Only load/store values in the range of the memory_size. This is needed
 236 // when the memory_size is lower than the hardware supported max vector size.
 237 // And this might happen for Vector API mask vector load/store.
 238 instruct loadV_partial(vReg dst, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
 239   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() > 16 &&
 240             n->as_LoadVector()->memory_size() < MaxVectorSize);
 241   match(Set dst (LoadVector mem));
 242   effect(TEMP pgtmp, KILL cr);
 243   ins_cost(6 * SVE_COST);
 244   format %{ "sve_ptrue $pgtmp, vector_length\n\t"
 245             "sve_ldr $dst, $pgtmp, $mem\t# load vector partial" %}
 246   ins_encode %{
 247     BasicType bt = Matcher::vector_element_basic_type(this);
 248     __ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 249                          Matcher::vector_length(this));
 250     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 251     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg,
 252                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 253                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 254   %}
 255   ins_pipe(pipe_slow);
 256 %}
 257 
 258 instruct storeV_partial(vReg src, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
 259   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() > 16 &&
 260             n->as_StoreVector()->memory_size() < MaxVectorSize);
 261   match(Set mem (StoreVector mem src));
 262   effect(TEMP pgtmp, KILL cr);
 263   ins_cost(5 * SVE_COST);
 264   format %{ "sve_ptrue $pgtmp, vector_length\n\t"
 265             "sve_str $src, $pgtmp, $mem\t# store vector partial" %}
 266   ins_encode %{
 267     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 268     __ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 269                          Matcher::vector_length(this, $src));
 270     FloatRegister src_reg = as_FloatRegister($src$$reg);
 271     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg,
 272                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 273                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 274   %}
 275   ins_pipe(pipe_slow);
 276 %}
 277 
 278 // vector load/store - predicated
 279 
 280 instruct loadV_masked(vReg dst, vmemA mem, pRegGov pg) %{
 281   predicate(UseSVE > 0);
 282   match(Set dst (LoadVectorMasked mem pg));
 283   ins_cost(4 * SVE_COST);
 284   format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated (sve)" %}
 285   ins_encode %{
 286     BasicType bt = Matcher::vector_element_basic_type(this);
 287     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
 288                           as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
 289                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 290   %}
 291   ins_pipe(pipe_slow);
 292 %}
 293 
 294 instruct storeV_masked(vReg src, vmemA mem, pRegGov pg) %{
 295   predicate(UseSVE > 0);
 296   match(Set mem (StoreVectorMasked mem (Binary src pg)));
 297   ins_cost(4 * SVE_COST);
 298   format %{ "sve_str $mem, $pg, $src\t# store vector predicated (sve)" %}
 299   ins_encode %{
 300     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 301     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
 302                           as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
 303                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 304   %}
 305   ins_pipe(pipe_slow);
 306 %}
 307 
 308 dnl
 309 dnl MASK_LOGICAL_OP($1,        $2,      $3  )
 310 dnl MASK_LOGICAL_OP(insn_name, op_name, insn)
 311 define(`MASK_LOGICAL_OP', `
 312 instruct vmask_$1(pRegGov pd, pRegGov pn, pRegGov pm) %{
 313   predicate(UseSVE > 0);
 314   match(Set pd ($2 pn pm));
 315   ins_cost(SVE_COST);
 316   format %{ "$3 $pd, $pn, $pm\t# predicate (sve)" %}
 317   ins_encode %{
 318     __ $3(as_PRegister($pd$$reg), ptrue,
 319                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 320   %}
 321   ins_pipe(pipe_slow);
 322 %}')dnl
 323 dnl
 324 // mask logical and/or/xor
 325 MASK_LOGICAL_OP(and, AndVMask, sve_and)
 326 MASK_LOGICAL_OP(or, OrVMask, sve_orr)
 327 MASK_LOGICAL_OP(xor, XorVMask, sve_eor)
 328 
 329 dnl
 330 dnl MASK_LOGICAL_AND_NOT($1,   $2  )
 331 dnl MASK_LOGICAL_AND_NOT(type, size)
 332 define(`MASK_LOGICAL_AND_NOT', `
 333 instruct vmask_and_not$1(pRegGov pd, pRegGov pn, pRegGov pm, imm$1_M1 m1) %{
 334   predicate(UseSVE > 0);
 335   match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
 336   ins_cost(SVE_COST);
 337   format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) ($2)" %}
 338   ins_encode %{
 339     __ sve_bic(as_PRegister($pd$$reg), ptrue,
 340                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 341   %}
 342   ins_pipe(pipe_slow);
 343 %}')dnl
 344 dnl
 345 // mask logical and_not
 346 MASK_LOGICAL_AND_NOT(I, B/H/S)
 347 MASK_LOGICAL_AND_NOT(L, D)
 348 
 349 // vector reinterpret
 350 
 351 instruct reinterpret(vReg dst) %{
 352   predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() ==
 353                           n->in(1)->bottom_type()->is_vect()->length_in_bytes());  // src == dst
 354   match(Set dst (VectorReinterpret dst));
 355   ins_cost(0);
 356   format %{ "# reinterpret $dst\t# do nothing" %}
 357   ins_encode %{
 358     // empty
 359   %}
 360   ins_pipe(pipe_class_empty);
 361 %}
 362 
 363 instruct reinterpretResize(vReg dst, vReg src, pRegGov pgtmp, rFlagsReg cr) %{
 364   predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() !=
 365                           n->in(1)->bottom_type()->is_vect()->length_in_bytes());  // src != dst
 366   match(Set dst (VectorReinterpret src));
 367   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
 368   ins_cost(3 * SVE_COST);
 369   format %{ "reinterpretResize $dst, $src\t# vector (sve)" %}
 370   ins_encode %{
 371     uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
 372     uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
 373     uint length_in_bytes_resize = length_in_bytes_src < length_in_bytes_dst ?
 374                                   length_in_bytes_src : length_in_bytes_dst;
 375     assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
 376            "invalid vector length");
 377     __ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ B, length_in_bytes_resize);
 378     __ sve_dup(as_FloatRegister($dst$$reg), __ B, 0);
 379     __ sve_sel(as_FloatRegister($dst$$reg), __ B, as_PRegister($pgtmp$$reg),
 380                as_FloatRegister($src$$reg), as_FloatRegister($dst$$reg));
 381   %}
 382   ins_pipe(pipe_slow);
 383 %}
 384 
 385 // vector mask reinterpret
 386 
 387 instruct vmask_reinterpret_same_esize(pRegGov dst_src) %{
 388   predicate(UseSVE > 0 &&
 389             n->as_Vector()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
 390             n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
 391   match(Set dst_src (VectorReinterpret dst_src));
 392   ins_cost(0);
 393   format %{ "# vmask_reinterpret $dst_src\t# do nothing" %}
 394   ins_encode %{
 395     // empty
 396   %}
 397   ins_pipe(pipe_class_empty);
 398 %}
 399 
 400 instruct vmask_reinterpret_diff_esize(pRegGov dst, pRegGov src, vReg tmp, rFlagsReg cr) %{
 401   predicate(UseSVE > 0 &&
 402             n->as_Vector()->length() != n->in(1)->bottom_type()->is_vect()->length() &&
 403             n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
 404   match(Set dst (VectorReinterpret src));
 405   effect(TEMP tmp, KILL cr);
 406   ins_cost(2 * SVE_COST);
 407   format %{ "# vmask_reinterpret $dst, $src\t# vector (sve)" %}
 408   ins_encode %{
 409     BasicType from_bt = Matcher::vector_element_basic_type(this, $src);
 410     Assembler::SIMD_RegVariant from_size = __ elemType_to_regVariant(from_bt);
 411     BasicType to_bt = Matcher::vector_element_basic_type(this);
 412     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
 413     __ sve_cpy(as_FloatRegister($tmp$$reg), from_size, as_PRegister($src$$reg), -1, false);
 414     __ sve_cmp(Assembler::EQ, as_PRegister($dst$$reg), to_size, ptrue, as_FloatRegister($tmp$$reg), -1);
 415   %}
 416   ins_pipe(pipe_slow);
 417 %}
 418 dnl
 419 dnl UNARY_OP_TRUE_PREDICATE($1,        $2,      $3,   $4  )
 420 dnl UNARY_OP_TRUE_PREDICATE(insn_name, op_name, size, insn)
 421 define(`UNARY_OP_TRUE_PREDICATE', `
 422 instruct $1(vReg dst, vReg src) %{
 423   predicate(UseSVE > 0 &&
 424             !n->as_Vector()->is_predicated_vector());
 425   match(Set dst ($2 src));
 426   ins_cost(SVE_COST);
 427   format %{ "$4 $dst, $src\t# vector (sve) ($3)" %}
 428   ins_encode %{dnl
 429 ifelse($1, `vnegI', `
 430     BasicType bt = Matcher::vector_element_basic_type(this);', `')
 431     __ $4(as_FloatRegister($dst$$reg), ifelse($1, `vnegI', `__ elemType_to_regVariant(bt)', `__ $3'),
 432          ptrue, as_FloatRegister($src$$reg));
 433   %}
 434   ins_pipe(pipe_slow);
 435 %}')dnl
 436 dnl
 437 
 438 // vector abs
 439 UNARY_OP_TRUE_PREDICATE(vabsB, AbsVB, B, sve_abs)
 440 UNARY_OP_TRUE_PREDICATE(vabsS, AbsVS, H, sve_abs)
 441 UNARY_OP_TRUE_PREDICATE(vabsI, AbsVI, S, sve_abs)
 442 UNARY_OP_TRUE_PREDICATE(vabsL, AbsVL, D, sve_abs)
 443 UNARY_OP_TRUE_PREDICATE(vabsF, AbsVF, S, sve_fabs)
 444 UNARY_OP_TRUE_PREDICATE(vabsD, AbsVD, D, sve_fabs)
 445 
 446 dnl UNARY_OP_PREDICATE($1,        $2,      $3,   $4  )
 447 dnl UNARY_OP_PREDICATE(insn_name, op_name, size, insn)
 448 define(`UNARY_OP_PREDICATE', `
 449 instruct $1_masked(vReg dst_src, pRegGov pg) %{
 450   predicate(UseSVE > 0);
 451   match(Set dst_src ($2 dst_src pg));
 452   ins_cost(SVE_COST);
 453   format %{ "$4 $dst_src, $pg, $dst_src\t# vector (sve) ($3)" %}
 454   ins_encode %{dnl
 455 ifelse($1, `vnegI', `
 456     BasicType bt = Matcher::vector_element_basic_type(this);', `')
 457     __ $4(as_FloatRegister($dst_src$$reg), ifelse($1, `vnegI', `__ elemType_to_regVariant(bt)', `__ $3'),
 458             as_PRegister($pg$$reg),
 459             as_FloatRegister($dst_src$$reg));
 460   %}
 461   ins_pipe(pipe_slow);
 462 %}')dnl
 463 // vector abs - predicated
 464 UNARY_OP_PREDICATE(vabsB, AbsVB, B, sve_abs)
 465 UNARY_OP_PREDICATE(vabsS, AbsVS, H, sve_abs)
 466 UNARY_OP_PREDICATE(vabsI, AbsVI, S, sve_abs)
 467 UNARY_OP_PREDICATE(vabsL, AbsVL, D, sve_abs)
 468 UNARY_OP_PREDICATE(vabsF, AbsVF, S, sve_fabs)
 469 UNARY_OP_PREDICATE(vabsD, AbsVD, D, sve_fabs)
 470 
 471 dnl
 472 dnl BINARY_OP_UNPREDICATE($1,        $2       $3,   $4           $5  )
 473 dnl BINARY_OP_UNPREDICATE(insn_name, op_name, size, min_vec_len, insn)
 474 define(`BINARY_OP_UNPREDICATE', `
 475 instruct $1(vReg dst, vReg src1, vReg src2) %{
 476   predicate(UseSVE > 0);
 477   match(Set dst ($2 src1 src2));
 478   ins_cost(SVE_COST);
 479   format %{ "$5 $dst, $src1, $src2\t # vector (sve) ($3)" %}
 480   ins_encode %{
 481     __ $5(as_FloatRegister($dst$$reg), __ $3,
 482          as_FloatRegister($src1$$reg),
 483          as_FloatRegister($src2$$reg));
 484   %}
 485   ins_pipe(pipe_slow);
 486 %}')dnl
 487 dnl
 488 dnl
 489 dnl BINARY_OP_PREDICATE($1,        $2,      $3,   $4  )
 490 dnl BINARY_OP_PREDICATE(insn_name, op_name, size, insn)
 491 define(`BINARY_OP_PREDICATE', `
 492 instruct $1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 493   predicate(UseSVE > 0);
 494   match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
 495   ins_cost(SVE_COST);
 496   format %{ "$4 $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) ($3)" %}
 497   ins_encode %{
 498     __ $4(as_FloatRegister($dst_src1$$reg), __ $3,
 499             as_PRegister($pg$$reg),
 500             as_FloatRegister($src2$$reg));
 501   %}
 502   ins_pipe(pipe_slow);
 503 %}')dnl
 504 dnl
 505 // vector add
 506 BINARY_OP_UNPREDICATE(vaddB, AddVB, B, 16, sve_add)
 507 BINARY_OP_UNPREDICATE(vaddS, AddVS, H, 8,  sve_add)
 508 BINARY_OP_UNPREDICATE(vaddI, AddVI, S, 4,  sve_add)
 509 BINARY_OP_UNPREDICATE(vaddL, AddVL, D, 2,  sve_add)
 510 BINARY_OP_UNPREDICATE(vaddF, AddVF, S, 4,  sve_fadd)
 511 BINARY_OP_UNPREDICATE(vaddD, AddVD, D, 2,  sve_fadd)
 512 
 513 // vector add - predicated
 514 BINARY_OP_PREDICATE(vaddB, AddVB, B, sve_add)
 515 BINARY_OP_PREDICATE(vaddS, AddVS, H, sve_add)
 516 BINARY_OP_PREDICATE(vaddI, AddVI, S, sve_add)
 517 BINARY_OP_PREDICATE(vaddL, AddVL, D, sve_add)
 518 BINARY_OP_PREDICATE(vaddF, AddVF, S, sve_fadd)
 519 BINARY_OP_PREDICATE(vaddD, AddVD, D, sve_fadd)
 520 dnl
 521 dnl ADD_IMM($1,          $2,   $3      )
 522 dnl ADD_IMM(name_suffix, size, imm_type)
 523 define(`ADD_IMM', `
 524 instruct vaddImm$1(vReg dst_src, $3 con) %{
 525   predicate(UseSVE > 0);
 526   match(Set dst_src (AddV$1 dst_src (Replicate$1 con)));
 527   ins_cost(SVE_COST);
 528   format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) ($2)" %}
 529   ins_encode %{
 530     int32_t val = $con$$constant;
 531     if (val > 0){
 532       __ sve_add(as_FloatRegister($dst_src$$reg), __ $2, val);
 533     } else if (val < 0){
 534       __ sve_sub(as_FloatRegister($dst_src$$reg), __ $2, -val);
 535     }
 536   %}
 537   ins_pipe(pipe_slow);
 538 %}')dnl
 539 
 540 // vector add reg imm (unpredicated)
 541 ADD_IMM(B, B, immBAddSubV)
 542 ADD_IMM(S, H, immIAddSubV)
 543 ADD_IMM(I, S, immIAddSubV)
 544 ADD_IMM(L, D, immLAddSubV)
 545 dnl
 546 dnl BITWISE_OP_IMM($1,        $2        $3,   $4    $5      )
 547 dnl BITWISE_OP_IMM(insn_name, op_name1, size, type, op_name2)
 548 define(`BITWISE_OP_IMM', `
 549 instruct $1(vReg dst_src, imm$4Log con) %{
 550   predicate(UseSVE > 0);
 551   match(Set dst_src ($2 dst_src (Replicate$4 con)));
 552   ins_cost(SVE_COST);
 553   format %{ "$5 $dst_src, $dst_src, $con\t # vector (sve) ($3)" %}
 554   ins_encode %{
 555     __ $5(as_FloatRegister($dst_src$$reg), __ $3,
 556          (uint64_t)($con$$constant));
 557   %}
 558   ins_pipe(pipe_slow);
 559 %}')dnl
 560 
 561 // vector binary op reg imm (unpredicated)
 562 BITWISE_OP_IMM(vandB, AndV, B, B, sve_and)
 563 BITWISE_OP_IMM(vandH, AndV, H, S, sve_and)
 564 BITWISE_OP_IMM(vandS, AndV, S, I, sve_and)
 565 BITWISE_OP_IMM(vandD, AndV, D, L, sve_and)
 566 BITWISE_OP_IMM(vorB,  OrV,  B, B, sve_orr)
 567 BITWISE_OP_IMM(vorH,  OrV,  H, S, sve_orr)
 568 BITWISE_OP_IMM(vorS,  OrV,  S, I, sve_orr)
 569 BITWISE_OP_IMM(vorD,  OrV,  D, L, sve_orr)
 570 BITWISE_OP_IMM(vxorB, XorV, B, B, sve_eor)
 571 BITWISE_OP_IMM(vxorH, XorV, H, S, sve_eor)
 572 BITWISE_OP_IMM(vxorS, XorV, S, I, sve_eor)
 573 BITWISE_OP_IMM(vxorD, XorV, D, L, sve_eor)
 574 dnl
 575 dnl
 576 dnl BINARY_OP_UNSIZED($1,        $2,      $3  )
 577 dnl BINARY_OP_UNSIZED(insn_name, op_name, insn)
 578 define(`BINARY_OP_UNSIZED', `
 579 instruct $1(vReg dst, vReg src1, vReg src2) %{
 580   predicate(UseSVE > 0);
 581   match(Set dst ($2 src1 src2));
 582   ins_cost(SVE_COST);
 583   format %{ "$3  $dst, $src1, $src2\t# vector (sve)" %}
 584   ins_encode %{
 585     __ $3(as_FloatRegister($dst$$reg),
 586          as_FloatRegister($src1$$reg),
 587          as_FloatRegister($src2$$reg));
 588   %}
 589   ins_pipe(pipe_slow);
 590 %}')dnl
 591 dnl
 592 // vector and
 593 BINARY_OP_UNSIZED(vand, AndV, sve_and)
 594 
 595 // vector or
 596 BINARY_OP_UNSIZED(vor, OrV, sve_orr)
 597 
 598 // vector xor
 599 BINARY_OP_UNSIZED(vxor, XorV, sve_eor)
 600 
 601 dnl BINARY_LOGIC_OP_PREDICATE($1,        $2,      $3  )
 602 dnl BINARY_LOGIC_OP_PREDICATE(insn_name, op_name, insn)
 603 define(`BINARY_LOGIC_OP_PREDICATE', `
 604 instruct $1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 605   predicate(UseSVE > 0);
 606   match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
 607   ins_cost(SVE_COST);
 608   format %{ "$3 $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
 609   ins_encode %{
 610     BasicType bt = Matcher::vector_element_basic_type(this);
 611     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
 612     __ $3(as_FloatRegister($dst_src1$$reg), size,
 613           as_PRegister($pg$$reg),
 614           as_FloatRegister($src2$$reg));
 615   %}
 616   ins_pipe(pipe_slow);
 617 %}')dnl
 618 dnl
 619 // vector and - predicated
 620 BINARY_LOGIC_OP_PREDICATE(vand, AndV, sve_and)
 621 
 622 // vector or - predicated
 623 BINARY_LOGIC_OP_PREDICATE(vor, OrV, sve_orr)
 624 
 625 // vector xor - predicated
 626 BINARY_LOGIC_OP_PREDICATE(vxor, XorV, sve_eor)
 627 
 628 // vector not
 629 dnl
 630 define(`MATCH_RULE', `ifelse($1, I,
 631 `match(Set dst (XorV src (ReplicateB m1)));
 632   match(Set dst (XorV src (ReplicateS m1)));
 633   match(Set dst (XorV src (ReplicateI m1)));',
 634 `match(Set dst (XorV src (ReplicateL m1)));')')dnl
 635 dnl
 636 define(`VECTOR_NOT', `
 637 instruct vnot$1`'(vReg dst, vReg src, imm$1_M1 m1) %{
 638   predicate(UseSVE > 0);
 639   MATCH_RULE($1)
 640   ins_cost(SVE_COST);
 641   format %{ "sve_not $dst, $src\t# vector (sve) $2" %}
 642   ins_encode %{
 643     __ sve_not(as_FloatRegister($dst$$reg), __ D,
 644                ptrue, as_FloatRegister($src$$reg));
 645   %}
 646   ins_pipe(pipe_slow);
 647 %}')dnl
 648 dnl        $1,$2
 649 VECTOR_NOT(I, B/H/S)
 650 VECTOR_NOT(L, D)
 651 undefine(MATCH_RULE)
 652 dnl
 653 // vector not - predicated
 654 dnl
 655 define(`MATCH_RULE', `ifelse($1, I,
 656 `match(Set dst (XorV (Binary src (ReplicateB m1)) pg));
 657   match(Set dst (XorV (Binary src (ReplicateS m1)) pg));
 658   match(Set dst (XorV (Binary src (ReplicateI m1)) pg));',
 659 `match(Set dst (XorV (Binary src (ReplicateL m1)) pg));')')dnl
 660 dnl
 661 define(`VECTOR_NOT_PREDICATE', `
 662 instruct vnot$1_masked`'(vReg dst, vReg src, imm$1_M1 m1, pRegGov pg) %{
 663   predicate(UseSVE > 0);
 664   MATCH_RULE($1)
 665   ins_cost(SVE_COST);
 666   format %{ "sve_not $dst, $pg, $src\t# vector (sve) $2" %}
 667   ins_encode %{
 668     BasicType bt = Matcher::vector_element_basic_type(this);
 669     __ sve_not(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
 670                as_PRegister($pg$$reg), as_FloatRegister($src$$reg));
 671   %}
 672   ins_pipe(pipe_slow);
 673 %}')dnl
 674 dnl                 $1, $2
 675 VECTOR_NOT_PREDICATE(I, B/H/S)
 676 VECTOR_NOT_PREDICATE(L, D)
 677 undefine(MATCH_RULE)
 678 dnl
 679 // vector and_not
 680 dnl
 681 define(`MATCH_RULE', `ifelse($1, I,
 682 `match(Set dst (AndV src1 (XorV src2 (ReplicateB m1))));
 683   match(Set dst (AndV src1 (XorV src2 (ReplicateS m1))));
 684   match(Set dst (AndV src1 (XorV src2 (ReplicateI m1))));',
 685 `match(Set dst (AndV src1 (XorV src2 (ReplicateL m1))));')')dnl
 686 dnl
 687 define(`VECTOR_AND_NOT', `
 688 instruct vand_not$1`'(vReg dst, vReg src1, vReg src2, imm$1_M1 m1) %{
 689   predicate(UseSVE > 0);
 690   MATCH_RULE($1)
 691   ins_cost(SVE_COST);
 692   format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) $2" %}
 693   ins_encode %{
 694     __ sve_bic(as_FloatRegister($dst$$reg),
 695                as_FloatRegister($src1$$reg),
 696                as_FloatRegister($src2$$reg));
 697   %}
 698   ins_pipe(pipe_slow);
 699 %}')dnl
 700 dnl            $1,$2
 701 VECTOR_AND_NOT(I, B/H/S)
 702 VECTOR_AND_NOT(L, D)
 703 undefine(MATCH_RULE)
 704 dnl
 705 // vector and_not - predicated
 706 dnl
 707 define(`MATCH_RULE', `ifelse($1, I,
 708 `match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateB m1))) pg));
 709   match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateS m1))) pg));
 710   match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateI m1))) pg));',
 711 `match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateL m1))) pg));')')dnl
 712 dnl
 713 define(`VECTOR_AND_NOT_PREDICATE', `
 714 instruct vand_not$1_masked`'(vReg dst_src1, vReg src2, imm$1_M1 m1, pRegGov pg) %{
 715   predicate(UseSVE > 0);
 716   MATCH_RULE($1)
 717   ins_cost(SVE_COST);
 718   format %{ "sve_bic $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) $2" %}
 719   ins_encode %{
 720     BasicType bt = Matcher::vector_element_basic_type(this);
 721     __ sve_bic(as_FloatRegister($dst_src1$$reg), __ elemType_to_regVariant(bt),
 722                as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
 723   %}
 724   ins_pipe(pipe_slow);
 725 %}')dnl
 726 dnl                     $1, $2
 727 VECTOR_AND_NOT_PREDICATE(I, B/H/S)
 728 VECTOR_AND_NOT_PREDICATE(L, D)
 729 undefine(MATCH_RULE)
 730 dnl
 731 dnl VDIVF($1,          $2  , $3         )
 732 dnl VDIVF(name_suffix, size, min_vec_len)
 733 define(`VDIVF', `
 734 instruct vdiv$1(vReg dst_src1, vReg src2) %{
 735   predicate(UseSVE > 0);
 736   match(Set dst_src1 (DivV$1 dst_src1 src2));
 737   ins_cost(SVE_COST);
 738   format %{ "sve_fdiv  $dst_src1, $dst_src1, $src2\t# vector (sve) ($2)" %}
 739   ins_encode %{
 740     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ $2,
 741          ptrue, as_FloatRegister($src2$$reg));
 742   %}
 743   ins_pipe(pipe_slow);
 744 %}')dnl
 745 dnl
 746 // vector float div
 747 VDIVF(F, S, 4)
 748 VDIVF(D, D, 2)
 749 
 750 // vector float div - predicated
 751 BINARY_OP_PREDICATE(vfdivF, DivVF, S, sve_fdiv)
 752 BINARY_OP_PREDICATE(vfdivD, DivVD, D, sve_fdiv)
 753 
 754 dnl
 755 dnl VMINMAX($1     , $2, $3   , $4  )
 756 dnl VMINMAX(op_name, op, finsn, insn)
 757 define(`VMINMAX', `
 758 instruct v$1(vReg dst_src1, vReg src2) %{
 759   predicate(UseSVE > 0);
 760   match(Set dst_src1 ($2 dst_src1 src2));
 761   ins_cost(SVE_COST);
 762   format %{ "sve_$1 $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
 763   ins_encode %{
 764     BasicType bt = Matcher::vector_element_basic_type(this);
 765     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
 766     if (is_floating_point_type(bt)) {
 767       __ $3(as_FloatRegister($dst_src1$$reg), size,
 768                   ptrue, as_FloatRegister($src2$$reg));
 769     } else {
 770       assert(is_integral_type(bt), "unsupported type");
 771       __ $4(as_FloatRegister($dst_src1$$reg), size,
 772                   ptrue, as_FloatRegister($src2$$reg));
 773     }
 774   %}
 775   ins_pipe(pipe_slow);
 776 %}')dnl
 777 dnl
 778 // vector min/max
 779 VMINMAX(min, MinV, sve_fmin, sve_smin)
 780 VMINMAX(max, MaxV, sve_fmax, sve_smax)
 781 
 782 dnl
 783 dnl VMINMAX_PREDICATE($1     , $2, $3   , $4  )
 784 dnl VMINMAX_PREDICATE(op_name, op, finsn, insn)
 785 define(`VMINMAX_PREDICATE', `
 786 instruct v$1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 787   predicate(UseSVE > 0);
 788   match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
 789   ins_cost(SVE_COST);
 790   format %{ "sve_$1 $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %}
 791   ins_encode %{
 792     BasicType bt = Matcher::vector_element_basic_type(this);
 793     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
 794     if (is_floating_point_type(bt)) {
 795       __ $3(as_FloatRegister($dst_src1$$reg), size,
 796                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
 797     } else {
 798       assert(is_integral_type(bt), "unsupported type");
 799       __ $4(as_FloatRegister($dst_src1$$reg), size,
 800                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
 801     }
 802   %}
 803   ins_pipe(pipe_slow);
 804 %}')dnl
 805 dnl
 806 // vector min/max - predicated
 807 VMINMAX_PREDICATE(min, MinV, sve_fmin, sve_smin)
 808 VMINMAX_PREDICATE(max, MaxV, sve_fmax, sve_smax)
 809 
 810 dnl
 811 dnl VFMLA($1           $2  )
 812 dnl VFMLA(name_suffix, size)
 813 define(`VFMLA', `
 814 // dst_src1 = dst_src1 + src2 * src3
 815 instruct vfmla$1(vReg dst_src1, vReg src2, vReg src3) %{
 816   predicate(UseFMA && UseSVE > 0);
 817   match(Set dst_src1 (FmaV$1 dst_src1 (Binary src2 src3)));
 818   ins_cost(SVE_COST);
 819   format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
 820   ins_encode %{
 821     __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ $2,
 822          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 823   %}
 824   ins_pipe(pipe_slow);
 825 %}')dnl
 826 dnl
 827 // vector fmla
 828 VFMLA(F, S)
 829 VFMLA(D, D)
 830 
 831 dnl
 832 dnl VFMAD_PREDICATE($1           $2  )
 833 dnl VFMAD_PREDICATE(name_suffix, size)
 834 define(`VFMAD_PREDICATE', `
 835 // dst_src1 = dst_src1 * src2 + src3
 836 instruct vfmad$1_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
 837   predicate(UseFMA && UseSVE > 0);
 838   match(Set dst_src1 (FmaV$1 (Binary dst_src1 src2) (Binary src3 pg)));
 839   ins_cost(SVE_COST);
 840   format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) ($2)" %}
 841   ins_encode %{
 842     __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ $2, as_PRegister($pg$$reg),
 843          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 844   %}
 845   ins_pipe(pipe_slow);
 846 %}')dnl
 847 dnl
 848 // vector fmad - predicated
 849 VFMAD_PREDICATE(F, S)
 850 VFMAD_PREDICATE(D, D)
 851 
 852 dnl
 853 dnl VFMLS1($1           $2  )
 854 dnl VFMLS1(name_suffix, size)
 855 define(`VFMLS1', `
 856 // dst_src1 = dst_src1 + -src2 * src3
 857 // The NegV$1 must not be predicated.
 858 instruct vfmls`$1'1(vReg dst_src1, vReg src2, vReg src3) %{
 859   predicate(UseFMA && UseSVE > 0 &&
 860             !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
 861   match(Set dst_src1 (FmaV$1 dst_src1 (Binary (NegV$1 src2) src3)));
 862   ins_cost(SVE_COST);
 863   format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
 864   ins_encode %{
 865     __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ $2,
 866          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 867   %}
 868   ins_pipe(pipe_slow);
 869 %}')dnl
 870 dnl
 871 dnl VFMLS2($1           $2  )
 872 dnl VFMLS2(name_suffix, size)
 873 define(`VFMLS2', `
 874 // dst_src1 = dst_src1 + src2 * -src3
 875 // The NegV$1 must not be predicated.
 876 instruct vfmls`$1'2(vReg dst_src1, vReg src2, vReg src3) %{
 877   predicate(UseFMA && UseSVE > 0 &&
 878             !n->in(2)->in(2)->as_Vector()->is_predicated_vector());
 879   match(Set dst_src1 (FmaV$1 dst_src1 (Binary src2 (NegV$1 src3))));
 880   ins_cost(SVE_COST);
 881   format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
 882   ins_encode %{
 883     __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ $2,
 884          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 885   %}
 886   ins_pipe(pipe_slow);
 887 %}')dnl
 888 dnl
 889 // vector fmls
 890 VFMLS1(F, S)
 891 VFMLS2(F, S)
 892 VFMLS1(D, D)
 893 VFMLS2(D, D)
 894 
 895 dnl
 896 dnl VFMSB_PREDICATE($1           $2  )
 897 dnl VFMSB_PREDICATE(name_suffix, size)
 898 define(`VFMSB_PREDICATE', `
 899 // dst_src1 = dst_src1 * -src2 + src3
 900 // The NegV$1 must not be predicated.
 901 instruct vfmsb$1_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
 902   predicate(UseFMA && UseSVE > 0 &&
 903             !n->in(1)->in(2)->as_Vector()->is_predicated_vector());
 904   match(Set dst_src1 (FmaV$1 (Binary dst_src1 (NegV$1 src2)) (Binary src3 pg)));
 905   ins_cost(SVE_COST);
 906   format %{ "sve_fmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) ($2)" %}
 907   ins_encode %{
 908     __ sve_fmsb(as_FloatRegister($dst_src1$$reg), __ $2, as_PRegister($pg$$reg),
 909          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 910   %}
 911   ins_pipe(pipe_slow);
 912 %}')dnl
 913 dnl
 914 // vector fmsb - predicated
 915 VFMSB_PREDICATE(F, S)
 916 VFMSB_PREDICATE(D, D)
 917 
 918 dnl
 919 dnl VFNMLA1($1           $2  )
 920 dnl VFNMLA1(name_suffix, size)
 921 define(`VFNMLA1', `
 922 // dst_src1 = -dst_src1 + -src2 * src3
 923 // The NegV$1 must not be predicated.
 924 instruct vfnmla`$1'1(vReg dst_src1, vReg src2, vReg src3) %{
 925   predicate(UseFMA && UseSVE > 0 &&
 926             !n->in(1)->as_Vector()->is_predicated_vector() &&
 927             !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
 928   match(Set dst_src1 (FmaV$1 (NegV$1 dst_src1) (Binary (NegV$1 src2) src3)));
 929   ins_cost(SVE_COST);
 930   format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
 931   ins_encode %{
 932     __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ $2,
 933          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 934   %}
 935   ins_pipe(pipe_slow);
 936 %}')dnl
 937 dnl
 938 dnl VFNMLA2($1           $2  )
 939 dnl VFNMLA2(name_suffix, size)
 940 define(`VFNMLA2', `
 941 // dst_src1 = -dst_src1 + src2 * -src3
 942 // The NegV$1 must not be predicated.
 943 instruct vfnmla`$1'2(vReg dst_src1, vReg src2, vReg src3) %{
 944   predicate(UseFMA && UseSVE > 0 &&
 945             !n->in(1)->as_Vector()->is_predicated_vector() &&
 946             !n->in(2)->in(2)->as_Vector()->is_predicated_vector());
 947   match(Set dst_src1 (FmaV$1 (NegV$1 dst_src1) (Binary src2 (NegV$1 src3))));
 948   ins_cost(SVE_COST);
 949   format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
 950   ins_encode %{
 951     __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ $2,
 952          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 953   %}
 954   ins_pipe(pipe_slow);
 955 %}')dnl
 956 dnl
 957 // vector fnmla
 958 VFNMLA1(F, S)
 959 VFNMLA2(F, S)
 960 VFNMLA1(D, D)
 961 VFNMLA2(D, D)
 962 
 963 dnl
 964 dnl VFNMAD_PREDICATE($1           $2  )
 965 dnl VFNMAD_PREDICATE(name_suffix, size)
 966 define(`VFNMAD_PREDICATE', `
 967 // dst_src1 = -src3 + dst_src1 * -src2
 968 // The NegV$1 must not be predicated.
 969 instruct vfnmad$1_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
 970   predicate(UseFMA && UseSVE > 0 &&
 971             !n->in(1)->in(2)->as_Vector()->is_predicated_vector() &&
 972             !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
 973   match(Set dst_src1 (FmaV$1 (Binary dst_src1 (NegV$1 src2)) (Binary (NegV$1 src3) pg)));
 974   ins_cost(SVE_COST);
 975   format %{ "sve_fnmad $dst_src1, $pg, $src2, $src3\t # vector (sve) ($2)" %}
 976   ins_encode %{
 977     __ sve_fnmad(as_FloatRegister($dst_src1$$reg), __ $2, as_PRegister($pg$$reg),
 978          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 979   %}
 980   ins_pipe(pipe_slow);
 981 %}')dnl
 982 dnl
 983 // vector fnmad - predicated
 984 VFNMAD_PREDICATE(F, S)
 985 VFNMAD_PREDICATE(D, D)
 986 
 987 dnl
 988 dnl VFNMLS($1           $2  )
 989 dnl VFNMLS(name_suffix, size)
 990 define(`VFNMLS', `
 991 // dst_src1 = -dst_src1 + src2 * src3
 992 // The NegV$1 must not be predicated.
 993 instruct vfnmls$1(vReg dst_src1, vReg src2, vReg src3) %{
 994   predicate(UseFMA && UseSVE > 0 &&
 995             !n->in(1)->as_Vector()->is_predicated_vector());
 996   match(Set dst_src1 (FmaV$1 (NegV$1 dst_src1) (Binary src2 src3)));
 997   ins_cost(SVE_COST);
 998   format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
 999   ins_encode %{
1000     __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ $2,
1001          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1002   %}
1003   ins_pipe(pipe_slow);
1004 %}')dnl
1005 dnl
1006 // vector fnmls
1007 VFNMLS(F, S)
1008 VFNMLS(D, D)
1009 
1010 dnl
1011 dnl VFNMSB_PREDICATE($1           $2  )
1012 dnl VFNMSB_PREDICATE(name_suffix, size)
1013 define(`VFNMSB_PREDICATE', `
1014 // dst_src1 = -src3 + dst_src1 * src2
1015 // The NegV$1 must not be predicated.
1016 instruct vfnmsb$1_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1017   predicate(UseFMA && UseSVE > 0 &&
1018             !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1019   match(Set dst_src1 (FmaV$1 (Binary dst_src1 src2) (Binary (NegV$1 src3) pg)));
1020   ins_cost(SVE_COST);
1021   format %{ "sve_fnmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) ($2)" %}
1022   ins_encode %{
1023     __ sve_fnmsb(as_FloatRegister($dst_src1$$reg), __ $2, as_PRegister($pg$$reg),
1024          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1025   %}
1026   ins_pipe(pipe_slow);
1027 %}')dnl
1028 dnl
1029 // vector fnmsb - predicated
1030 VFNMSB_PREDICATE(F, S)
1031 VFNMSB_PREDICATE(D, D)
1032 
1033 dnl
1034 dnl VMLA($1           $2  )
1035 dnl VMLA(name_suffix, size)
1036 define(`VMLA', `
1037 // dst_src1 = dst_src1 + src2 * src3
1038 instruct vmla$1(vReg dst_src1, vReg src2, vReg src3)
1039 %{
1040   predicate(UseSVE > 0);
1041   match(Set dst_src1 (AddV$1 dst_src1 (MulV$1 src2 src3)));
1042   ins_cost(SVE_COST);
1043   format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) ($2)" %}
1044   ins_encode %{
1045     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ $2,
1046       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1047   %}
1048   ins_pipe(pipe_slow);
1049 %}')dnl
1050 dnl
1051 // vector mla
1052 VMLA(B, B)
1053 VMLA(S, H)
1054 VMLA(I, S)
1055 VMLA(L, D)
1056 
1057 dnl
1058 dnl VMLA_PREDICATE($1           $2  )
1059 dnl VMLA_PREDICATE(name_suffix, size)
1060 define(`VMLA_PREDICATE', `
1061 // dst_src1 = dst_src1 + src2 * src3
1062 instruct vmla$1_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1063 %{
1064   predicate(UseSVE > 0);
1065   match(Set dst_src1 (AddV$1 (Binary dst_src1 (MulV$1 src2 src3)) pg));
1066   ins_cost(SVE_COST);
1067   format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) ($2)" %}
1068   ins_encode %{
1069     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ $2, as_PRegister($pg$$reg),
1070          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1071   %}
1072   ins_pipe(pipe_slow);
1073 %}')dnl
1074 dnl
1075 // vector mla - predicated
1076 VMLA_PREDICATE(B, B)
1077 VMLA_PREDICATE(S, H)
1078 VMLA_PREDICATE(I, S)
1079 VMLA_PREDICATE(L, D)
1080 
1081 dnl
1082 dnl VMLS($1           $2  )
1083 dnl VMLS(name_suffix, size)
1084 define(`VMLS', `
1085 // dst_src1 = dst_src1 - src2 * src3
1086 instruct vmls$1(vReg dst_src1, vReg src2, vReg src3)
1087 %{
1088   predicate(UseSVE > 0);
1089   match(Set dst_src1 (SubV$1 dst_src1 (MulV$1 src2 src3)));
1090   ins_cost(SVE_COST);
1091   format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) ($2)" %}
1092   ins_encode %{
1093     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ $2,
1094       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1095   %}
1096   ins_pipe(pipe_slow);
1097 %}')dnl
1098 dnl
1099 // vector mls
1100 VMLS(B, B)
1101 VMLS(S, H)
1102 VMLS(I, S)
1103 VMLS(L, D)
1104 
1105 dnl
1106 dnl VMLS_PREDICATE($1           $2  )
1107 dnl VMLS_PREDICATE(name_suffix, size)
1108 define(`VMLS_PREDICATE', `
1109 // dst_src1 = dst_src1 - src2 * src3
1110 instruct vmls$1_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1111 %{
1112   predicate(UseSVE > 0);
1113   match(Set dst_src1 (SubV$1 (Binary dst_src1 (MulV$1 src2 src3)) pg));
1114   ins_cost(SVE_COST);
1115   format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) ($2)" %}
1116   ins_encode %{
1117     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ $2, as_PRegister($pg$$reg),
1118          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1119   %}
1120   ins_pipe(pipe_slow);
1121 %}')dnl
1122 dnl
1123 // vector mls - predicated
1124 VMLS_PREDICATE(B, B)
1125 VMLS_PREDICATE(S, H)
1126 VMLS_PREDICATE(I, S)
1127 VMLS_PREDICATE(L, D)
1128 
1129 dnl
1130 dnl BINARY_OP_TRUE_PREDICATE($1,        $2,      $3,   $4,          $5  )
1131 dnl BINARY_OP_TRUE_PREDICATE(insn_name, op_name, size, min_vec_len, insn)
1132 define(`BINARY_OP_TRUE_PREDICATE', `
1133 instruct $1(vReg dst_src1, vReg src2) %{
1134   predicate(UseSVE > 0);
1135   match(Set dst_src1 ($2 dst_src1 src2));
1136   ins_cost(SVE_COST);
1137   format %{ "$5 $dst_src1, $dst_src1, $src2\t # vector (sve) ($3)" %}
1138   ins_encode %{
1139     __ $5(as_FloatRegister($dst_src1$$reg), __ $3,
1140          ptrue, as_FloatRegister($src2$$reg));
1141   %}
1142   ins_pipe(pipe_slow);
1143 %}')dnl
1144 dnl
1145 // vector mul
1146 BINARY_OP_TRUE_PREDICATE(vmulB, MulVB, B, 16, sve_mul)
1147 BINARY_OP_TRUE_PREDICATE(vmulS, MulVS, H, 8,  sve_mul)
1148 BINARY_OP_TRUE_PREDICATE(vmulI, MulVI, S, 4,  sve_mul)
1149 BINARY_OP_TRUE_PREDICATE(vmulL, MulVL, D, 2,  sve_mul)
1150 BINARY_OP_UNPREDICATE(vmulF, MulVF, S, 4, sve_fmul)
1151 BINARY_OP_UNPREDICATE(vmulD, MulVD, D, 2, sve_fmul)
1152 
1153 // vector mul - predicated
1154 BINARY_OP_PREDICATE(vmulB, MulVB, B, sve_mul)
1155 BINARY_OP_PREDICATE(vmulS, MulVS, H, sve_mul)
1156 BINARY_OP_PREDICATE(vmulI, MulVI, S, sve_mul)
1157 BINARY_OP_PREDICATE(vmulL, MulVL, D, sve_mul)
1158 BINARY_OP_PREDICATE(vmulF, MulVF, S, sve_fmul)
1159 BINARY_OP_PREDICATE(vmulD, MulVD, D, sve_fmul)
1160 
1161 // vector neg
1162 UNARY_OP_TRUE_PREDICATE(vnegI, NegVI, B/H/S, sve_neg)
1163 UNARY_OP_TRUE_PREDICATE(vnegL, NegVL, D, sve_neg)
1164 UNARY_OP_TRUE_PREDICATE(vnegF, NegVF, S, sve_fneg)
1165 UNARY_OP_TRUE_PREDICATE(vnegD, NegVD, D, sve_fneg)
1166 
1167 // vector neg - predicated
1168 UNARY_OP_PREDICATE(vnegI, NegVI, B/H/S, sve_neg)
1169 UNARY_OP_PREDICATE(vnegL, NegVL, D, sve_neg)
1170 UNARY_OP_PREDICATE(vnegF, NegVF, S, sve_fneg)
1171 UNARY_OP_PREDICATE(vnegD, NegVD, D, sve_fneg)
1172 
1173 // popcount vector
1174 
1175 instruct vpopcountI(vReg dst, vReg src) %{
1176   predicate(UseSVE > 0);
1177   match(Set dst (PopCountVI src));
1178   format %{ "sve_cnt $dst, $src\t# vector (sve) (S)\n\t" %}
1179   ins_encode %{
1180      __ sve_cnt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg));
1181   %}
1182   ins_pipe(pipe_slow);
1183 %}
1184 
1185 // vector blend
1186 
1187 instruct vblend(vReg dst, vReg src1, vReg src2, pRegGov pg) %{
1188   predicate(UseSVE > 0);
1189   match(Set dst (VectorBlend (Binary src1 src2) pg));
1190   ins_cost(SVE_COST);
1191   format %{ "sve_sel $dst, $pg, $src2, $src1\t# vector blend (sve)" %}
1192   ins_encode %{
1193     Assembler::SIMD_RegVariant size =
1194                __ elemType_to_regVariant(Matcher::vector_element_basic_type(this));
1195     __ sve_sel(as_FloatRegister($dst$$reg), size, as_PRegister($pg$$reg),
1196                as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
1197   %}
1198   ins_pipe(pipe_slow);
1199 %}
1200 
1201 // vector store mask
1202 
1203 instruct vstoremaskB(vReg dst, pRegGov src, immI_1 size) %{
1204   predicate(UseSVE > 0);
1205   match(Set dst (VectorStoreMask src size));
1206   ins_cost(SVE_COST);
1207   format %{ "vstoremask $dst, $src\t# vector store mask (sve) (B)" %}
1208   ins_encode %{
1209     __ sve_cpy(as_FloatRegister($dst$$reg), __ B, as_PRegister($src$$reg), 1, false);
1210   %}
1211   ins_pipe(pipe_slow);
1212 %}
1213 
1214 instruct vstoremask_narrow(vReg dst, pRegGov src, vReg tmp, immI_gt_1 size) %{
1215   predicate(UseSVE > 0);
1216   match(Set dst (VectorStoreMask src size));
1217   effect(TEMP_DEF dst, TEMP tmp);
1218   ins_cost(3 * SVE_COST);
1219   format %{ "vstoremask $dst, $src\t# vector store mask (sve) (H/S/D)" %}
1220   ins_encode %{
1221     Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant((int)$size$$constant);
1222     __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($src$$reg), 1, false);
1223     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
1224                          as_FloatRegister($dst$$reg), size, as_FloatRegister($tmp$$reg));
1225   %}
1226   ins_pipe(pipe_slow);
1227 %}
1228 
1229 // Combine LoadVector+VectorLoadMask when the vector element type is not T_BYTE
1230 
1231 instruct vloadmask_loadV(pRegGov dst, indirect mem, vReg tmp, rFlagsReg cr) %{
1232   predicate(UseSVE > 0 &&
1233             n->as_Vector()->length_in_bytes() == MaxVectorSize &&
1234             type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
1235   match(Set dst (VectorLoadMask (LoadVector mem)));
1236   effect(TEMP tmp, KILL cr);
1237   ins_cost(3 * SVE_COST);
1238   format %{ "sve_ld1b $tmp, $mem\n\t"
1239             "sve_cmpne $dst, $tmp, 0\t# load vector mask (sve) (H/S/D)" %}
1240   ins_encode %{
1241     // Load mask values which are boolean type, and extend them to the
1242     // expected vector element type. Convert the vector to predicate.
1243     BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
1244     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($tmp$$reg),
1245                           ptrue, T_BOOLEAN, to_vect_bt, $mem->opcode(),
1246                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1247     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ elemType_to_regVariant(to_vect_bt),
1248                ptrue, as_FloatRegister($tmp$$reg), 0);
1249   %}
1250   ins_pipe(pipe_slow);
1251 %}
1252 
1253 instruct vloadmask_loadV_partial(pRegGov dst, indirect mem, vReg vtmp, pRegGov ptmp, rFlagsReg cr) %{
1254   predicate(UseSVE > 0 &&
1255             n->as_Vector()->length_in_bytes() > 16 &&
1256             n->as_Vector()->length_in_bytes() < MaxVectorSize &&
1257             type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
1258   match(Set dst (VectorLoadMask (LoadVector mem)));
1259   effect(TEMP vtmp, TEMP ptmp, KILL cr);
1260   ins_cost(6 * SVE_COST);
1261   format %{ "vloadmask_loadV $dst, $mem\t# load vector mask partial (sve) (H/S/D)" %}
1262   ins_encode %{
1263     // Load valid mask values which are boolean type, and extend them to the
1264     // expected vector element type. Convert the vector to predicate.
1265     BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
1266     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(to_vect_bt);
1267     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this));
1268     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($vtmp$$reg),
1269                           as_PRegister($ptmp$$reg), T_BOOLEAN, to_vect_bt, $mem->opcode(),
1270                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1271     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($vtmp$$reg), 0);
1272   %}
1273   ins_pipe(pipe_slow);
1274 %}
1275 
1276 // Combine VectorStoreMask+StoreVector when the vector element type is not T_BYTE
1277 
1278 instruct storeV_vstoremask(indirect mem, pRegGov src, vReg tmp, immI_gt_1 esize) %{
1279   predicate(UseSVE > 0 &&
1280             Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
1281   match(Set mem (StoreVector mem (VectorStoreMask src esize)));
1282   effect(TEMP tmp);
1283   ins_cost(3 * SVE_COST);
1284   format %{ "sve_cpy $tmp, $src, 1\n\t"
1285             "sve_st1b $tmp, $mem\t# store vector mask (sve) (H/S/D)" %}
1286   ins_encode %{
1287     BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
1288     assert(type2aelembytes(from_vect_bt) == (int)$esize$$constant, "unsupported type.");
1289     Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
1290     __ sve_cpy(as_FloatRegister($tmp$$reg), size, as_PRegister($src$$reg), 1, false);
1291     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($tmp$$reg),
1292                           ptrue, T_BOOLEAN, from_vect_bt, $mem->opcode(),
1293                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1294   %}
1295   ins_pipe(pipe_slow);
1296 %}
1297 
1298 instruct storeV_vstoremask_partial(indirect mem, pRegGov src, vReg vtmp,
1299                                    immI_gt_1 esize, pRegGov ptmp, rFlagsReg cr) %{
1300   predicate(UseSVE > 0 &&
1301             n->as_StoreVector()->memory_size() > 16 &&
1302             type2aelembytes(n->as_StoreVector()->vect_type()->element_basic_type()) > 1 &&
1303             Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
1304   match(Set mem (StoreVector mem (VectorStoreMask src esize)));
1305   effect(TEMP vtmp, TEMP ptmp, KILL cr);
1306   format %{ "storeV_vstoremask $src, $mem\t# store vector mask partial (sve) (H/S/D)" %}
1307   ins_cost(6 * SVE_COST);
1308   ins_encode %{
1309     // Convert the valid src predicate to vector, and store the vector
1310     // elements as boolean values.
1311     BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
1312     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(from_vect_bt);
1313     __ sve_cpy(as_FloatRegister($vtmp$$reg), size, as_PRegister($src$$reg), 1, false);
1314     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
1315     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($vtmp$$reg),
1316                           as_PRegister($ptmp$$reg), T_BOOLEAN, from_vect_bt, $mem->opcode(),
1317                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1318   %}
1319   ins_pipe(pipe_slow);
1320 %}
1321 dnl
1322 dnl REDUCE_I($1,        $2     )
1323 dnl REDUCE_I(insn_name, op_name)
1324 define(`REDUCE_I', `
1325 instruct reduce_$1I(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
1326   ifelse($2, AddReductionVI,
1327        `predicate(UseSVE > 0 &&
1328             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);',
1329        `predicate(UseSVE > 0 &&
1330             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1331             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);')
1332   match(Set dst ($2 src1 src2));
1333   effect(TEMP_DEF dst, TEMP tmp);
1334   ins_cost(SVE_COST);
1335   format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction (sve) (may extend)" %}
1336   ins_encode %{
1337     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1338     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1339                            $src1$$Register, as_FloatRegister($src2$$reg),
1340                            ptrue, as_FloatRegister($tmp$$reg));
1341   %}
1342   ins_pipe(pipe_slow);
1343 %}')dnl
1344 dnl
1345 dnl
1346 dnl REDUCE_L($1,        $2    )
1347 dnl REDUCE_L(insn_name, op_name)
1348 define(`REDUCE_L', `
1349 instruct reduce_$1L(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
1350   ifelse($2, AddReductionVL,
1351        `predicate(UseSVE > 0 &&
1352             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);',
1353        `predicate(UseSVE > 0 &&
1354             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1355             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);')
1356   match(Set dst ($2 src1 src2));
1357   effect(TEMP_DEF dst, TEMP tmp);
1358   ins_cost(SVE_COST);
1359   format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction (sve)" %}
1360   ins_encode %{
1361     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1362                            $src1$$Register, as_FloatRegister($src2$$reg),
1363                            ptrue, as_FloatRegister($tmp$$reg));
1364   %}
1365   ins_pipe(pipe_slow);
1366 %}')dnl
1367 dnl
1368 dnl REDUCE_I_PARTIAL($1,        $2     )
1369 dnl REDUCE_I_PARTIAL(insn_name, op_name)
1370 define(`REDUCE_I_PARTIAL', `
1371 instruct reduce_$1I_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1372                              pRegGov ptmp, rFlagsReg cr) %{
1373   ifelse($2, AddReductionVI,
1374        `predicate(UseSVE > 0 &&
1375             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);',
1376        `predicate(UseSVE > 0 &&
1377             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1378             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);')
1379   match(Set dst ($2 src1 src2));
1380   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1381   ins_cost(2 * SVE_COST);
1382   format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction partial (sve) (may extend)" %}
1383   ins_encode %{
1384     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1385     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1386     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
1387     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1388                            $src1$$Register, as_FloatRegister($src2$$reg),
1389                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1390   %}
1391   ins_pipe(pipe_slow);
1392 %}')dnl
1393 dnl
1394 dnl REDUCE_L_PARTIAL($1,        $2    )
1395 dnl REDUCE_L_PARTIAL(insn_name, op_name)
1396 define(`REDUCE_L_PARTIAL', `
1397 instruct reduce_$1L_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1398                              pRegGov ptmp, rFlagsReg cr) %{
1399   ifelse($2, AddReductionVL,
1400        `predicate(UseSVE > 0 &&
1401             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);',
1402        `predicate(UseSVE > 0 &&
1403             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1404             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);')
1405   match(Set dst ($2 src1 src2));
1406   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1407   ins_cost(2 * SVE_COST);
1408   format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction partial (sve)" %}
1409   ins_encode %{
1410     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
1411     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1412                            $src1$$Register, as_FloatRegister($src2$$reg),
1413                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1414   %}
1415   ins_pipe(pipe_slow);
1416 %}')dnl
1417 dnl
1418 dnl REDUCE_ADDF($1,        $2,      $3,      $4  )
1419 dnl REDUCE_ADDF(insn_name, op_name, reg_dst, size)
1420 define(`REDUCE_ADDF', `
1421 instruct reduce_$1($3 src1_dst, vReg src2) %{
1422   predicate(UseSVE > 0 &&
1423             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1424   match(Set src1_dst ($2 src1_dst src2));
1425   ins_cost(SVE_COST);
1426   format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) ($4)" %}
1427   ins_encode %{
1428     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1429          ptrue, as_FloatRegister($src2$$reg));
1430   %}
1431   ins_pipe(pipe_slow);
1432 %}')dnl
1433 dnl
1434 dnl
1435 dnl REDUCE_ADDF_PARTIAL($1,        $2,     $3,      $4  )
1436 dnl REDUCE_ADDF_PARTIAL(insn_name, suffix, reg_dst, size)
1437 define(`REDUCE_ADDF_PARTIAL', `
1438 instruct reduce_$1_partial($3 src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
1439   predicate(UseSVE > 0 &&
1440             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1441   match(Set src1_dst ($2 src1_dst src2));
1442   ins_cost(SVE_COST);
1443   effect(TEMP ptmp, KILL cr);
1444   format %{ "sve_reduce_$1 $src1_dst, $src1_dst, $src2\t# $1 reduction partial (sve) ($4)" %}
1445   ins_encode %{
1446     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ $4, Matcher::vector_length(this, $src2));
1447     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1448                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1449   %}
1450   ins_pipe(pipe_slow);
1451 %}')dnl
1452 dnl
1453 dnl
1454 dnl REDUCE_I_PREDICATE($1,        $2     )
1455 dnl REDUCE_I_PREDICATE(insn_name, op_name)
1456 define(`REDUCE_I_PREDICATE', `
1457 instruct reduce_$1I_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
1458   ifelse($2, AddReductionVI,
1459        `predicate(UseSVE > 0);',
1460        `predicate(UseSVE > 0 &&
1461             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG);')
1462   match(Set dst ($2 (Binary src1 src2) pg));
1463   effect(TEMP_DEF dst, TEMP tmp);
1464   ins_cost(SVE_COST);
1465   format %{ "sve_reduce_$1I $dst, $src1, $pg, $src2\t# $1I reduction predicated (sve) (may extend)" %}
1466   ins_encode %{
1467     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1468     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1469                            $src1$$Register, as_FloatRegister($src2$$reg),
1470                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
1471   %}
1472   ins_pipe(pipe_slow);
1473 %}')dnl
1474 dnl
1475 dnl REDUCE_L_PREDICATE($1,        $2    )
1476 dnl REDUCE_L_PREDICATE(insn_name, op_name)
1477 define(`REDUCE_L_PREDICATE', `
1478 instruct reduce_$1L_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
1479   ifelse($2, AddReductionVL,
1480        `predicate(UseSVE > 0);',
1481        `predicate(UseSVE > 0 &&
1482             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);')
1483   match(Set dst ($2 (Binary src1 src2) pg));
1484   effect(TEMP_DEF dst, TEMP tmp);
1485   ins_cost(SVE_COST);
1486   format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated (sve)" %}
1487   ins_encode %{
1488     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1489                            $src1$$Register, as_FloatRegister($src2$$reg),
1490                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
1491   %}
1492   ins_pipe(pipe_slow);
1493 %}')dnl
1494 dnl
1495 dnl REDUCE_ADDF_PREDICATE($1,        $2,      $3,      $4  )
1496 dnl REDUCE_ADDF_PREDICATE(insn_name, op_name, reg_dst, size)
1497 define(`REDUCE_ADDF_PREDICATE', `
1498 instruct reduce_$1_masked($3 src1_dst, vReg src2, pRegGov pg) %{
1499   predicate(UseSVE > 0);
1500   match(Set src1_dst ($2 (Binary src1_dst src2) pg));
1501   ins_cost(SVE_COST);
1502   format %{ "sve_reduce_$1 $src1_dst, $pg, $src2\t# $1 reduction predicated (sve)" %}
1503   ins_encode %{
1504     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1505                  as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1506   %}
1507   ins_pipe(pipe_slow);
1508 %}')dnl
1509 dnl
1510 
1511 // vector add reduction
1512 REDUCE_I(add, AddReductionVI)
1513 REDUCE_L(add, AddReductionVL)
1514 REDUCE_ADDF(addF, AddReductionVF, vRegF, S)
1515 REDUCE_ADDF(addD, AddReductionVD, vRegD, D)
1516 REDUCE_I_PARTIAL(add, AddReductionVI)
1517 REDUCE_L_PARTIAL(add, AddReductionVL)
1518 REDUCE_ADDF_PARTIAL(addF, AddReductionVF, vRegF, S)
1519 REDUCE_ADDF_PARTIAL(addD, AddReductionVD, vRegD, D)
1520 
1521 // vector add reduction - predicated
1522 REDUCE_I_PREDICATE(add, AddReductionVI)
1523 REDUCE_L_PREDICATE(add, AddReductionVL)
1524 REDUCE_ADDF_PREDICATE(addF, AddReductionVF, vRegF, S)
1525 REDUCE_ADDF_PREDICATE(addD, AddReductionVD, vRegD, D)
1526 
1527 // vector and reduction
1528 REDUCE_I(and, AndReductionV)
1529 REDUCE_L(and, AndReductionV)
1530 REDUCE_I_PARTIAL(and, AndReductionV)
1531 REDUCE_L_PARTIAL(and, AndReductionV)
1532 
1533 // vector and reduction - predicated
1534 REDUCE_I_PREDICATE(and, AndReductionV)
1535 REDUCE_L_PREDICATE(and, AndReductionV)
1536 
1537 // vector or reduction
1538 REDUCE_I(or, OrReductionV)
1539 REDUCE_L(or, OrReductionV)
1540 REDUCE_I_PARTIAL(or, OrReductionV)
1541 REDUCE_L_PARTIAL(or, OrReductionV)
1542 
1543 // vector or reduction - predicated
1544 REDUCE_I_PREDICATE(or, OrReductionV)
1545 REDUCE_L_PREDICATE(or, OrReductionV)
1546 
1547 // vector xor reduction
1548 REDUCE_I(eor, XorReductionV)
1549 REDUCE_L(eor, XorReductionV)
1550 REDUCE_I_PARTIAL(eor, XorReductionV)
1551 REDUCE_L_PARTIAL(eor, XorReductionV)
1552 
1553 // vector xor reduction - predicated
1554 REDUCE_I_PREDICATE(eor, XorReductionV)
1555 REDUCE_L_PREDICATE(eor, XorReductionV)
1556 
1557 dnl
1558 dnl REDUCE_MAXMIN_I($1,        $2     )
1559 dnl REDUCE_MAXMIN_I(insn_name, op_name)
1560 define(`REDUCE_MAXMIN_I', `
1561 instruct reduce_$1I(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
1562   predicate(UseSVE > 0 &&
1563             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1564             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1565             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
1566   match(Set dst ($2 src1 src2));
1567   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1568   ins_cost(SVE_COST);
1569   format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction (sve)" %}
1570   ins_encode %{
1571     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1572     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1573                            $src1$$Register, as_FloatRegister($src2$$reg),
1574                            ptrue, as_FloatRegister($tmp$$reg));
1575   %}
1576   ins_pipe(pipe_slow);
1577 %}')dnl
1578 dnl
1579 dnl REDUCE_MAXMIN_L($1,        $2     )
1580 dnl REDUCE_MAXMIN_L(insn_name, op_name)
1581 define(`REDUCE_MAXMIN_L', `
1582 instruct reduce_$1L(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
1583   predicate(UseSVE > 0 &&
1584             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1585             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1586   match(Set dst ($2 src1 src2));
1587   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1588   ins_cost(SVE_COST);
1589   format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction (sve)" %}
1590   ins_encode %{
1591     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1592                            $src1$$Register, as_FloatRegister($src2$$reg),
1593                            ptrue, as_FloatRegister($tmp$$reg));
1594   %}
1595   ins_pipe(pipe_slow);
1596 %}')dnl
1597 dnl
1598 dnl REDUCE_MAXMIN_I_PARTIAL($1     , $2     )
1599 dnl REDUCE_MAXMIN_I_PARTIAL(min_max, op_name)
1600 define(`REDUCE_MAXMIN_I_PARTIAL', `
1601 instruct reduce_$1I_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1602                              pRegGov ptmp, rFlagsReg cr) %{
1603   predicate(UseSVE > 0 &&
1604             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1605             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1606             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
1607   match(Set dst ($2 src1 src2));
1608   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1609   ins_cost(2 * SVE_COST);
1610   format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction partial (sve)" %}
1611   ins_encode %{
1612     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1613     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1614     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
1615     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1616                            $src1$$Register, as_FloatRegister($src2$$reg),
1617                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1618   %}
1619   ins_pipe(pipe_slow);
1620 %}')dnl
1621 dnl
1622 dnl REDUCE_MAXMIN_L_PARTIAL($1     , $2     )
1623 dnl REDUCE_MAXMIN_L_PARTIAL(min_max, op_name)
1624 define(`REDUCE_MAXMIN_L_PARTIAL', `
1625 instruct reduce_$1L_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1626                              pRegGov ptmp, rFlagsReg cr) %{
1627   predicate(UseSVE > 0 &&
1628             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1629             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1630   match(Set dst ($2 src1 src2));
1631   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1632   ins_cost(2 * SVE_COST);
1633   format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction  partial (sve)" %}
1634   ins_encode %{
1635     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
1636     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1637                            $src1$$Register, as_FloatRegister($src2$$reg),
1638                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1639   %}
1640   ins_pipe(pipe_slow);
1641 %}')dnl
1642 dnl
1643 dnl REDUCE_MAXMIN_I_PREDICATE($1     , $2     )
1644 dnl REDUCE_MAXMIN_I_PREDICATE(min_max, op_name)
1645 define(`REDUCE_MAXMIN_I_PREDICATE', `
1646 instruct reduce_$1I_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp,
1647                            pRegGov pg, rFlagsReg cr) %{
1648   predicate(UseSVE > 0 &&
1649             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1650             is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
1651   match(Set dst ($2 (Binary src1 src2) pg));
1652   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1653   ins_cost(SVE_COST);
1654   format %{ "sve_reduce_$1I $dst, $src1, $pg, $src2\t# $1I reduction predicated (sve)" %}
1655   ins_encode %{
1656     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1657     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1658                            $src1$$Register, as_FloatRegister($src2$$reg),
1659                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
1660   %}
1661   ins_pipe(pipe_slow);
1662 %}')dnl
1663 dnl
1664 dnl REDUCE_MAXMIN_L_PREDICATE($1     , $2     )
1665 dnl REDUCE_MAXMIN_L_PREDICATE(min_max, op_name)
1666 define(`REDUCE_MAXMIN_L_PREDICATE', `
1667 instruct reduce_$1L_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp,
1668                           pRegGov pg, rFlagsReg cr) %{
1669   predicate(UseSVE > 0 &&
1670             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1671   match(Set dst ($2 (Binary src1 src2) pg));
1672   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1673   ins_cost(SVE_COST);
1674   format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated (sve)" %}
1675   ins_encode %{
1676     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1677                            $src1$$Register, as_FloatRegister($src2$$reg),
1678                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
1679   %}
1680   ins_pipe(pipe_slow);
1681 %}')dnl
1682 dnl
1683 dnl REDUCE_FMINMAX($1,      $2,          $3,           $4,   $5         )
1684 dnl REDUCE_FMINMAX(min_max, name_suffix, element_type, size, reg_src_dst)
1685 define(`REDUCE_FMINMAX', `
1686 instruct reduce_$1$2($5 dst, $5 src1, vReg src2) %{
1687   predicate(UseSVE > 0 &&
1688             n->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1689             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1690   match(Set dst (translit($1, `m', `M')ReductionV src1 src2));
1691   ins_cost(INSN_COST);
1692   effect(TEMP_DEF dst);
1693   format %{ "sve_reduce_$1$2 $dst, $src1, $src2\t# $1$2 reduction (sve)" %}
1694   ins_encode %{
1695     __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src2$$reg));
1696     __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1697   %}
1698   ins_pipe(pipe_slow);
1699 %}')dnl
1700 dnl
1701 dnl REDUCE_FMINMAX_PARTIAL($1,      $2,          $3,           $4,   $5         )
1702 dnl REDUCE_FMINMAX_PARTIAL(min_max, name_suffix, element_type, size, reg_src_dst)
1703 define(`REDUCE_FMINMAX_PARTIAL', `
1704 instruct reduce_$1$2_partial($5 dst, $5 src1, vReg src2,
1705                              pRegGov ptmp, rFlagsReg cr) %{
1706   predicate(UseSVE > 0 &&
1707             n->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1708             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1709   match(Set dst (translit($1, `m', `M')ReductionV src1 src2));
1710   ins_cost(INSN_COST);
1711   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
1712   format %{ "sve_reduce_$1$2 $dst, $src1, $src2\t# $1$2 reduction partial (sve)" %}
1713   ins_encode %{
1714     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ $4, Matcher::vector_length(this, $src2));
1715     __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1716     __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1717   %}
1718   ins_pipe(pipe_slow);
1719 %}')dnl
1720 dnl
1721 dnl REDUCE_FMINMAX_PREDICATE($1,      $2,          $3,           $4,   $5         )
1722 dnl REDUCE_FMINMAX_PREDICATE(min_max, name_suffix, element_type, size, reg_src_dst)
1723 define(`REDUCE_FMINMAX_PREDICATE', `
1724 instruct reduce_$1$2_masked($5 dst, $5 src1, vReg src2, pRegGov pg) %{
1725   predicate(UseSVE > 0 &&
1726             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == $3);
1727   match(Set dst (translit($1, `m', `M')ReductionV (Binary src1 src2) pg));
1728   ins_cost(SVE_COST);
1729   effect(TEMP_DEF dst);
1730   format %{ "sve_reduce_$1$2 $dst, $src1, $pg, $src2\t# $1$2 reduction predicated (sve)" %}
1731   ins_encode %{
1732     __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1733     __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1734   %}
1735   ins_pipe(pipe_slow);
1736 %}')dnl
1737 dnl
1738 // vector max reduction
1739 REDUCE_MAXMIN_I(max, MaxReductionV)
1740 REDUCE_MAXMIN_L(max, MaxReductionV)
1741 REDUCE_MAXMIN_I_PARTIAL(max, MaxReductionV)
1742 REDUCE_MAXMIN_L_PARTIAL(max, MaxReductionV)
1743 REDUCE_FMINMAX(max, F, T_FLOAT,  S, vRegF)
1744 REDUCE_FMINMAX_PARTIAL(max, F, T_FLOAT,  S, vRegF)
1745 REDUCE_FMINMAX(max, D, T_DOUBLE, D, vRegD)
1746 REDUCE_FMINMAX_PARTIAL(max, D, T_DOUBLE, D, vRegD)
1747 
1748 // vector max reduction - predicated
1749 REDUCE_MAXMIN_I_PREDICATE(max, MaxReductionV)
1750 REDUCE_MAXMIN_L_PREDICATE(max, MaxReductionV)
1751 REDUCE_FMINMAX_PREDICATE(max, F, T_FLOAT,  S, vRegF)
1752 REDUCE_FMINMAX_PREDICATE(max, D, T_DOUBLE, D, vRegD)
1753 
1754 // vector min reduction
1755 REDUCE_MAXMIN_I(min, MinReductionV)
1756 REDUCE_MAXMIN_L(min, MinReductionV)
1757 REDUCE_MAXMIN_I_PARTIAL(min, MinReductionV)
1758 REDUCE_MAXMIN_L_PARTIAL(min, MinReductionV)
1759 REDUCE_FMINMAX(min, F, T_FLOAT,  S, vRegF)
1760 REDUCE_FMINMAX_PARTIAL(min, F, T_FLOAT,  S, vRegF)
1761 REDUCE_FMINMAX(min, D, T_DOUBLE, D, vRegD)
1762 REDUCE_FMINMAX_PARTIAL(min, D, T_DOUBLE, D, vRegD)
1763 
1764 // vector min reduction - predicated
1765 REDUCE_MAXMIN_I_PREDICATE(min, MinReductionV)
1766 REDUCE_MAXMIN_L_PREDICATE(min, MinReductionV)
1767 REDUCE_FMINMAX_PREDICATE(min, F, T_FLOAT,  S, vRegF)
1768 REDUCE_FMINMAX_PREDICATE(min, D, T_DOUBLE, D, vRegD)
1769 
1770 // vector Math.rint, floor, ceil
1771 
1772 instruct vroundD(vReg dst, vReg src, immI rmode) %{
1773   predicate(UseSVE > 0 &&
1774             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
1775   match(Set dst (RoundDoubleModeV src rmode));
1776   format %{ "sve_frint $dst, $src, $rmode\t# vector (sve) (D)" %}
1777   ins_encode %{
1778     switch ($rmode$$constant) {
1779       case RoundDoubleModeNode::rmode_rint:
1780         __ sve_frintn(as_FloatRegister($dst$$reg), __ D,
1781              ptrue, as_FloatRegister($src$$reg));
1782         break;
1783       case RoundDoubleModeNode::rmode_floor:
1784         __ sve_frintm(as_FloatRegister($dst$$reg), __ D,
1785              ptrue, as_FloatRegister($src$$reg));
1786         break;
1787       case RoundDoubleModeNode::rmode_ceil:
1788         __ sve_frintp(as_FloatRegister($dst$$reg), __ D,
1789              ptrue, as_FloatRegister($src$$reg));
1790         break;
1791     }
1792   %}
1793   ins_pipe(pipe_slow);
1794 %}
1795 define(`VECTOR_JAVA_FROUND', `
1796 instruct vround$1to$3($7 dst, $7 src, $7 tmp1, $7 tmp2, $7 tmp3, pRegGov ptmp)
1797 %{
1798   predicate(UseSVE > 0);
1799   match(Set dst (RoundV$1 src));
1800   effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP ptmp);
1801   format %{ "sve_vround  $dst, $4, $src\t# round $1 to $3 vector" %}
1802   ins_encode %{
1803     BasicType bt = Matcher::vector_element_basic_type(this);
1804     int vlen = Matcher::vector_length_in_bytes(this);
1805     if (vlen > 16) {
1806       __ vector_round_sve(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
1807                           as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg),
1808                           as_PRegister($ptmp$$reg), __ $4);
1809     } else {
1810       __ vector_round_neon(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
1811                            as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg),
1812                            as_FloatRegister($tmp3$$reg),
1813                            __ esize2arrangement(type2aelembytes(bt),
1814                               /*isQ*/ vlen == 16));
1815     }
1816   %}
1817   ins_pipe(pipe_class_default);
1818 %}')dnl           $1  $2  $3 $4 $5    $6    $7
1819 VECTOR_JAVA_FROUND(F, 8F,  I, S, 8,  INT, vReg)
1820 VECTOR_JAVA_FROUND(D, 4D,  L, D, 4, LONG, vReg)
1821 dnl
1822 dnl REPLICATE($1,        $2,      $3,      $4,   $5         )
1823 dnl REPLICATE(insn_name, op_name, reg_src, size, min_vec_len)
1824 define(`REPLICATE', `
1825 instruct $1(vReg dst, $3 src) %{
1826   predicate(UseSVE > 0);
1827   match(Set dst ($2 src));
1828   ins_cost(SVE_COST);
1829   format %{ "sve_dup  $dst, $src\t# vector (sve) ($4)" %}
1830   ins_encode %{
1831     __ sve_dup(as_FloatRegister($dst$$reg), __ $4, as_Register($src$$reg));
1832   %}
1833   ins_pipe(pipe_slow);
1834 %}')dnl
1835 dnl
1836 dnl REPLICATE_IMM8($1,        $2,      $3,       $4,   $5         )
1837 dnl REPLICATE_IMM8(insn_name, op_name, imm_type, size, min_vec_len)
1838 define(`REPLICATE_IMM8', `
1839 instruct $1(vReg dst, $3 con) %{
1840   predicate(UseSVE > 0);
1841   match(Set dst ($2 con));
1842   ins_cost(SVE_COST);
1843   format %{ "sve_dup  $dst, $con\t# vector (sve) ($4)" %}
1844   ins_encode %{
1845     __ sve_dup(as_FloatRegister($dst$$reg), __ $4, $con$$constant);
1846   %}
1847   ins_pipe(pipe_slow);
1848 %}')dnl
1849 dnl
1850 dnl FREPLICATE($1,        $2,      $3,        $4)
1851 dnl FREPLICATE(insn_name, op_name, reg_src, size)
1852 define(`FREPLICATE', `
1853 instruct $1(vReg dst, $3 src) %{
1854   predicate(UseSVE > 0);
1855   match(Set dst ($2 src));
1856   ins_cost(SVE_COST);
1857   format %{ "sve_cpy  $dst, $src\t# vector (sve) ($4)" %}
1858   ins_encode %{
1859     __ sve_cpy(as_FloatRegister($dst$$reg), __ $4,
1860          ptrue, as_FloatRegister($src$$reg));
1861   %}
1862   ins_pipe(pipe_slow);
1863 %}')dnl
1864 
1865 // vector replicate
1866 REPLICATE(replicateB, ReplicateB, iRegIorL2I, B, 16)
1867 REPLICATE(replicateS, ReplicateS, iRegIorL2I, H, 8)
1868 REPLICATE(replicateI, ReplicateI, iRegIorL2I, S, 4)
1869 REPLICATE(replicateL, ReplicateL, iRegL,      D, 2)
1870 REPLICATE_IMM8(replicateB_imm8, ReplicateB, immI8,        B, 16)
1871 REPLICATE_IMM8(replicateS_imm8, ReplicateS, immI8_shift8, H, 8)
1872 REPLICATE_IMM8(replicateI_imm8, ReplicateI, immI8_shift8, S, 4)
1873 REPLICATE_IMM8(replicateL_imm8, ReplicateL, immL8_shift8, D, 2)
1874 FREPLICATE(replicateF, ReplicateF, vRegF, S, 4)
1875 FREPLICATE(replicateD, ReplicateD, vRegD, D, 2)
1876 dnl
1877 dnl VSHIFT_TRUE_PREDICATE($1,        $2,      $3,   $4,          $5  )
1878 dnl VSHIFT_TRUE_PREDICATE(insn_name, op_name, size, min_vec_len, insn)
1879 define(`VSHIFT_TRUE_PREDICATE', `
1880 instruct $1(vReg dst, vReg shift) %{
1881   predicate(UseSVE > 0);
1882   match(Set dst ($2 dst shift));
1883   ins_cost(SVE_COST);
1884   format %{ "$5 $dst, $dst, $shift\t# vector (sve) ($3)" %}
1885   ins_encode %{
1886     __ $5(as_FloatRegister($dst$$reg), __ $3,
1887          ptrue, as_FloatRegister($shift$$reg));
1888   %}
1889   ins_pipe(pipe_slow);
1890 %}')dnl
1891 dnl
1892 dnl VSHIFT_IMM_UNPREDICATED($1,        $2,      $3,       $4,   $5,          $6  )
1893 dnl VSHIFT_IMM_UNPREDICATED(insn_name, op_name, op_name2, size, min_vec_len, insn)
1894 define(`VSHIFT_IMM_UNPREDICATED', `
1895 instruct $1(vReg dst, vReg src, immI shift) %{
1896   predicate(UseSVE > 0);
1897   match(Set dst ($2 src ($3 shift)));
1898   ins_cost(SVE_COST);
1899   format %{ "$6 $dst, $src, $shift\t# vector (sve) ($4)" %}
1900   ins_encode %{
1901     int con = (int)$shift$$constant;dnl
1902 ifelse(eval(index(`$1', `vasr') == 0 || index(`$1', `vlsr') == 0), 1, `
1903     if (con == 0) {
1904       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
1905            as_FloatRegister($src$$reg));
1906       return;
1907     }')dnl
1908 ifelse(eval(index(`$1', `vasr') == 0), 1, `ifelse(eval(index(`$4', `B') == 0), 1, `
1909     if (con >= 8) con = 7;')ifelse(eval(index(`$4', `H') == 0), 1, `
1910     if (con >= 16) con = 15;')')dnl
1911 ifelse(eval(index(`$1', `vlsl') == 0  || index(`$1', `vlsr') == 0), 1, `ifelse(eval(index(`$4', `B') == 0), 1, `
1912     if (con >= 8) {
1913       __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
1914            as_FloatRegister($src$$reg));
1915       return;
1916     }')ifelse(eval(index(`$4', `H') == 0), 1, `
1917     if (con >= 16) {
1918       __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
1919            as_FloatRegister($src$$reg));
1920       return;
1921     }')')
1922     __ $6(as_FloatRegister($dst$$reg), __ $4,
1923          as_FloatRegister($src$$reg), con);
1924   %}
1925   ins_pipe(pipe_slow);
1926 %}')dnl
1927 dnl
1928 dnl VSHIFT_COUNT($1,        $2,   $3,          $4  )
1929 dnl VSHIFT_COUNT(insn_name, size, min_vec_len, type)
1930 define(`VSHIFT_COUNT', `
1931 instruct $1(vReg dst, iRegIorL2I cnt) %{
1932   predicate(UseSVE > 0 &&
1933             ELEMENT_SHORT_CHAR($4, n));
1934   match(Set dst (LShiftCntV cnt));
1935   match(Set dst (RShiftCntV cnt));
1936   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) ($2)" %}
1937   ins_encode %{
1938     __ sve_dup(as_FloatRegister($dst$$reg), __ $2, as_Register($cnt$$reg));
1939   %}
1940   ins_pipe(pipe_slow);
1941 %}')dnl
1942 
1943 // vector shift
1944 VSHIFT_TRUE_PREDICATE(vasrB, RShiftVB,  B, 16, sve_asr)
1945 VSHIFT_TRUE_PREDICATE(vasrS, RShiftVS,  H,  8, sve_asr)
1946 VSHIFT_TRUE_PREDICATE(vasrI, RShiftVI,  S,  4, sve_asr)
1947 VSHIFT_TRUE_PREDICATE(vasrL, RShiftVL,  D,  2, sve_asr)
1948 VSHIFT_TRUE_PREDICATE(vlslB, LShiftVB,  B, 16, sve_lsl)
1949 VSHIFT_TRUE_PREDICATE(vlslS, LShiftVS,  H,  8, sve_lsl)
1950 VSHIFT_TRUE_PREDICATE(vlslI, LShiftVI,  S,  4, sve_lsl)
1951 VSHIFT_TRUE_PREDICATE(vlslL, LShiftVL,  D,  2, sve_lsl)
1952 VSHIFT_TRUE_PREDICATE(vlsrB, URShiftVB, B, 16, sve_lsr)
1953 VSHIFT_TRUE_PREDICATE(vlsrS, URShiftVS, H,  8, sve_lsr)
1954 VSHIFT_TRUE_PREDICATE(vlsrI, URShiftVI, S,  4, sve_lsr)
1955 VSHIFT_TRUE_PREDICATE(vlsrL, URShiftVL, D,  2, sve_lsr)
1956 VSHIFT_IMM_UNPREDICATED(vasrB_imm, RShiftVB,  RShiftCntV, B, 16, sve_asr)
1957 VSHIFT_IMM_UNPREDICATED(vasrS_imm, RShiftVS,  RShiftCntV, H,  8, sve_asr)
1958 VSHIFT_IMM_UNPREDICATED(vasrI_imm, RShiftVI,  RShiftCntV, S,  4, sve_asr)
1959 VSHIFT_IMM_UNPREDICATED(vasrL_imm, RShiftVL,  RShiftCntV, D,  2, sve_asr)
1960 VSHIFT_IMM_UNPREDICATED(vlsrB_imm, URShiftVB, RShiftCntV, B, 16, sve_lsr)
1961 VSHIFT_IMM_UNPREDICATED(vlsrS_imm, URShiftVS, RShiftCntV, H,  8, sve_lsr)
1962 VSHIFT_IMM_UNPREDICATED(vlsrI_imm, URShiftVI, RShiftCntV, S,  4, sve_lsr)
1963 VSHIFT_IMM_UNPREDICATED(vlsrL_imm, URShiftVL, RShiftCntV, D,  2, sve_lsr)
1964 VSHIFT_IMM_UNPREDICATED(vlslB_imm, LShiftVB,  LShiftCntV, B, 16, sve_lsl)
1965 VSHIFT_IMM_UNPREDICATED(vlslS_imm, LShiftVS,  LShiftCntV, H,  8, sve_lsl)
1966 VSHIFT_IMM_UNPREDICATED(vlslI_imm, LShiftVI,  LShiftCntV, S,  4, sve_lsl)
1967 VSHIFT_IMM_UNPREDICATED(vlslL_imm, LShiftVL,  LShiftCntV, D,  2, sve_lsl)
1968 VSHIFT_COUNT(vshiftcntB, B, 16, T_BYTE)
1969 VSHIFT_COUNT(vshiftcntS, H,  8, T_SHORT)
1970 VSHIFT_COUNT(vshiftcntI, S,  4, T_INT)
1971 VSHIFT_COUNT(vshiftcntL, D,  2, T_LONG)
1972 
1973 // vector shift - predicated
1974 BINARY_OP_PREDICATE(vasrB, RShiftVB,  B, sve_asr)
1975 BINARY_OP_PREDICATE(vasrS, RShiftVS,  H, sve_asr)
1976 BINARY_OP_PREDICATE(vasrI, RShiftVI,  S, sve_asr)
1977 BINARY_OP_PREDICATE(vasrL, RShiftVL,  D, sve_asr)
1978 BINARY_OP_PREDICATE(vlslB, LShiftVB,  B, sve_lsl)
1979 BINARY_OP_PREDICATE(vlslS, LShiftVS,  H, sve_lsl)
1980 BINARY_OP_PREDICATE(vlslI, LShiftVI,  S, sve_lsl)
1981 BINARY_OP_PREDICATE(vlslL, LShiftVL,  D, sve_lsl)
1982 BINARY_OP_PREDICATE(vlsrB, URShiftVB, B, sve_lsr)
1983 BINARY_OP_PREDICATE(vlsrS, URShiftVS, H, sve_lsr)
1984 BINARY_OP_PREDICATE(vlsrI, URShiftVI, S, sve_lsr)
1985 BINARY_OP_PREDICATE(vlsrL, URShiftVL, D, sve_lsr)
1986 dnl
1987 dnl VSHIFT_IMM_PREDICATED($1,        $2,      $3,       $4,   $5,   $6  )
1988 dnl VSHIFT_IMM_PREDICATED(insn_name, op_name, op_name2, type, size, insn)
1989 define(`VSHIFT_IMM_PREDICATED', `
1990 instruct $1_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
1991   predicate(UseSVE > 0);
1992   match(Set dst_src ($2 (Binary dst_src ($3 shift)) pg));
1993   ins_cost(SVE_COST);
1994   format %{ "$6 $dst_src, $pg, $dst_src, $shift\t# vector (sve) ($4)" %}
1995   ins_encode %{
1996     int con = (int)$shift$$constant;
1997     assert(con ifelse(index(`$1', `vlsl'), 0, `>=', `>') 0 && con < $5, "invalid shift immediate");
1998     __ $6(as_FloatRegister($dst_src$$reg), __ $4, as_PRegister($pg$$reg), con);
1999   %}
2000   ins_pipe(pipe_slow);
2001 %}')dnl
2002 dnl
2003 VSHIFT_IMM_PREDICATED(vasrB, RShiftVB,  RShiftCntV, B, 8,  sve_asr)
2004 VSHIFT_IMM_PREDICATED(vasrS, RShiftVS,  RShiftCntV, H, 16, sve_asr)
2005 VSHIFT_IMM_PREDICATED(vasrI, RShiftVI,  RShiftCntV, S, 32, sve_asr)
2006 VSHIFT_IMM_PREDICATED(vasrL, RShiftVL,  RShiftCntV, D, 64, sve_asr)
2007 VSHIFT_IMM_PREDICATED(vlsrB, URShiftVB, RShiftCntV, B, 8,  sve_lsr)
2008 VSHIFT_IMM_PREDICATED(vlsrS, URShiftVS, RShiftCntV, H, 16, sve_lsr)
2009 VSHIFT_IMM_PREDICATED(vlsrI, URShiftVI, RShiftCntV, S, 32, sve_lsr)
2010 VSHIFT_IMM_PREDICATED(vlsrL, URShiftVL, RShiftCntV, D, 64, sve_lsr)
2011 VSHIFT_IMM_PREDICATED(vlslB, LShiftVB,  LShiftCntV, B, 8,  sve_lsl)
2012 VSHIFT_IMM_PREDICATED(vlslS, LShiftVS,  LShiftCntV, H, 16, sve_lsl)
2013 VSHIFT_IMM_PREDICATED(vlslI, LShiftVI,  LShiftCntV, S, 32, sve_lsl)
2014 VSHIFT_IMM_PREDICATED(vlslL, LShiftVL,  LShiftCntV, D, 64, sve_lsl)
2015 
2016 // vector sqrt
2017 UNARY_OP_TRUE_PREDICATE(vsqrtF, SqrtVF, S, sve_fsqrt)
2018 UNARY_OP_TRUE_PREDICATE(vsqrtD, SqrtVD, D, sve_fsqrt)
2019 
2020 // vector sqrt - predicated
2021 UNARY_OP_PREDICATE(vsqrtF, SqrtVF, S, sve_fsqrt)
2022 UNARY_OP_PREDICATE(vsqrtD, SqrtVD, D, sve_fsqrt)
2023 
2024 // vector sub
2025 BINARY_OP_UNPREDICATE(vsubB, SubVB, B, 16, sve_sub)
2026 BINARY_OP_UNPREDICATE(vsubS, SubVS, H, 8, sve_sub)
2027 BINARY_OP_UNPREDICATE(vsubI, SubVI, S, 4, sve_sub)
2028 BINARY_OP_UNPREDICATE(vsubL, SubVL, D, 2, sve_sub)
2029 BINARY_OP_UNPREDICATE(vsubF, SubVF, S, 4, sve_fsub)
2030 BINARY_OP_UNPREDICATE(vsubD, SubVD, D, 2, sve_fsub)
2031 
2032 // vector sub - predicated
2033 BINARY_OP_PREDICATE(vsubB, SubVB, B, sve_sub)
2034 BINARY_OP_PREDICATE(vsubS, SubVS, H, sve_sub)
2035 BINARY_OP_PREDICATE(vsubI, SubVI, S, sve_sub)
2036 BINARY_OP_PREDICATE(vsubL, SubVL, D, sve_sub)
2037 BINARY_OP_PREDICATE(vsubF, SubVF, S, sve_fsub)
2038 BINARY_OP_PREDICATE(vsubD, SubVD, D, sve_fsub)
2039 
2040 // ------------------------------ Vector mask cast --------------------------
2041 
2042 instruct vmaskcast(pRegGov dst_src) %{
2043   predicate(UseSVE > 0 &&
2044             n->bottom_type()->is_vect()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
2045             n->bottom_type()->is_vect()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
2046   match(Set dst_src (VectorMaskCast dst_src));
2047   ins_cost(0);
2048   format %{ "vmaskcast $dst_src\t# empty (sve)" %}
2049   ins_encode %{
2050     // empty
2051   %}
2052   ins_pipe(pipe_class_empty);
2053 %}
2054 
2055 instruct vmaskcast_extend(pRegGov dst, pReg src)
2056 %{
2057   predicate(UseSVE > 0 &&
2058             (Matcher::vector_length_in_bytes(n) == 2 * Matcher::vector_length_in_bytes(n->in(1)) ||
2059              Matcher::vector_length_in_bytes(n) == 4 * Matcher::vector_length_in_bytes(n->in(1)) ||
2060              Matcher::vector_length_in_bytes(n) == 8 * Matcher::vector_length_in_bytes(n->in(1))));
2061   match(Set dst (VectorMaskCast src));
2062   ins_cost(SVE_COST * 3);
2063   format %{ "sve_vmaskcast_extend  $dst, $src\t# extend predicate $src" %}
2064   ins_encode %{
2065     __ sve_vmaskcast_extend(as_PRegister($dst$$reg), as_PRegister($src$$reg),
2066                             Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
2067   %}
2068   ins_pipe(pipe_slow);
2069 %}
2070 
2071 instruct vmaskcast_narrow(pRegGov dst, pReg src)
2072 %{
2073   predicate(UseSVE > 0 &&
2074             (Matcher::vector_length_in_bytes(n) * 2 == Matcher::vector_length_in_bytes(n->in(1)) ||
2075              Matcher::vector_length_in_bytes(n) * 4 == Matcher::vector_length_in_bytes(n->in(1)) ||
2076              Matcher::vector_length_in_bytes(n) * 8 == Matcher::vector_length_in_bytes(n->in(1))));
2077   match(Set dst (VectorMaskCast src));
2078   ins_cost(SVE_COST * 3);
2079   format %{ "sve_vmaskcast_narrow  $dst, $src\t# narrow predicate $src" %}
2080   ins_encode %{
2081     __ sve_vmaskcast_narrow(as_PRegister($dst$$reg), as_PRegister($src$$reg),
2082                             Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
2083   %}
2084   ins_pipe(pipe_slow);
2085 %}
2086 dnl
2087 
2088 // ------------------------------ Vector cast -------------------------------
2089 dnl
2090 dnl
2091 define(`VECTOR_CAST_X2X', `
2092 instruct vcvt$1to$2`'(vReg dst, vReg src)
2093 %{
2094   predicate(UseSVE > 0 &&
2095             n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
2096   match(Set dst (VectorCast$1`'2X src));
2097   ins_cost(SVE_COST);
2098   format %{ "sve_vectorcast_$5  $dst, $src\t# convert $1 to $2 vector" %}
2099   ins_encode %{
2100     __ sve_$3(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src$$reg), __ $4);
2101   %}
2102   ins_pipe(pipe_slow);
2103 %}')dnl
2104 
2105 dnl
2106 dnl Start of vector cast rules
2107 dnl
2108 instruct vcvtBtoX_extend(vReg dst, vReg src)
2109 %{
2110   predicate(UseSVE > 0);
2111   match(Set dst (VectorCastB2X src));
2112   ins_cost(2 * SVE_COST);
2113   format %{ "sve_vectorcast_b2x  $dst, $src\t# convert B to X vector (extend)" %}
2114   ins_encode %{
2115     BasicType to_bt = Matcher::vector_element_basic_type(this);
2116     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2117     __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ B);
2118     if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
2119       __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
2120     }
2121   %}
2122   ins_pipe(pipe_slow);
2123 %}
2124 
2125 instruct vcvtStoB(vReg dst, vReg src, vReg tmp)
2126 %{
2127   predicate(UseSVE > 0 &&
2128             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2129   match(Set dst (VectorCastS2X src));
2130   effect(TEMP tmp);
2131   ins_cost(2 * SVE_COST);
2132   format %{ "sve_vectorcast_s2b  $dst, $src\t# convert H to B vector" %}
2133   ins_encode %{
2134     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
2135                          as_FloatRegister($src$$reg), __ H, as_FloatRegister($tmp$$reg));
2136   %}
2137   ins_pipe(pipe_slow);
2138 %}
2139 
2140 instruct vcvtStoX_extend(vReg dst, vReg src)
2141 %{
2142   predicate(UseSVE > 0 &&
2143             type2aelembytes(Matcher::vector_element_basic_type(n)) > 2);
2144   match(Set dst (VectorCastS2X src));
2145   ins_cost(2 * SVE_COST);
2146   format %{ "sve_vectorcast_s2x  $dst, $src\t# convert H to X vector (extend)" %}
2147   ins_encode %{
2148     BasicType to_bt = Matcher::vector_element_basic_type(this);
2149     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2150     __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ H);
2151     if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
2152       __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
2153     }
2154   %}
2155   ins_pipe(pipe_slow);
2156 %}
2157 
2158 instruct vcvtItoB(vReg dst, vReg src, vReg tmp)
2159 %{
2160   predicate(UseSVE > 0 &&
2161             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2162   match(Set dst (VectorCastI2X src));
2163   effect(TEMP_DEF dst, TEMP tmp);
2164   ins_cost(3 * SVE_COST);
2165   format %{ "sve_vectorcast_i2b  $dst, $src\t# convert I to B vector" %}
2166   ins_encode %{
2167     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
2168                          as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
2169   %}
2170   ins_pipe(pipe_slow);
2171 %}
2172 
2173 instruct vcvtItoS(vReg dst, vReg src, vReg tmp)
2174 %{
2175   predicate(UseSVE > 0 &&
2176             n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
2177   match(Set dst (VectorCastI2X src));
2178   effect(TEMP tmp);
2179   ins_cost(2 * SVE_COST);
2180   format %{ "sve_vectorcast_i2s $dst, $src\t# convert I to H vector" %}
2181   ins_encode %{
2182     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ H,
2183                          as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
2184   %}
2185   ins_pipe(pipe_slow);
2186 %}
2187 
2188 instruct vcvtItoL(vReg dst, vReg src)
2189 %{
2190   predicate(UseSVE > 0 &&
2191             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2192   match(Set dst (VectorCastI2X src));
2193   ins_cost(SVE_COST);
2194   format %{ "sve_vectorcast_i2l  $dst, $src\t# convert I to L vector" %}
2195   ins_encode %{
2196     __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
2197   %}
2198   ins_pipe(pipe_slow);
2199 %}
2200 dnl
2201 dnl vcvtItoF
2202 VECTOR_CAST_X2X(I, F, scvtf, S, i2f)
2203 
2204 instruct vcvtItoD(vReg dst, vReg src)
2205 %{
2206   predicate(UseSVE > 0 &&
2207             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
2208   match(Set dst (VectorCastI2X src));
2209   ins_cost(2 * SVE_COST);
2210   format %{ "sve_vectorcast_i2d  $dst, $src\t# convert I to D vector" %}
2211   ins_encode %{
2212     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
2213     __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ D);
2214   %}
2215   ins_pipe(pipe_slow);
2216 %}
2217 
2218 instruct vcvtLtoX_narrow(vReg dst, vReg src, vReg tmp)
2219 %{
2220   predicate(UseSVE > 0 && is_integral_type(Matcher::vector_element_basic_type(n)));
2221   match(Set dst (VectorCastL2X src));
2222   effect(TEMP_DEF dst, TEMP tmp);
2223   ins_cost(2 * SVE_COST);
2224   format %{ "sve_vectorcast_l2x  $dst, $src\t# convert L to B/H/S vector (narrow)" %}
2225   ins_encode %{
2226     BasicType to_bt = Matcher::vector_element_basic_type(this);
2227     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2228     __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
2229                          as_FloatRegister($src$$reg), __ D, as_FloatRegister($tmp$$reg));
2230   %}
2231   ins_pipe(pipe_slow);
2232 %}
2233 
2234 instruct vcvtLtoF(vReg dst, vReg src, vReg tmp)
2235 %{
2236   predicate(UseSVE > 0 &&
2237             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2238   match(Set dst (VectorCastL2X src));
2239   effect(TEMP_DEF dst, TEMP tmp);
2240   ins_cost(3 * SVE_COST);
2241   format %{ "sve_vectorcast_l2f  $dst, $src\t# convert L to F vector" %}
2242   ins_encode %{
2243     __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
2244     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
2245                          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
2246 
2247   %}
2248   ins_pipe(pipe_slow);
2249 %}
2250 dnl
2251 dnl vcvtLtoD
2252 VECTOR_CAST_X2X(L, D, scvtf, D, l2d)
2253 
2254 instruct vcvtFtoX_narrow(vReg dst, vReg src, vReg tmp)
2255 %{
2256   predicate(UseSVE > 0 &&
2257             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2258              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT));
2259   match(Set dst (VectorCastF2X src));
2260   effect(TEMP_DEF dst, TEMP tmp);
2261   ins_cost(3 * SVE_COST);
2262   format %{ "sve_vectorcast_f2x  $dst, $src\t# convert F to B/H vector" %}
2263   ins_encode %{
2264     BasicType to_bt = Matcher::vector_element_basic_type(this);
2265     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2266     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
2267     __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
2268                          as_FloatRegister($dst$$reg), __ S, as_FloatRegister($tmp$$reg));
2269   %}
2270   ins_pipe(pipe_slow);
2271 %}
2272 
2273 instruct vcvtFtoI(vReg dst, vReg src)
2274 %{
2275   predicate(UseSVE > 0 &&
2276             (n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2277   match(Set dst (VectorCastF2X src));
2278   ins_cost(SVE_COST);
2279   format %{ "sve_vectorcast_f2x  $dst, $src\t# convert F to I vector" %}
2280   ins_encode %{
2281     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
2282   %}
2283   ins_pipe(pipe_slow);
2284 %}
2285 
2286 instruct vcvtFtoL(vReg dst, vReg src)
2287 %{
2288   predicate(UseSVE > 0 &&
2289             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
2290   match(Set dst (VectorCastF2X src));
2291   ins_cost(SVE_COST * 2);
2292   format %{ "sve_vectorcast_f2x  $dst, $src\t# convert F to L vector" %}
2293   ins_encode %{
2294     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
2295     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);
2296   %}
2297   ins_pipe(pipe_slow);
2298 %}
2299 
2300 instruct vcvtFtoD(vReg dst, vReg src)
2301 %{
2302   predicate(UseSVE > 0 &&
2303             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
2304   match(Set dst (VectorCastF2X src));
2305   ins_cost(2 * SVE_COST);
2306   format %{ "sve_vectorcast_f2d  $dst, $dst\t# convert F to D vector" %}
2307   ins_encode %{
2308     __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
2309     __ sve_fcvt(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);
2310   %}
2311   ins_pipe(pipe_slow);
2312 %}
2313 
2314 instruct vcvtDtoX_narrow(vReg dst, vReg src, vReg tmp)
2315 %{
2316   predicate(UseSVE > 0 &&
2317             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2318              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2319              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2320   match(Set dst (VectorCastD2X src));
2321   effect(TEMP_DEF dst, TEMP tmp);
2322   ins_cost(3 * SVE_COST);
2323   format %{ "sve_vectorcast_d2x  $dst, $src\t# convert D to X vector (narrow)" %}
2324   ins_encode %{
2325     BasicType to_bt = Matcher::vector_element_basic_type(this);
2326     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2327     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
2328     __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
2329                          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
2330   %}
2331   ins_pipe(pipe_slow);
2332 %}
2333 dnl
2334 dnl vcvtDtoL
2335 VECTOR_CAST_X2X(D, L, fcvtzs, D, d2l)
2336 
2337 instruct vcvtDtoF(vReg dst, vReg src, vReg tmp)
2338 %{
2339   predicate(UseSVE > 0 &&
2340             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2341   match(Set dst (VectorCastD2X src));
2342   effect(TEMP_DEF dst, TEMP tmp);
2343   ins_cost(3 * SVE_COST);
2344   format %{ "sve_vectorcast_d2f  $dst, S, $dst\t# convert D to F vector" %}
2345   ins_encode %{
2346     __ sve_fcvt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
2347     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
2348                          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
2349   %}
2350   ins_pipe(pipe_slow);
2351 %}
2352 
2353 dnl
2354 dnl
2355 // ------------------------------ Vector extract ---------------------------------
2356 dnl
2357 define(`VECTOR_EXTRACT_I', `
2358 instruct extract$1`'($3 dst, vReg src, immI idx, vReg vtmp)
2359 %{
2360   predicate(UseSVE > 0 && n->in(2)->get_int() >= $2);
2361   match(Set dst (Extract$1 src idx));
2362   effect(TEMP vtmp);
2363   ins_cost(2 * SVE_COST);
2364   format %{ "sve_extract_integral $dst, $4, $src, $idx\t# extract from vector($1)" %}
2365   ins_encode %{
2366     __ sve_extract_integral(as_Register($dst$$reg), __ $4, as_FloatRegister($src$$reg),
2367                             (int)($idx$$constant), /* is_signed */ ifelse($1, L, false, true), as_FloatRegister($vtmp$$reg));
2368   %}
2369   ins_pipe(pipe_slow);
2370 %}')dnl
2371 dnl              $1 $2  $3         $4
2372 VECTOR_EXTRACT_I(B, 16, iRegINoSp, B)
2373 VECTOR_EXTRACT_I(S, 8,  iRegINoSp, H)
2374 VECTOR_EXTRACT_I(I, 4,  iRegINoSp, S)
2375 VECTOR_EXTRACT_I(L, 2,  iRegLNoSp, D)
2376 dnl
2377 define(`VECTOR_EXTRACT_I_LT', `
2378 instruct extract$1_LT$2`'($3 dst, vReg src, immI idx)
2379 %{
2380   predicate(UseSVE > 0 && n->in(2)->get_int() < $2);
2381   match(Set dst (Extract$1 src idx));
2382   ins_cost(INSN_COST);
2383   format %{ "ifelse($4, D, umov, smov) $dst, $4, $src, $idx\t# extract from vector($1)" %}
2384   ins_encode %{
2385     __ ifelse($4, D, umov, smov)(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ $4, $idx$$constant);
2386   %}
2387   ins_pipe(pipe_class_default);
2388 %}')dnl
2389 dnl                 $1  $2  $3         $4
2390 VECTOR_EXTRACT_I_LT(B,  16, iRegINoSp, B)
2391 VECTOR_EXTRACT_I_LT(S,  8,  iRegINoSp, H)
2392 VECTOR_EXTRACT_I_LT(I,  4,  iRegINoSp, S)
2393 VECTOR_EXTRACT_I_LT(L,  2,  iRegLNoSp, D)
2394 
2395 instruct extractF(vRegF dst, vReg src, immI idx)
2396 %{
2397   predicate(UseSVE > 0);
2398   match(Set dst (ExtractF src idx));
2399   ins_cost(2 * SVE_COST);
2400   format %{ "sve_extract_f $dst, S, $src, $idx\t# extract from vector(F)" %}
2401   ins_encode %{
2402     if ((as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg)) && ($idx$$constant == 0)) {
2403       /* empty */
2404     } else if ($idx$$constant == 0) {
2405       __ fmovs(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
2406     } else if ($idx$$constant < 4) {
2407       __ ins(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($src$$reg), 0, (int)($idx$$constant));
2408     } else {
2409       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2410       __ sve_ext(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), $idx$$constant << 2);
2411     }
2412   %}
2413   ins_pipe(pipe_slow);
2414 %}
2415 
2416 instruct extractD(vRegD dst, vReg src, immI idx)
2417 %{
2418   predicate(UseSVE > 0);
2419   match(Set dst (ExtractD src idx));
2420   ins_cost(2 * SVE_COST);
2421   format %{ "sve_extract_d $dst, D, $src, $idx\t# extract from vector(D)" %}
2422   ins_encode %{
2423     if ((as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg)) && ($idx$$constant == 0)) {
2424       /* empty */
2425     } else if ($idx$$constant == 0) {
2426       __ fmovd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
2427     } else if ($idx$$constant == 1) {
2428       __ ins(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), 0, 1);
2429     } else {
2430       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2431       __ sve_ext(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), $idx$$constant << 3);
2432     }
2433   %}
2434   ins_pipe(pipe_slow);
2435 %}
2436 
2437 // ------------------------------- VectorTest ----------------------------------
2438 
2439 instruct vtest_alltrue(iRegINoSp dst, pRegGov src1, pRegGov src2, pReg ptmp, rFlagsReg cr)
2440 %{
2441   predicate(UseSVE > 0 &&
2442             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
2443   match(Set dst (VectorTest src1 src2));
2444   effect(TEMP ptmp, KILL cr);
2445   ins_cost(SVE_COST);
2446   format %{ "sve_eors $ptmp, $src1, $src2\t# $src2 is all true mask\n"
2447             "csetw $dst, EQ\t# VectorTest (sve) - alltrue" %}
2448   ins_encode %{
2449     __ sve_eors(as_PRegister($ptmp$$reg), ptrue,
2450                 as_PRegister($src1$$reg), as_PRegister($src2$$reg));
2451     __ csetw(as_Register($dst$$reg), Assembler::EQ);
2452   %}
2453   ins_pipe(pipe_slow);
2454 %}
2455 
2456 instruct vtest_anytrue(iRegINoSp dst, pRegGov src1, pRegGov src2, rFlagsReg cr)
2457 %{
2458   predicate(UseSVE > 0 &&
2459             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
2460   match(Set dst (VectorTest src1 src2));
2461   effect(KILL cr);
2462   ins_cost(SVE_COST);
2463   format %{ "sve_ptest $src1\n\t"
2464             "csetw $dst, NE\t# VectorTest (sve) - anytrue" %}
2465   ins_encode %{
2466     // "src2" is not used for sve.
2467     __ sve_ptest(ptrue, as_PRegister($src1$$reg));
2468     __ csetw(as_Register($dst$$reg), Assembler::NE);
2469   %}
2470   ins_pipe(pipe_slow);
2471 %}
2472 dnl
2473 // ------------------------------ Vector insert ---------------------------------
2474 
2475 instruct insertI_le128bits(vReg dst, vReg src, iRegIorL2I val, immI idx) %{
2476   predicate(UseSVE > 0 &&
2477             (Matcher::vector_length_in_bytes(n) == 8 || Matcher::vector_length_in_bytes(n) == 16) &&
2478             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2479              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2480              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2481   match(Set dst (VectorInsert (Binary src val) idx));
2482   ins_cost(2 * INSN_COST);
2483   format %{ "orr    $dst, T8/16B, $src, $src\n\t"
2484             "mov    $dst, B/H/S, $idx, $val\t# insertI into vector(64/128bits)" %}
2485   ins_encode %{
2486     BasicType bt = Matcher::vector_element_basic_type(this);
2487     if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
2488       __ orr(as_FloatRegister($dst$$reg), Matcher::vector_length_in_bytes(this) == 8 ? __ T8B : __ T16B,
2489              as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2490     }
2491     __ mov(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(Matcher::vector_element_basic_type(this)),
2492            $idx$$constant, $val$$Register);
2493   %}
2494   ins_pipe(pipe_slow);
2495 %}
2496 
2497 instruct insertI_small_index(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg vtmp, pRegGov pgtmp, rFlagsReg cr) %{
2498   predicate(UseSVE > 0 && n->in(2)->get_int() < 32 &&
2499             Matcher::vector_length_in_bytes(n) > 16 &&
2500             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2501              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2502              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2503   match(Set dst (VectorInsert (Binary src val) idx));
2504   effect(TEMP vtmp, TEMP pgtmp, KILL cr);
2505   ins_cost(4 * SVE_COST);
2506   format %{ "sve_index $vtmp, -16, 1\t# (B/H/S)\n\t"
2507             "sve_cmpeq $pgtmp, $vtmp, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2508             "sve_orr $dst, $src, $src\n\t"
2509             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
2510   ins_encode %{
2511     BasicType bt = Matcher::vector_element_basic_type(this, $src);
2512     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2513     __ block_comment("insert into vector (B/H/S) {");
2514       __ sve_index(as_FloatRegister($vtmp$$reg), size, -16, 1);
2515       __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
2516                  as_FloatRegister($vtmp$$reg), (int)($idx$$constant) - 16);
2517       if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
2518         __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2519       }
2520       __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), $val$$Register);
2521     __ block_comment("} insert into vector (B/H/S)");
2522   %}
2523   ins_pipe(pipe_slow);
2524 %}
2525 
2526 instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg vtmp1, vReg vtmp2, pRegGov pgtmp, rFlagsReg cr) %{
2527   predicate(UseSVE > 0 && n->in(2)->get_int() >= 32 &&
2528             Matcher::vector_length_in_bytes(n) > 16 &&
2529             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2530              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2531              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2532   match(Set dst (VectorInsert (Binary src val) idx));
2533   effect(TEMP vtmp1, TEMP vtmp2, TEMP pgtmp, KILL cr);
2534   ins_cost(5 * SVE_COST);
2535   format %{ "sve_index $vtmp1, 0, 1\t# (B/H/S)\n\t"
2536             "sve_dup $vtmp2, $idx\t# (B/H/S)\n\t"
2537             "sve_cmpeq $pgtmp, $vtmp1, $vtmp2\n\t"
2538             "sve_orr $dst, $src, $src\n\t"
2539             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
2540   ins_encode %{
2541     BasicType bt = Matcher::vector_element_basic_type(this, $src);
2542     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2543     __ block_comment("insert into vector (B/H/S) {");
2544       __ sve_index(as_FloatRegister($vtmp1$$reg), size, 0, 1);
2545       __ sve_dup(as_FloatRegister($vtmp2$$reg), size, (int)($idx$$constant));
2546       __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
2547                  as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg));
2548       if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
2549         __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2550       }
2551       __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), $val$$Register);
2552     __ block_comment("} insert into vector (B/H/S)");
2553   %}
2554   ins_pipe(pipe_slow);
2555 %}
2556 
2557 instruct insertL_128bits(vReg dst, vReg src, iRegL val, immI idx) %{
2558   predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) == 16 &&
2559             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2560   match(Set dst (VectorInsert (Binary src val) idx));
2561   ins_cost(2 * INSN_COST);
2562   format %{ "orr    $dst, T16B, $src, $src\n\t"
2563             "mov    $dst, D, $idx, $val\t# insertL into vector(128bits)" %}
2564   ins_encode %{
2565     BasicType bt = Matcher::vector_element_basic_type(this);
2566     if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
2567       __ orr(as_FloatRegister($dst$$reg), __ T16B,
2568              as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2569     }
2570     __ mov(as_FloatRegister($dst$$reg), __ D, $idx$$constant, $val$$Register);
2571   %}
2572   ins_pipe(pipe_slow);
2573 %}
2574 
2575 instruct insertL(vReg dst, vReg src, iRegL val, immI idx, vReg vtmp, pRegGov pgtmp, rFlagsReg cr) %{
2576   predicate(UseSVE > 0 &&
2577             Matcher::vector_length_in_bytes(n) > 16 &&
2578             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2579   match(Set dst (VectorInsert (Binary src val) idx));
2580   effect(TEMP vtmp, TEMP pgtmp, KILL cr);
2581   ins_cost(4 * SVE_COST);
2582   format %{ "sve_index $vtmp, D, -16, 1\n\t"
2583             "sve_cmpeq $pgtmp, $vtmp, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2584             "sve_orr $dst, $src, $src\n\t"
2585             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (L)" %}
2586   ins_encode %{
2587     __ block_comment("insert into vector (L) {");
2588       __ sve_index(as_FloatRegister($vtmp$$reg), __ D, -16, 1);
2589       __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
2590                  as_FloatRegister($vtmp$$reg), (int)($idx$$constant) - 16);
2591       if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
2592         __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2593       }
2594       __ sve_cpy(as_FloatRegister($dst$$reg), __ D,
2595                  as_PRegister($pgtmp$$reg), $val$$Register);
2596     __ block_comment("} insert into vector (L)");
2597   %}
2598   ins_pipe(pipe_slow);
2599 %}
2600 
2601 instruct insertF_le128bits(vReg dst, vReg src, vRegF val, immI idx) %{
2602   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
2603             (Matcher::vector_length_in_bytes(n) == 8 || Matcher::vector_length_in_bytes(n) == 16));
2604   match(Set dst (VectorInsert (Binary src val) idx));
2605   ins_cost(2 * INSN_COST);
2606   effect(TEMP_DEF dst);
2607   format %{ "orr    $dst, T8/16B, $src, $src\n\t"
2608             "ins    $dst, S, $val, $idx, 0\t# insertF into vector(64/128bits)" %}
2609   ins_encode %{
2610     __ orr(as_FloatRegister($dst$$reg), Matcher::vector_length_in_bytes(this) == 8 ? __ T8B : __ T16B,
2611            as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2612     __ ins(as_FloatRegister($dst$$reg), __ S,
2613            as_FloatRegister($val$$reg), $idx$$constant, 0);
2614   %}
2615   ins_pipe(pipe_slow);
2616 %}
2617 
2618 instruct insertF_small_index(vReg dst, vReg src, vRegF val, immI idx, pRegGov pgtmp, rFlagsReg cr) %{
2619   predicate(UseSVE > 0 && n->in(2)->get_int() < 32 &&
2620             Matcher::vector_length_in_bytes(n) > 16 &&
2621             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2622   match(Set dst (VectorInsert (Binary src val) idx));
2623   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
2624   ins_cost(4 * SVE_COST);
2625   format %{ "sve_index $dst, S, -16, 1\n\t"
2626             "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2627             "sve_orr $dst, $src, $src\n\t"
2628             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
2629   ins_encode %{
2630     __ block_comment("insert into vector (F) {");
2631       __ sve_index(as_FloatRegister($dst$$reg), __ S, -16, 1);
2632       __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
2633                  as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2634       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2635       __ sve_cpy(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
2636     __ block_comment("} insert into vector (F)");
2637   %}
2638   ins_pipe(pipe_slow);
2639 %}
2640 
2641 instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr) %{
2642   predicate(UseSVE > 0 && n->in(2)->get_int() >= 32 &&
2643             Matcher::vector_length_in_bytes(n) > 16 &&
2644             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2645   match(Set dst (VectorInsert (Binary src val) idx));
2646   effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
2647   ins_cost(5 * SVE_COST);
2648   format %{ "sve_index $tmp1, S, 0, 1\n\t"
2649             "sve_dup $dst, S, $idx\n\t"
2650             "sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
2651             "sve_orr $dst, $src, $src\n\t"
2652             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
2653   ins_encode %{
2654     __ block_comment("insert into vector (F) {");
2655       __ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
2656       __ sve_dup(as_FloatRegister($dst$$reg), __ S, (int)($idx$$constant));
2657       __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
2658                  as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
2659       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
2660                  as_FloatRegister($src$$reg));
2661       __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
2662                  as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
2663     __ block_comment("} insert into vector (F)");
2664   %}
2665   ins_pipe(pipe_slow);
2666 %}
2667 
2668 instruct insertD_128bits(vReg dst, vReg src, vRegD val, immI idx) %{
2669   predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) == 16 &&
2670             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
2671   match(Set dst (VectorInsert (Binary src val) idx));
2672   ins_cost(2 * INSN_COST);
2673   effect(TEMP_DEF dst);
2674   format %{ "orr    $dst, T16B, $src, $src\n\t"
2675             "ins    $dst, D, $val, $idx, 0\t# insertD into vector(128bits)" %}
2676   ins_encode %{
2677     __ orr(as_FloatRegister($dst$$reg), __ T16B,
2678            as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2679     __ ins(as_FloatRegister($dst$$reg), __ D,
2680            as_FloatRegister($val$$reg), $idx$$constant, 0);
2681   %}
2682   ins_pipe(pipe_slow);
2683 %}
2684 
2685 instruct insertD(vReg dst, vReg src, vRegD val, immI idx, pRegGov pgtmp, rFlagsReg cr) %{
2686   predicate(UseSVE > 0 &&
2687             Matcher::vector_length_in_bytes(n) > 16 &&
2688             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
2689   match(Set dst (VectorInsert (Binary src val) idx));
2690   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
2691   ins_cost(4 * SVE_COST);
2692   format %{ "sve_index $dst, D, -16, 1\n\t"
2693             "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2694             "sve_orr $dst, $src, $src\n\t"
2695             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (D)" %}
2696   ins_encode %{
2697     __ block_comment("insert into vector (D) {");
2698       __ sve_index(as_FloatRegister($dst$$reg), __ D, -16, 1);
2699       __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
2700                  as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2701       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
2702                  as_FloatRegister($src$$reg));
2703       __ sve_cpy(as_FloatRegister($dst$$reg), __ D,
2704                  as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
2705     __ block_comment("} insert into vector (D)");
2706   %}
2707   ins_pipe(pipe_slow);
2708 %}
2709 
2710 // ------------------------------ Vector shuffle -------------------------------
2711 
2712 instruct loadshuffle(vReg dst, vReg src) %{
2713   predicate(UseSVE > 0);
2714   match(Set dst (VectorLoadShuffle src));
2715   ins_cost(SVE_COST);
2716   format %{ "sve_loadshuffle $dst, $src\t# vector load shuffle (B/H/S/D)" %}
2717   ins_encode %{
2718     BasicType bt = Matcher::vector_element_basic_type(this);
2719     if (bt == T_BYTE) {
2720       if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
2721         __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
2722                    as_FloatRegister($src$$reg));
2723       }
2724     } else {
2725       __ sve_vector_extend(as_FloatRegister($dst$$reg),  __ elemType_to_regVariant(bt),
2726                            as_FloatRegister($src$$reg), __ B);
2727     }
2728   %}
2729   ins_pipe(pipe_slow);
2730 %}
2731 
2732 // ------------------------------ Vector rearrange -------------------------------
2733 
2734 instruct rearrange(vReg dst, vReg src, vReg shuffle)
2735 %{
2736   predicate(UseSVE > 0);
2737   match(Set dst (VectorRearrange src shuffle));
2738   ins_cost(SVE_COST);
2739   format %{ "sve_tbl $dst, $src, $shuffle\t# vector rearrange" %}
2740   ins_encode %{
2741     BasicType bt = Matcher::vector_element_basic_type(this, $src);
2742     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2743     __ sve_tbl(as_FloatRegister($dst$$reg), size,
2744                as_FloatRegister($src$$reg), as_FloatRegister($shuffle$$reg));
2745   %}
2746   ins_pipe(pipe_slow);
2747 %}
2748 
2749 // ------------------------------ Vector Load Gather ---------------------------------
2750 
2751 instruct gatherI(vReg dst, indirect mem, vReg idx) %{
2752   predicate(UseSVE > 0 &&
2753             n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
2754             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2755              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2756   match(Set dst (LoadVectorGather mem idx));
2757   ins_cost(SVE_COST);
2758   format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (S)" %}
2759   ins_encode %{
2760     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), ptrue,
2761                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2762   %}
2763   ins_pipe(pipe_slow);
2764 %}
2765 
2766 instruct gatherL(vReg dst, indirect mem, vReg idx) %{
2767   predicate(UseSVE > 0 &&
2768             n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
2769             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2770              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2771   match(Set dst (LoadVectorGather mem idx));
2772   ins_cost(2 * SVE_COST);
2773   format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (D)" %}
2774   ins_encode %{
2775     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2776     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), ptrue, as_Register($mem$$base),
2777                        as_FloatRegister($idx$$reg));
2778   %}
2779   ins_pipe(pipe_slow);
2780 %}
2781 
2782 // ------------------------------ Vector Load Gather Partial-------------------------------
2783 
2784 instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2785   predicate(UseSVE > 0 &&
2786             n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
2787             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2788              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2789   match(Set dst (LoadVectorGather mem idx));
2790   effect(TEMP ptmp, KILL cr);
2791   ins_cost(2 * SVE_COST + INSN_COST);
2792   format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (S)" %}
2793   ins_encode %{
2794     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this));
2795     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
2796                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2797   %}
2798   ins_pipe(pipe_slow);
2799 %}
2800 
2801 instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2802   predicate(UseSVE > 0 &&
2803             n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
2804             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2805              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2806   match(Set dst (LoadVectorGather mem idx));
2807   effect(TEMP ptmp, KILL cr);
2808   ins_cost(3 * SVE_COST + INSN_COST);
2809   format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (D)" %}
2810   ins_encode %{
2811     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this));
2812     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2813     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
2814                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2815   %}
2816   ins_pipe(pipe_slow);
2817 %}
2818 
2819 // ------------------------------ Vector Load Gather Predicated -------------------------------
2820 
2821 instruct gatherI_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
2822   predicate(UseSVE > 0 &&
2823             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2824              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2825   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
2826   ins_cost(SVE_COST);
2827   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (S)" %}
2828   ins_encode %{
2829     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
2830                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2831   %}
2832   ins_pipe(pipe_slow);
2833 %}
2834 
2835 instruct gatherL_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
2836   predicate(UseSVE > 0 &&
2837             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2838              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2839   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
2840   ins_cost(2 * SVE_COST);
2841   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (D)" %}
2842   ins_encode %{
2843     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2844     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
2845                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2846   %}
2847   ins_pipe(pipe_slow);
2848 %}
2849 
2850 // ------------------------------ Vector Store Scatter -------------------------------
2851 
2852 instruct scatterI(indirect mem, vReg src, vReg idx) %{
2853   predicate(UseSVE > 0 &&
2854             n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
2855             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2856              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2857   match(Set mem (StoreVectorScatter mem (Binary src idx)));
2858   ins_cost(SVE_COST);
2859   format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (S)" %}
2860   ins_encode %{
2861     __ sve_st1w_scatter(as_FloatRegister($src$$reg), ptrue,
2862                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2863   %}
2864   ins_pipe(pipe_slow);
2865 %}
2866 
2867 instruct scatterL(indirect mem, vReg src, vReg idx) %{
2868   predicate(UseSVE > 0 &&
2869             n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
2870             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2871              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2872   match(Set mem (StoreVectorScatter mem (Binary src idx)));
2873   ins_cost(2 * SVE_COST);
2874   format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (D)" %}
2875   ins_encode %{
2876     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2877     __ sve_st1d_scatter(as_FloatRegister($src$$reg), ptrue,
2878                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2879   %}
2880   ins_pipe(pipe_slow);
2881 %}
2882 
2883 // ------------------------------ Vector Store Scatter Partial -------------------------------
2884 
2885 instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2886   predicate(UseSVE > 0 &&
2887             n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
2888             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2889              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2890   match(Set mem (StoreVectorScatter mem (Binary src idx)));
2891   effect(TEMP ptmp, KILL cr);
2892   ins_cost(2 * SVE_COST + INSN_COST);
2893   format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (S)" %}
2894   ins_encode %{
2895     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src));
2896     __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
2897                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2898   %}
2899   ins_pipe(pipe_slow);
2900 %}
2901 
2902 instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2903   predicate(UseSVE > 0 &&
2904             n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
2905             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2906              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2907   match(Set mem (StoreVectorScatter mem (Binary src idx)));
2908   effect(TEMP ptmp, KILL cr);
2909   ins_cost(3 * SVE_COST + INSN_COST);
2910   format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (D)" %}
2911   ins_encode %{
2912     __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src));
2913     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2914     __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
2915                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2916   %}
2917   ins_pipe(pipe_slow);
2918 %}
2919 
2920 // ------------------------------ Vector Store Scatter Predicated -------------------------------
2921 
2922 instruct scatterI_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
2923   predicate(UseSVE > 0 &&
2924             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2925              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2926   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
2927   ins_cost(SVE_COST);
2928   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicate (S)" %}
2929   ins_encode %{
2930     __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
2931                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2932   %}
2933   ins_pipe(pipe_slow);
2934 %}
2935 
2936 instruct scatterL_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
2937   predicate(UseSVE > 0 &&
2938             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2939              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2940   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
2941   ins_cost(2 * SVE_COST);
2942   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated (D)" %}
2943   ins_encode %{
2944     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2945     __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
2946                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2947   %}
2948   ins_pipe(pipe_slow);
2949 %}
2950 
2951 // ------------------------------ Vector Load Const -------------------------------
2952 
2953 instruct loadconB(vReg dst, immI0 src) %{
2954   predicate(UseSVE > 0 &&
2955             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2956   match(Set dst (VectorLoadConst src));
2957   ins_cost(SVE_COST);
2958   format %{ "sve_index $dst, 0, 1\t# generate iota indices" %}
2959   ins_encode %{
2960     __ sve_index(as_FloatRegister($dst$$reg), __ B, 0, 1);
2961   %}
2962   ins_pipe(pipe_slow);
2963 %}
2964 
2965 // Intrisics for String.indexOf(char)
2966 
2967 dnl
2968 define(`STRING_INDEXOF_CHAR', `
2969 instruct string$1_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
2970                                   iRegI_R0 result, vReg ztmp1, vReg ztmp2,
2971                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
2972 %{
2973   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
2974   predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::$1));
2975   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
2976 
2977   format %{ "String$2 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
2978 
2979   ins_encode %{
2980     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
2981                                as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
2982                                as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), $3 /* isL */);
2983   %}
2984   ins_pipe(pipe_class_memory);
2985 %}')dnl
2986 dnl                 $1 $2      $3
2987 STRING_INDEXOF_CHAR(L, Latin1, true)
2988 STRING_INDEXOF_CHAR(U, UTF16,  false)
2989 
2990 // ---------------------------- Vector mask reductions ---------------------------
2991 instruct vmask_truecount(iRegINoSp dst, pReg src) %{
2992   predicate(UseSVE > 0);
2993   match(Set dst (VectorMaskTrueCount src));
2994   ins_cost(SVE_COST);
2995   format %{ "vmask_truecount $dst, $src\t# vector mask truecount (sve)" %}
2996   ins_encode %{
2997     BasicType bt = Matcher::vector_element_basic_type(this, $src);
2998     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2999     __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($src$$reg));
3000   %}
3001   ins_pipe(pipe_slow);
3002 %}
3003 
3004 // Return the index of the first mask lane that is set, or vector length if none of
3005 // them are set.
3006 instruct vmask_firsttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
3007   predicate(UseSVE > 0);
3008   match(Set dst (VectorMaskFirstTrue src));
3009   effect(TEMP ptmp);
3010   ins_cost(3 * SVE_COST);
3011   format %{ "vmask_firsttrue $dst, $src\t# vector mask firsttrue (sve)" %}
3012   ins_encode %{
3013     uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
3014     BasicType bt = Matcher::vector_element_basic_type(this, $src);
3015     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3016     // When the input predicate is all-false, the result should be the vector length
3017     // instead of max vector register size.
3018     if (length_in_bytes == MaxVectorSize) {
3019       __ sve_brkb(as_PRegister($ptmp$$reg), ptrue, as_PRegister($src$$reg), false);
3020     } else {
3021       __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
3022       __ sve_brkb(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg), as_PRegister($src$$reg), false);
3023     }
3024     __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($ptmp$$reg));
3025   %}
3026   ins_pipe(pipe_slow);
3027 %}
3028 
3029 instruct vmask_lasttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
3030   predicate(UseSVE > 0);
3031   match(Set dst (VectorMaskLastTrue src));
3032   effect(TEMP ptmp);
3033   ins_cost(3 * SVE_COST);
3034   format %{ "vmask_lasttrue $dst, $src\t# vector mask lasttrue (sve)" %}
3035   ins_encode %{
3036     BasicType bt = Matcher::vector_element_basic_type(this, $src);
3037     __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($src$$reg), as_PRegister($ptmp$$reg));
3038   %}
3039   ins_pipe(pipe_slow);
3040 %}
3041 
3042 instruct vmask_tolong(iRegLNoSp dst, pReg src, vReg vtmp1, vReg vtmp2) %{
3043   predicate(UseSVE > 0 &&
3044             n->in(1)->bottom_type()->is_vect()->length() <= 64);
3045   match(Set dst (VectorMaskToLong src));
3046   effect(TEMP vtmp1, TEMP vtmp2);
3047   ins_cost(13 * SVE_COST);
3048   format %{ "vmask_tolong $dst, $src\t# vector mask tolong (sve)" %}
3049   ins_encode %{
3050     __ sve_vmask_tolong(as_Register($dst$$reg), as_PRegister($src$$reg),
3051                         Matcher::vector_element_basic_type(this, $src),
3052                         Matcher::vector_length(this, $src),
3053                         as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg));
3054   %}
3055   ins_pipe(pipe_slow);
3056 %}
3057 
3058 // ---------------------------- Vector mask generation ---------------------------
3059 // The rules below set predicate registers. They can guarantee the high bits of dst
3060 // are cleared with zero when the vector length is less than the full size of
3061 // hardware vector register width.
3062 
3063 define(`MASKALL_IMM', `
3064 instruct vmaskAll_imm$1(pRegGov dst, imm$1 src) %{
3065   predicate(UseSVE > 0);
3066   match(Set dst (MaskAll src));
3067   ins_cost(SVE_COST);
3068   format %{ "sve_ptrue_lanecnt/sve_pfalse $dst\t# mask all (sve) ($2)" %}
3069   ins_encode %{
3070     ifelse($1, `I', int, long) con = (ifelse($1, `I', int, long))$src$$constant;
3071     if (con == 0) {
3072       __ sve_pfalse(as_PRegister($dst$$reg));
3073     } else {
3074       assert(con == -1, "invalid constant value for mask");
3075       BasicType bt = Matcher::vector_element_basic_type(this);
3076       __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
3077                            Matcher::vector_length(this));
3078     }
3079   %}
3080   ins_pipe(pipe_slow);
3081 %}')dnl
3082 
3083 define(`MASKALL', `
3084 instruct vmaskAll$1(pRegGov dst, ifelse($1, `I', iRegIorL2I, iRegL) src, vReg tmp, rFlagsReg cr) %{
3085   predicate(UseSVE > 0);
3086   match(Set dst (MaskAll src));
3087   effect(TEMP tmp, KILL cr);
3088   ins_cost(3 * SVE_COST);
3089   format %{ "sve_dup $tmp, $src\n\t"
3090             "sve_ptrue $dst, vector_length\n\t"
3091             "sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) ($2)" %}
3092   ins_encode %{
3093     BasicType bt = Matcher::vector_element_basic_type(this);
3094     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3095     uint length_in_bytes = Matcher::vector_length_in_bytes(this);
3096     __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
3097     if (length_in_bytes < MaxVectorSize) {
3098       __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), size, Matcher::vector_length(this));
3099       __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size,
3100                  as_PRegister($dst$$reg), as_FloatRegister($tmp$$reg), 0);
3101     } else {
3102       __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
3103     }
3104   %}
3105   ins_pipe(pipe_slow);
3106 %}')dnl
3107 dnl
3108 // maskAll (full or partial predicate size)
3109 MASKALL_IMM(I, B/H/S)
3110 MASKALL(I, B/H/S)
3111 MASKALL_IMM(L, D)
3112 MASKALL(L, D)
3113 
3114 // vector mask compare
3115 
3116 instruct vmaskcmp(pRegGov dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %{
3117   predicate(UseSVE > 0);
3118   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
3119   effect(KILL cr);
3120   ins_cost(2 * SVE_COST);
3121   format %{ "sve_cmp $dst, $src1, $src2\t# vector mask cmp (sve)" %}
3122   ins_encode %{
3123     uint length_in_bytes = Matcher::vector_length_in_bytes(this);
3124     BasicType bt = Matcher::vector_element_basic_type(this);
3125     if (length_in_bytes == MaxVectorSize) {
3126       __ sve_compare(as_PRegister($dst$$reg), bt, ptrue, as_FloatRegister($src1$$reg),
3127                      as_FloatRegister($src2$$reg), (int)$cond$$constant);
3128     } else {
3129       __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
3130                            Matcher::vector_length(this));
3131       __ sve_compare(as_PRegister($dst$$reg), bt, as_PRegister($dst$$reg), as_FloatRegister($src1$$reg),
3132                      as_FloatRegister($src2$$reg), (int)$cond$$constant);
3133     }
3134   %}
3135   ins_pipe(pipe_slow);
3136 %}
3137 
3138 instruct vmaskcmp_masked(pRegGov dst, vReg src1, vReg src2, immI cond, pRegGov pg, rFlagsReg cr) %{
3139   predicate(UseSVE > 0);
3140   match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond pg)));
3141   effect(KILL cr);
3142   ins_cost(SVE_COST);
3143   format %{ "sve_cmp $dst, $pg, $src1, $src2\t# vector mask cmp (sve)" %}
3144   ins_encode %{
3145     BasicType bt = Matcher::vector_element_basic_type(this);
3146     __ sve_compare(as_PRegister($dst$$reg), bt, as_PRegister($pg$$reg), as_FloatRegister($src1$$reg),
3147                    as_FloatRegister($src2$$reg), (int)$cond$$constant);
3148   %}
3149   ins_pipe(pipe_slow);
3150 %}
3151 
3152 // vector load mask
3153 
3154 instruct vloadmaskB(pRegGov dst, vReg src, rFlagsReg cr) %{
3155   predicate(UseSVE > 0 &&
3156             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
3157   match(Set dst (VectorLoadMask src));
3158   effect(KILL cr);
3159   ins_cost(SVE_COST);
3160   format %{ "vloadmaskB $dst, $src\t# vector load mask (sve) (B)" %}
3161   ins_encode %{
3162     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ B,
3163                ptrue, as_FloatRegister($src$$reg), 0);
3164   %}
3165   ins_pipe(pipe_slow);
3166 %}
3167 
3168 instruct vloadmask_extend(pRegGov dst, vReg src, vReg tmp, rFlagsReg cr) %{
3169   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() != T_BYTE);
3170   match(Set dst (VectorLoadMask src));
3171   effect(TEMP tmp, KILL cr);
3172   ins_cost(3 * SVE_COST);
3173   format %{ "vloadmask $dst, $src\t# vector load mask (sve) (H/S/D)" %}
3174   ins_encode %{
3175     BasicType bt = Matcher::vector_element_basic_type(this);
3176     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3177     __ sve_vector_extend(as_FloatRegister($tmp$$reg), size, as_FloatRegister($src$$reg), __ B);
3178     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
3179   %}
3180   ins_pipe(pipe_slow);
3181 %}
3182 
3183 instruct vmask_gen(pRegGov pg, iRegL len, rFlagsReg cr) %{
3184   predicate(UseSVE > 0);
3185   match(Set pg (VectorMaskGen len));
3186   effect(KILL cr);
3187   ins_cost(SVE_COST);
3188   format %{ "sve_whilelo $pg, zr, $len\t # sve" %}
3189   ins_encode %{
3190     BasicType bt = Matcher::vector_element_basic_type(this);
3191     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3192     __ sve_whilelo(as_PRegister($pg$$reg), size, zr, as_Register($len$$reg));
3193   %}
3194   ins_pipe(pipe_slow);
3195 %}