1 //
   2 // Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
   3 // Copyright (c) 2020, 2021, Arm Limited. All rights reserved.
   4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5 //
   6 // This code is free software; you can redistribute it and/or modify it
   7 // under the terms of the GNU General Public License version 2 only, as
   8 // published by the Free Software Foundation.
   9 //
  10 // This code is distributed in the hope that it will be useful, but WITHOUT
  11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 // version 2 for more details (a copy is included in the LICENSE file that
  14 // accompanied this code).
  15 //
  16 // You should have received a copy of the GNU General Public License version
  17 // 2 along with this work; if not, write to the Free Software Foundation,
  18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21 // or visit www.oracle.com if you need additional information or have any
  22 // questions.
  23 //
  24 //
  25 
  26 dnl Generate the warning
  27 // This file is automatically generated by running "m4 aarch64_sve_ad.m4". Do not edit ----
  28 dnl
  29 
  30 // AArch64 SVE Architecture Description File
  31 
  32 dnl
  33 define(`TYPE2DATATYPE',
  34 `ifelse($1, `B', `BYTE',
  35         $1, `S', `SHORT',
  36         $1, `I', `INT',
  37         $1, `L', `LONG',
  38         $1, `F', `FLOAT',
  39         $1, `D', `DOUBLE',
  40         `error($1)')')dnl
  41 dnl
  42 dnl OPERAND_VMEMORYA_IMMEDIATE_OFFSET($1,            $2,       $3       $4   )
  43 dnl OPERAND_VMEMORYA_IMMEDIATE_OFFSET(imm_type_abbr, imm_type, imm_len, scale)
  44 define(`OPERAND_VMEMORYA_IMMEDIATE_OFFSET', `
  45 operand vmemA_imm$1Offset$3()
  46 %{
  47   // (esize / msize) = $4
  48   predicate(Address::offset_ok_for_sve_immed(n->get_$2(), $3,
  49             Matcher::scalable_vector_reg_size(T_BYTE)ifelse($4, `1', `', ` / $4')));
  50   match(Con$1);
  51 
  52   op_cost(0);
  53   format %{ %}
  54   interface(CONST_INTER);
  55 %}')dnl
  56 
  57 // 4 bit signed offset -- for predicated load/store
  58 OPERAND_VMEMORYA_IMMEDIATE_OFFSET(I, int,  4, 1)
  59 OPERAND_VMEMORYA_IMMEDIATE_OFFSET(L, long, 4, 1)
  60 dnl
  61 dnl OPERAND_VMEMORYA_INDIRECT_OFFSET($1,            $2     )
  62 dnl OPERAND_VMEMORYA_INDIRECT_OFFSET(imm_type_abbr, imm_len)
  63 define(`OPERAND_VMEMORYA_INDIRECT_OFFSET', `
  64 operand vmemA_indOff$1$2$3(iRegP reg, vmemA_imm$1Offset$2 off)
  65 %{
  66   constraint(ALLOC_IN_RC(ptr_reg));
  67   match(AddP reg off);
  68   op_cost(0);
  69   format %{ "[$reg, $off]" %}
  70   interface(MEMORY_INTER) %{
  71     base($reg);
  72     `index'(0xffffffff);
  73     scale(0x0);
  74     disp($off);
  75   %}
  76 %}')dnl
  77 OPERAND_VMEMORYA_INDIRECT_OFFSET(I, 4)
  78 OPERAND_VMEMORYA_INDIRECT_OFFSET(L, 4)
  79 
  80 // The indOff of vmemA is valid only when the vector element (load to/store from)
  81 // size equals to memory element (load from/store to) size.
  82 opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);
  83 
  84 source_hpp %{
  85   bool op_sve_supported(int opcode, int vlen, BasicType bt);
  86   bool masked_op_sve_supported(int opcode, int vlen, BasicType bt);
  87 %}
  88 
  89 source %{
  90 
  91   typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
  92                                                              PRegister Pg, const Address &adr);
  93 
  94   // Predicated load/store, with optional ptrue to all elements of given predicate register.
  95   static void loadStoreA_predicated(C2_MacroAssembler masm, bool is_store, FloatRegister reg,
  96                                     PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
  97                                     int opcode, Register base, int index, int size, int disp) {
  98     sve_mem_insn_predicate insn;
  99     int mesize = type2aelembytes(mem_elem_bt);
 100     if (index == -1) {
 101       assert(size == 0, "unsupported address mode: scale size = %d", size);
 102       switch(mesize) {
 103       case 1:
 104         insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
 105         break;
 106       case 2:
 107         insn = is_store ? &C2_MacroAssembler::sve_st1h : &C2_MacroAssembler::sve_ld1h;
 108         break;
 109       case 4:
 110         insn = is_store ? &C2_MacroAssembler::sve_st1w : &C2_MacroAssembler::sve_ld1w;
 111         break;
 112       case 8:
 113         insn = is_store ? &C2_MacroAssembler::sve_st1d : &C2_MacroAssembler::sve_ld1d;
 114         break;
 115       default:
 116         assert(false, "unsupported");
 117         ShouldNotReachHere();
 118       }
 119       int imm4 = disp / mesize / Matcher::scalable_vector_reg_size(vector_elem_bt);
 120       (masm.*insn)(reg, Assembler::elemType_to_regVariant(vector_elem_bt), pg, Address(base, imm4));
 121     } else {
 122       assert(false, "unimplemented");
 123       ShouldNotReachHere();
 124     }
 125   }
 126 
 127   bool op_sve_supported(int opcode, int vlen, BasicType bt) {
 128     int length_in_bytes = vlen * type2aelembytes(bt);
 129     switch (opcode) {
 130       case Op_MulAddVS2VI:
 131       // No multiply reduction instructions
 132       case Op_MulReductionVD:
 133       case Op_MulReductionVF:
 134       case Op_MulReductionVI:
 135       case Op_MulReductionVL:
 136       // Others
 137       case Op_ExtractC:
 138       case Op_ExtractUB:
 139         return false;
 140       // Vector API specific
 141       case Op_VectorLoadShuffle:
 142       case Op_VectorRearrange:
 143         return vlen >= 4 && length_in_bytes <= MaxVectorSize;
 144       case Op_LoadVector:
 145       case Op_StoreVector:
 146         return Matcher::vector_size_supported(bt, vlen);
 147       default:
 148         break;
 149     }
 150     // By default, we only support vector operations with no less than 8 bytes and 2 elements.
 151     return 8 <= length_in_bytes && length_in_bytes <= MaxVectorSize && vlen >= 2;
 152   }
 153 
 154   bool masked_op_sve_supported(int opcode, int vlen, BasicType bt) {
 155     if (opcode == Op_VectorRearrange) {
 156       return false;
 157     }
 158     return op_sve_supported(opcode, vlen, bt);
 159   }
 160 
 161 %}
 162 
 163 definitions %{
 164   int_def SVE_COST             (200, 200);
 165 %}
 166 
 167 dnl
 168 dnl ELEMENT_SHORT_CHART($1, $2)
 169 dnl ELEMENT_SHORT_CHART(etype, node)
 170 define(`ELEMENT_SHORT_CHAR',`ifelse(`$1', `T_SHORT',
 171   `($2->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
 172             ($2->bottom_type()->is_vect()->element_basic_type() == T_CHAR))',
 173    `($2->bottom_type()->is_vect()->element_basic_type() == $1)')')dnl
 174 dnl
 175 
 176 // All SVE instructions
 177 
 178 // vector load/store
 179 
 180 // Unpredicated vector load/store
 181 instruct loadV(vReg dst, vmemA mem) %{
 182   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() >= 16 &&
 183             n->as_LoadVector()->memory_size() == MaxVectorSize);
 184   match(Set dst (LoadVector mem));
 185   ins_cost(4 * SVE_COST);
 186   format %{ "sve_ldr $dst, $mem\t# vector (sve)" %}
 187   ins_encode %{
 188     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 189     BasicType bt = Matcher::vector_element_basic_type(this);
 190     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg, ptrue,
 191                           bt, bt, $mem->opcode(),
 192                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 193   %}
 194   ins_pipe(pipe_slow);
 195 %}
 196 
 197 instruct storeV(vReg src, vmemA mem) %{
 198   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() >= 16 &&
 199             n->as_StoreVector()->memory_size() == MaxVectorSize);
 200   match(Set mem (StoreVector mem src));
 201   ins_cost(4 * SVE_COST);
 202   format %{ "sve_str $mem, $src\t# vector (sve)" %}
 203   ins_encode %{
 204     FloatRegister src_reg = as_FloatRegister($src$$reg);
 205     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 206     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg, ptrue,
 207                           bt, bt, $mem->opcode(),
 208                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 209   %}
 210   ins_pipe(pipe_slow);
 211 %}dnl
 212 
 213 dnl
 214 define(`VLoadStore', `
 215 // ifelse(load, $3, Load, Store) Vector ($6 bits)
 216 instruct $3V$4_vreg`'(vReg $7, vmem$4 mem)
 217 %{
 218   predicate(UseSVE > 0 && `n->as_'ifelse(load, $3, Load, Store)Vector()->memory_size() == $4);
 219   match(Set ifelse(load, $3, dst (LoadVector mem), mem (StoreVector mem src)));
 220   ins_cost(4 * INSN_COST);
 221   format %{ "$1   ifelse(load, $3, `$dst,$mem', `$mem,$src')\t# vector ($6 bits)" %}
 222   ins_encode( `aarch64_enc_'ifelse(load, $3, ldr, str)v$2($7, mem) );
 223   ins_pipe(v$3`_reg_mem'ifelse(eval($4 * 8), 128, 128, 64));
 224 %}')dnl
 225 dnl        $1    $2 $3     $4  $5 $6   $7
 226 VLoadStore(ldrh, H, load,  2,  D, 16,  dst)
 227 VLoadStore(strh, H, store, 2,  D, 16,  src)
 228 VLoadStore(ldrs, S, load,  4,  D, 32,  dst)
 229 VLoadStore(strs, S, store, 4,  D, 32,  src)
 230 VLoadStore(ldrd, D, load,  8,  D, 64,  dst)
 231 VLoadStore(strd, D, store, 8,  D, 64,  src)
 232 VLoadStore(ldrq, Q, load, 16,  X, 128, dst)
 233 VLoadStore(strq, Q, store, 16, X, 128, src)
 234 
 235 // Predicated vector load/store, based on the vector length of the node.
 236 // Only load/store values in the range of the memory_size. This is needed
 237 // when the memory_size is lower than the hardware supported max vector size.
 238 // And this might happen for Vector API mask vector load/store.
 239 instruct loadV_partial(vReg dst, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
 240   predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() > 16 &&
 241             n->as_LoadVector()->memory_size() < MaxVectorSize);
 242   match(Set dst (LoadVector mem));
 243   effect(TEMP pgtmp, KILL cr);
 244   ins_cost(6 * SVE_COST);
 245   format %{ "sve_whilelo_zr_imm $pgtmp, vector_length\n\t"
 246             "sve_ldr $dst, $pgtmp, $mem\t# load vector partial" %}
 247   ins_encode %{
 248     BasicType bt = Matcher::vector_element_basic_type(this);
 249     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 250                           Matcher::vector_length(this));
 251     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 252     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg,
 253                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 254                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 255   %}
 256   ins_pipe(pipe_slow);
 257 %}
 258 
 259 instruct storeV_partial(vReg src, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
 260   predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() > 16 &&
 261             n->as_StoreVector()->memory_size() < MaxVectorSize);
 262   match(Set mem (StoreVector mem src));
 263   effect(TEMP pgtmp, KILL cr);
 264   ins_cost(5 * SVE_COST);
 265   format %{ "sve_whilelo_zr_imm $pgtmp, vector_length\n\t"
 266             "sve_str $src, $pgtmp, $mem\t# store vector partial" %}
 267   ins_encode %{
 268     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 269     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 270                           Matcher::vector_length(this, $src));
 271     FloatRegister src_reg = as_FloatRegister($src$$reg);
 272     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg,
 273                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 274                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 275   %}
 276   ins_pipe(pipe_slow);
 277 %}
 278 
 279 // vector load/store - predicated
 280 
 281 instruct loadV_masked(vReg dst, vmemA mem, pRegGov pg) %{
 282   predicate(UseSVE > 0 &&
 283             n->as_LoadVector()->memory_size() == MaxVectorSize);
 284   match(Set dst (LoadVectorMasked mem pg));
 285   ins_cost(4 * SVE_COST);
 286   format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated (sve)" %}
 287   ins_encode %{
 288     BasicType bt = Matcher::vector_element_basic_type(this);
 289     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
 290                           as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
 291                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 292   %}
 293   ins_pipe(pipe_slow);
 294 %}
 295 
 296 instruct loadV_masked_partial(vReg dst, vmemA mem, pRegGov pg, pRegGov pgtmp, rFlagsReg cr) %{
 297   predicate(UseSVE > 0 &&
 298             n->as_LoadVector()->memory_size() < MaxVectorSize);
 299   match(Set dst (LoadVectorMasked mem pg));
 300   effect(TEMP pgtmp, KILL cr);
 301   ins_cost(6 * SVE_COST);
 302   format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated partial (sve)" %}
 303   ins_encode %{
 304     BasicType bt = Matcher::vector_element_basic_type(this);
 305     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 306                           Matcher::vector_length(this));
 307     __ sve_and(as_PRegister($pgtmp$$reg), as_PRegister($pgtmp$$reg),
 308                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
 309     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
 310                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 311                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 312   %}
 313   ins_pipe(pipe_slow);
 314 %}
 315 
 316 instruct storeV_masked(vReg src, vmemA mem, pRegGov pg) %{
 317   predicate(UseSVE > 0 &&
 318             n->as_StoreVector()->memory_size() == MaxVectorSize);
 319   match(Set mem (StoreVectorMasked mem (Binary src pg)));
 320   ins_cost(4 * SVE_COST);
 321   format %{ "sve_str $mem, $pg, $src\t# store vector predicated (sve)" %}
 322   ins_encode %{
 323     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 324     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
 325                           as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
 326                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 327   %}
 328   ins_pipe(pipe_slow);
 329 %}
 330 
 331 instruct storeV_masked_partial(vReg src, vmemA mem, pRegGov pg, pRegGov pgtmp, rFlagsReg cr) %{
 332   predicate(UseSVE > 0 &&
 333             n->as_StoreVector()->memory_size() < MaxVectorSize);
 334   match(Set mem (StoreVectorMasked mem (Binary src pg)));
 335   effect(TEMP pgtmp, KILL cr);
 336   ins_cost(6 * SVE_COST);
 337   format %{ "sve_str $mem, $pg, $src\t# store vector predicated partial (sve)" %}
 338   ins_encode %{
 339     BasicType bt = Matcher::vector_element_basic_type(this, $src);
 340     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
 341                           Matcher::vector_length(this, $src));
 342     __ sve_and(as_PRegister($pgtmp$$reg), as_PRegister($pgtmp$$reg),
 343                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
 344     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
 345                           as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
 346                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 347   %}
 348   ins_pipe(pipe_slow);
 349 %}
 350 
 351 dnl
 352 dnl MASKALL_IMM($1,   $2  )
 353 dnl MASKALL_IMM(type, size)
 354 define(`MASKALL_IMM', `
 355 instruct vmaskAll_imm$1(pRegGov dst, imm$1 src) %{
 356   predicate(UseSVE > 0);
 357   match(Set dst (MaskAll src));
 358   ins_cost(SVE_COST);
 359   format %{ "sve_ptrue_lanecnt/sve_pfalse $dst\t# mask all (sve) ($2)" %}
 360   ins_encode %{
 361     ifelse($1, `I', int, long) con = (ifelse($1, `I', int, long))$src$$constant;
 362     if (con == 0) {
 363       __ sve_pfalse(as_PRegister($dst$$reg));
 364     } else {
 365       assert(con == -1, "invalid constant value for mask");
 366       BasicType bt = Matcher::vector_element_basic_type(this);
 367       __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
 368                            Matcher::vector_length(this));
 369     }
 370   %}
 371   ins_pipe(pipe_slow);
 372 %}')dnl
 373 dnl
 374 dnl MASKALL($1,   $2  )
 375 dnl MASKALL(type, size)
 376 define(`MASKALL', `
 377 instruct vmaskAll$1(pRegGov dst, ifelse($1, `I', iRegIorL2I, iRegL) src, vReg tmp, rFlagsReg cr) %{
 378   predicate(UseSVE > 0);
 379   match(Set dst (MaskAll src));
 380   effect(TEMP tmp, KILL cr);
 381   ins_cost(3 * SVE_COST);
 382   format %{ "sve_dup $tmp, $src\n\t"
 383             "sve_ptrue_lanecnt $dst\n\t"
 384             "sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) ($2)" %}
 385   ins_encode %{
 386     BasicType bt = Matcher::vector_element_basic_type(this);
 387     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
 388     uint length_in_bytes = Matcher::vector_length_in_bytes(this);
 389     __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
 390     if (length_in_bytes < MaxVectorSize) {
 391       __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), size, Matcher::vector_length(this));
 392       __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size,
 393                  as_PRegister($dst$$reg), as_FloatRegister($tmp$$reg), 0);
 394     } else {
 395       __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
 396     }
 397   %}
 398   ins_pipe(pipe_slow);
 399 %}')dnl
 400 dnl
 401 // maskAll (full or partial predicate size)
 402 MASKALL_IMM(I, B/H/S)
 403 MASKALL(I, B/H/S)
 404 MASKALL_IMM(L, D)
 405 MASKALL(L, D)
 406 
 407 dnl
 408 dnl MASK_LOGICAL_OP($1,        $2,      $3  )
 409 dnl MASK_LOGICAL_OP(insn_name, op_name, insn)
 410 define(`MASK_LOGICAL_OP', `
 411 instruct vmask_$1(pRegGov pd, pRegGov pn, pRegGov pm) %{
 412   predicate(UseSVE > 0);
 413   match(Set pd ($2 pn pm));
 414   ins_cost(SVE_COST);
 415   format %{ "$3 $pd, $pn, $pm\t# predicate (sve)" %}
 416   ins_encode %{
 417     __ $3(as_PRegister($pd$$reg), ptrue,
 418                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 419   %}
 420   ins_pipe(pipe_slow);
 421 %}')dnl
 422 dnl
 423 // mask logical and/or/xor
 424 MASK_LOGICAL_OP(and, AndVMask, sve_and)
 425 MASK_LOGICAL_OP(or, OrVMask, sve_orr)
 426 MASK_LOGICAL_OP(xor, XorVMask, sve_eor)
 427 
 428 dnl
 429 dnl MASK_LOGICAL_AND_NOT($1,   $2  )
 430 dnl MASK_LOGICAL_AND_NOT(type, size)
 431 define(`MASK_LOGICAL_AND_NOT', `
 432 instruct vmask_and_not$1(pRegGov pd, pRegGov pn, pRegGov pm, imm$1_M1 m1) %{
 433   predicate(UseSVE > 0);
 434   match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
 435   ins_cost(SVE_COST);
 436   format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) ($2)" %}
 437   ins_encode %{
 438     __ sve_bic(as_PRegister($pd$$reg), ptrue,
 439                as_PRegister($pn$$reg), as_PRegister($pm$$reg));
 440   %}
 441   ins_pipe(pipe_slow);
 442 %}')dnl
 443 dnl
 444 // mask logical and_not
 445 MASK_LOGICAL_AND_NOT(I, B/H/S)
 446 MASK_LOGICAL_AND_NOT(L, D)
 447 
 448 // vector reinterpret
 449 
 450 instruct reinterpret(vReg dst) %{
 451   predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() ==
 452                           n->in(1)->bottom_type()->is_vect()->length_in_bytes());  // src == dst
 453   match(Set dst (VectorReinterpret dst));
 454   ins_cost(0);
 455   format %{ "# reinterpret $dst\t# do nothing" %}
 456   ins_encode %{
 457     // empty
 458   %}
 459   ins_pipe(pipe_class_empty);
 460 %}
 461 
 462 instruct reinterpretResize(vReg dst, vReg src, pRegGov pgtmp, rFlagsReg cr) %{
 463   predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() !=
 464                           n->in(1)->bottom_type()->is_vect()->length_in_bytes());  // src != dst
 465   match(Set dst (VectorReinterpret src));
 466   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
 467   ins_cost(3 * SVE_COST);
 468   format %{ "reinterpretResize $dst, $src\t# vector (sve)" %}
 469   ins_encode %{
 470     uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
 471     uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
 472     uint length_in_bytes_resize = length_in_bytes_src < length_in_bytes_dst ?
 473                                   length_in_bytes_src : length_in_bytes_dst;
 474     assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
 475            "invalid vector length");
 476     __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ B, length_in_bytes_resize);
 477     __ sve_dup(as_FloatRegister($dst$$reg), __ B, 0);
 478     __ sve_sel(as_FloatRegister($dst$$reg), __ B, as_PRegister($pgtmp$$reg),
 479                as_FloatRegister($src$$reg), as_FloatRegister($dst$$reg));
 480   %}
 481   ins_pipe(pipe_slow);
 482 %}
 483 
 484 // vector mask reinterpret
 485 
 486 instruct vmask_reinterpret_same_esize(pRegGov dst_src) %{
 487   predicate(UseSVE > 0 &&
 488             n->as_Vector()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
 489             n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
 490   match(Set dst_src (VectorReinterpret dst_src));
 491   ins_cost(0);
 492   format %{ "# vmask_reinterpret $dst_src\t# do nothing" %}
 493   ins_encode %{
 494     // empty
 495   %}
 496   ins_pipe(pipe_class_empty);
 497 %}
 498 
 499 instruct vmask_reinterpret_diff_esize(pRegGov dst, pRegGov src, vReg tmp, rFlagsReg cr) %{
 500   predicate(UseSVE > 0 &&
 501             n->as_Vector()->length() != n->in(1)->bottom_type()->is_vect()->length() &&
 502             n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
 503   match(Set dst (VectorReinterpret src));
 504   effect(TEMP tmp, KILL cr);
 505   ins_cost(2 * SVE_COST);
 506   format %{ "# vmask_reinterpret $dst, $src\t# vector (sve)" %}
 507   ins_encode %{
 508     BasicType from_bt = Matcher::vector_element_basic_type(this, $src);
 509     Assembler::SIMD_RegVariant from_size = __ elemType_to_regVariant(from_bt);
 510     BasicType to_bt = Matcher::vector_element_basic_type(this);
 511     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
 512     __ sve_cpy(as_FloatRegister($tmp$$reg), from_size, as_PRegister($src$$reg), -1, false);
 513     __ sve_cmp(Assembler::EQ, as_PRegister($dst$$reg), to_size, ptrue, as_FloatRegister($tmp$$reg), -1);
 514   %}
 515   ins_pipe(pipe_slow);
 516 %}
 517 dnl
 518 dnl UNARY_OP_TRUE_PREDICATE($1,        $2,      $3,   $4  )
 519 dnl UNARY_OP_TRUE_PREDICATE(insn_name, op_name, size, insn)
 520 define(`UNARY_OP_TRUE_PREDICATE', `
 521 instruct $1(vReg dst, vReg src) %{
 522   predicate(UseSVE > 0 &&
 523             !n->as_Vector()->is_predicated_vector());
 524   match(Set dst ($2 src));
 525   ins_cost(SVE_COST);
 526   format %{ "$4 $dst, $src\t# vector (sve) ($3)" %}
 527   ins_encode %{
 528     __ $4(as_FloatRegister($dst$$reg), __ $3,
 529          ptrue, as_FloatRegister($src$$reg));
 530   %}
 531   ins_pipe(pipe_slow);
 532 %}')dnl
 533 dnl
 534 
 535 // vector abs
 536 UNARY_OP_TRUE_PREDICATE(vabsB, AbsVB, B, sve_abs)
 537 UNARY_OP_TRUE_PREDICATE(vabsS, AbsVS, H, sve_abs)
 538 UNARY_OP_TRUE_PREDICATE(vabsI, AbsVI, S, sve_abs)
 539 UNARY_OP_TRUE_PREDICATE(vabsL, AbsVL, D, sve_abs)
 540 UNARY_OP_TRUE_PREDICATE(vabsF, AbsVF, S, sve_fabs)
 541 UNARY_OP_TRUE_PREDICATE(vabsD, AbsVD, D, sve_fabs)
 542 
 543 dnl UNARY_OP_PREDICATE($1,        $2,      $3,   $4  )
 544 dnl UNARY_OP_PREDICATE(insn_name, op_name, size, insn)
 545 define(`UNARY_OP_PREDICATE', `
 546 instruct $1_masked(vReg dst_src, pRegGov pg) %{
 547   predicate(UseSVE > 0);
 548   match(Set dst_src ($2 dst_src pg));
 549   ins_cost(SVE_COST);
 550   format %{ "$4 $dst_src, $pg, $dst_src\t# vector (sve) ($3)" %}
 551   ins_encode %{
 552     __ $4(as_FloatRegister($dst_src$$reg), __ $3,
 553             as_PRegister($pg$$reg),
 554             as_FloatRegister($dst_src$$reg));
 555   %}
 556   ins_pipe(pipe_slow);
 557 %}')dnl
 558 // vector abs - predicated
 559 UNARY_OP_PREDICATE(vabsB, AbsVB, B, sve_abs)
 560 UNARY_OP_PREDICATE(vabsS, AbsVS, H, sve_abs)
 561 UNARY_OP_PREDICATE(vabsI, AbsVI, S, sve_abs)
 562 UNARY_OP_PREDICATE(vabsL, AbsVL, D, sve_abs)
 563 UNARY_OP_PREDICATE(vabsF, AbsVF, S, sve_fabs)
 564 UNARY_OP_PREDICATE(vabsD, AbsVD, D, sve_fabs)
 565 
 566 dnl
 567 dnl BINARY_OP_UNPREDICATE($1,        $2       $3,   $4           $5  )
 568 dnl BINARY_OP_UNPREDICATE(insn_name, op_name, size, min_vec_len, insn)
 569 define(`BINARY_OP_UNPREDICATE', `
 570 instruct $1(vReg dst, vReg src1, vReg src2) %{
 571   predicate(UseSVE > 0);
 572   match(Set dst ($2 src1 src2));
 573   ins_cost(SVE_COST);
 574   format %{ "$5 $dst, $src1, $src2\t # vector (sve) ($3)" %}
 575   ins_encode %{
 576     __ $5(as_FloatRegister($dst$$reg), __ $3,
 577          as_FloatRegister($src1$$reg),
 578          as_FloatRegister($src2$$reg));
 579   %}
 580   ins_pipe(pipe_slow);
 581 %}')dnl
 582 dnl
 583 dnl
 584 dnl BINARY_OP_PREDICATE($1,        $2,      $3,   $4  )
 585 dnl BINARY_OP_PREDICATE(insn_name, op_name, size, insn)
 586 define(`BINARY_OP_PREDICATE', `
 587 instruct $1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 588   predicate(UseSVE > 0);
 589   match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
 590   ins_cost(SVE_COST);
 591   format %{ "$4 $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) ($3)" %}
 592   ins_encode %{
 593     __ $4(as_FloatRegister($dst_src1$$reg), __ $3,
 594             as_PRegister($pg$$reg),
 595             as_FloatRegister($src2$$reg));
 596   %}
 597   ins_pipe(pipe_slow);
 598 %}')dnl
 599 dnl
 600 // vector add
 601 BINARY_OP_UNPREDICATE(vaddB, AddVB, B, 16, sve_add)
 602 BINARY_OP_UNPREDICATE(vaddS, AddVS, H, 8,  sve_add)
 603 BINARY_OP_UNPREDICATE(vaddI, AddVI, S, 4,  sve_add)
 604 BINARY_OP_UNPREDICATE(vaddL, AddVL, D, 2,  sve_add)
 605 BINARY_OP_UNPREDICATE(vaddF, AddVF, S, 4,  sve_fadd)
 606 BINARY_OP_UNPREDICATE(vaddD, AddVD, D, 2,  sve_fadd)
 607 
 608 // vector add - predicated
 609 BINARY_OP_PREDICATE(vaddB, AddVB, B, sve_add)
 610 BINARY_OP_PREDICATE(vaddS, AddVS, H, sve_add)
 611 BINARY_OP_PREDICATE(vaddI, AddVI, S, sve_add)
 612 BINARY_OP_PREDICATE(vaddL, AddVL, D, sve_add)
 613 BINARY_OP_PREDICATE(vaddF, AddVF, S, sve_fadd)
 614 BINARY_OP_PREDICATE(vaddD, AddVD, D, sve_fadd)
 615 dnl
 616 dnl ADD_IMM($1,          $2,   $3      )
 617 dnl ADD_IMM(name_suffix, size, imm_type)
 618 define(`ADD_IMM', `
 619 instruct vaddImm$1(vReg dst_src, $3 con) %{
 620   predicate(UseSVE > 0);
 621   match(Set dst_src (AddV$1 dst_src (Replicate$1 con)));
 622   ins_cost(SVE_COST);
 623   format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) ($2)" %}
 624   ins_encode %{
 625     int32_t val = $con$$constant;
 626     if (val > 0){
 627       __ sve_add(as_FloatRegister($dst_src$$reg), __ $2, val);
 628     } else if (val < 0){
 629       __ sve_sub(as_FloatRegister($dst_src$$reg), __ $2, -val);
 630     }
 631   %}
 632   ins_pipe(pipe_slow);
 633 %}')dnl
 634 
 635 // vector add reg imm (unpredicated)
 636 ADD_IMM(B, B, immBAddSubV)
 637 ADD_IMM(S, H, immIAddSubV)
 638 ADD_IMM(I, S, immIAddSubV)
 639 ADD_IMM(L, D, immLAddSubV)
 640 dnl
 641 dnl BITWISE_OP_IMM($1,        $2        $3,   $4    $5      )
 642 dnl BITWISE_OP_IMM(insn_name, op_name1, size, type, op_name2)
 643 define(`BITWISE_OP_IMM', `
 644 instruct $1(vReg dst_src, imm$4Log con) %{
 645   predicate(UseSVE > 0);
 646   match(Set dst_src ($2 dst_src (Replicate$4 con)));
 647   ins_cost(SVE_COST);
 648   format %{ "$5 $dst_src, $dst_src, $con\t # vector (sve) ($3)" %}
 649   ins_encode %{
 650     __ $5(as_FloatRegister($dst_src$$reg), __ $3,
 651          (uint64_t)($con$$constant));
 652   %}
 653   ins_pipe(pipe_slow);
 654 %}')dnl
 655 
 656 // vector binary op reg imm (unpredicated)
 657 BITWISE_OP_IMM(vandB, AndV, B, B, sve_and)
 658 BITWISE_OP_IMM(vandH, AndV, H, S, sve_and)
 659 BITWISE_OP_IMM(vandS, AndV, S, I, sve_and)
 660 BITWISE_OP_IMM(vandD, AndV, D, L, sve_and)
 661 BITWISE_OP_IMM(vorB,  OrV,  B, B, sve_orr)
 662 BITWISE_OP_IMM(vorH,  OrV,  H, S, sve_orr)
 663 BITWISE_OP_IMM(vorS,  OrV,  S, I, sve_orr)
 664 BITWISE_OP_IMM(vorD,  OrV,  D, L, sve_orr)
 665 BITWISE_OP_IMM(vxorB, XorV, B, B, sve_eor)
 666 BITWISE_OP_IMM(vxorH, XorV, H, S, sve_eor)
 667 BITWISE_OP_IMM(vxorS, XorV, S, I, sve_eor)
 668 BITWISE_OP_IMM(vxorD, XorV, D, L, sve_eor)
 669 dnl
 670 dnl
 671 dnl BINARY_OP_UNSIZED($1,        $2,      $3  )
 672 dnl BINARY_OP_UNSIZED(insn_name, op_name, insn)
 673 define(`BINARY_OP_UNSIZED', `
 674 instruct $1(vReg dst, vReg src1, vReg src2) %{
 675   predicate(UseSVE > 0);
 676   match(Set dst ($2 src1 src2));
 677   ins_cost(SVE_COST);
 678   format %{ "$3  $dst, $src1, $src2\t# vector (sve)" %}
 679   ins_encode %{
 680     __ $3(as_FloatRegister($dst$$reg),
 681          as_FloatRegister($src1$$reg),
 682          as_FloatRegister($src2$$reg));
 683   %}
 684   ins_pipe(pipe_slow);
 685 %}')dnl
 686 dnl
 687 // vector and
 688 BINARY_OP_UNSIZED(vand, AndV, sve_and)
 689 
 690 // vector or
 691 BINARY_OP_UNSIZED(vor, OrV, sve_orr)
 692 
 693 // vector xor
 694 BINARY_OP_UNSIZED(vxor, XorV, sve_eor)
 695 
 696 dnl BINARY_LOGIC_OP_PREDICATE($1,        $2,      $3  )
 697 dnl BINARY_LOGIC_OP_PREDICATE(insn_name, op_name, insn)
 698 define(`BINARY_LOGIC_OP_PREDICATE', `
 699 instruct $1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 700   predicate(UseSVE > 0);
 701   match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
 702   ins_cost(SVE_COST);
 703   format %{ "$3 $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
 704   ins_encode %{
 705     BasicType bt = Matcher::vector_element_basic_type(this);
 706     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
 707     __ $3(as_FloatRegister($dst_src1$$reg), size,
 708           as_PRegister($pg$$reg),
 709           as_FloatRegister($src2$$reg));
 710   %}
 711   ins_pipe(pipe_slow);
 712 %}')dnl
 713 dnl
 714 // vector and - predicated
 715 BINARY_LOGIC_OP_PREDICATE(vand, AndV, sve_and)
 716 
 717 // vector or - predicated
 718 BINARY_LOGIC_OP_PREDICATE(vor, OrV, sve_orr)
 719 
 720 // vector xor - predicated
 721 BINARY_LOGIC_OP_PREDICATE(vxor, XorV, sve_eor)
 722 
 723 // vector not
 724 dnl
 725 define(`MATCH_RULE', `ifelse($1, I,
 726 `match(Set dst (XorV src (ReplicateB m1)));
 727   match(Set dst (XorV src (ReplicateS m1)));
 728   match(Set dst (XorV src (ReplicateI m1)));',
 729 `match(Set dst (XorV src (ReplicateL m1)));')')dnl
 730 dnl
 731 define(`VECTOR_NOT', `
 732 instruct vnot$1`'(vReg dst, vReg src, imm$1_M1 m1) %{
 733   predicate(UseSVE > 0);
 734   MATCH_RULE($1)
 735   ins_cost(SVE_COST);
 736   format %{ "sve_not $dst, $src\t# vector (sve) $2" %}
 737   ins_encode %{
 738     __ sve_not(as_FloatRegister($dst$$reg), __ D,
 739                ptrue, as_FloatRegister($src$$reg));
 740   %}
 741   ins_pipe(pipe_slow);
 742 %}')dnl
 743 dnl        $1,$2
 744 VECTOR_NOT(I, B/H/S)
 745 VECTOR_NOT(L, D)
 746 undefine(MATCH_RULE)
 747 dnl
 748 // vector and_not
 749 dnl
 750 define(`MATCH_RULE', `ifelse($1, I,
 751 `match(Set dst (AndV src1 (XorV src2 (ReplicateB m1))));
 752   match(Set dst (AndV src1 (XorV src2 (ReplicateS m1))));
 753   match(Set dst (AndV src1 (XorV src2 (ReplicateI m1))));',
 754 `match(Set dst (AndV src1 (XorV src2 (ReplicateL m1))));')')dnl
 755 dnl
 756 define(`VECTOR_AND_NOT', `
 757 instruct vand_not$1`'(vReg dst, vReg src1, vReg src2, imm$1_M1 m1) %{
 758   predicate(UseSVE > 0);
 759   MATCH_RULE($1)
 760   ins_cost(SVE_COST);
 761   format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) $2" %}
 762   ins_encode %{
 763     __ sve_bic(as_FloatRegister($dst$$reg),
 764                as_FloatRegister($src1$$reg),
 765                as_FloatRegister($src2$$reg));
 766   %}
 767   ins_pipe(pipe_slow);
 768 %}')dnl
 769 dnl            $1,$2
 770 VECTOR_AND_NOT(I, B/H/S)
 771 VECTOR_AND_NOT(L, D)
 772 undefine(MATCH_RULE)
 773 dnl
 774 dnl VDIVF($1,          $2  , $3         )
 775 dnl VDIVF(name_suffix, size, min_vec_len)
 776 define(`VDIVF', `
 777 instruct vdiv$1(vReg dst_src1, vReg src2) %{
 778   predicate(UseSVE > 0);
 779   match(Set dst_src1 (DivV$1 dst_src1 src2));
 780   ins_cost(SVE_COST);
 781   format %{ "sve_fdiv  $dst_src1, $dst_src1, $src2\t# vector (sve) ($2)" %}
 782   ins_encode %{
 783     __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ $2,
 784          ptrue, as_FloatRegister($src2$$reg));
 785   %}
 786   ins_pipe(pipe_slow);
 787 %}')dnl
 788 dnl
 789 // vector float div
 790 VDIVF(F, S, 4)
 791 VDIVF(D, D, 2)
 792 
 793 // vector float div - predicated
 794 BINARY_OP_PREDICATE(vfdivF, DivVF, S, sve_fdiv)
 795 BINARY_OP_PREDICATE(vfdivD, DivVD, D, sve_fdiv)
 796 
 797 dnl
 798 dnl VMINMAX($1     , $2, $3   , $4  )
 799 dnl VMINMAX(op_name, op, finsn, insn)
 800 define(`VMINMAX', `
 801 instruct v$1(vReg dst_src1, vReg src2) %{
 802   predicate(UseSVE > 0);
 803   match(Set dst_src1 ($2 dst_src1 src2));
 804   ins_cost(SVE_COST);
 805   format %{ "sve_$1 $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
 806   ins_encode %{
 807     BasicType bt = Matcher::vector_element_basic_type(this);
 808     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
 809     if (is_floating_point_type(bt)) {
 810       __ $3(as_FloatRegister($dst_src1$$reg), size,
 811                   ptrue, as_FloatRegister($src2$$reg));
 812     } else {
 813       assert(is_integral_type(bt), "unsupported type");
 814       __ $4(as_FloatRegister($dst_src1$$reg), size,
 815                   ptrue, as_FloatRegister($src2$$reg));
 816     }
 817   %}
 818   ins_pipe(pipe_slow);
 819 %}')dnl
 820 dnl
 821 // vector min/max
 822 VMINMAX(min, MinV, sve_fmin, sve_smin)
 823 VMINMAX(max, MaxV, sve_fmax, sve_smax)
 824 
 825 dnl
 826 dnl VMINMAX_PREDICATE($1     , $2, $3   , $4  )
 827 dnl VMINMAX_PREDICATE(op_name, op, finsn, insn)
 828 define(`VMINMAX_PREDICATE', `
 829 instruct v$1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
 830   predicate(UseSVE > 0);
 831   match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
 832   ins_cost(SVE_COST);
 833   format %{ "sve_$1 $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %}
 834   ins_encode %{
 835     BasicType bt = Matcher::vector_element_basic_type(this);
 836     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
 837     if (is_floating_point_type(bt)) {
 838       __ $3(as_FloatRegister($dst_src1$$reg), size,
 839                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
 840     } else {
 841       assert(is_integral_type(bt), "unsupported type");
 842       __ $4(as_FloatRegister($dst_src1$$reg), size,
 843                   as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
 844     }
 845   %}
 846   ins_pipe(pipe_slow);
 847 %}')dnl
 848 dnl
 849 // vector min/max - predicated
 850 VMINMAX_PREDICATE(min, MinV, sve_fmin, sve_smin)
 851 VMINMAX_PREDICATE(max, MaxV, sve_fmax, sve_smax)
 852 
 853 dnl
 854 dnl VFMLA($1           $2    $3         )
 855 dnl VFMLA(name_suffix, size, min_vec_len)
 856 define(`VFMLA', `
 857 // dst_src1 = dst_src1 + src2 * src3
 858 instruct vfmla$1(vReg dst_src1, vReg src2, vReg src3) %{
 859   predicate(UseFMA && UseSVE > 0);
 860   match(Set dst_src1 (FmaV$1 dst_src1 (Binary src2 src3)));
 861   ins_cost(SVE_COST);
 862   format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
 863   ins_encode %{
 864     __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ $2,
 865          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 866   %}
 867   ins_pipe(pipe_slow);
 868 %}')dnl
 869 dnl
 870 // vector fmla
 871 VFMLA(F, S, 4)
 872 VFMLA(D, D, 2)
 873 
 874 dnl
 875 dnl VFMLA_PREDICATE($1,   $2  )
 876 dnl VFMLA_PREDICATE(type, size)
 877 define(`VFMLA_PREDICATE', `
 878 // dst_src1 = dst_src1 * src2 + src3
 879 instruct vfmla$1_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
 880   predicate(UseFMA && UseSVE > 0);
 881   match(Set dst_src1 (FmaV$1 (Binary dst_src1 src2) (Binary src3 pg)));
 882   ins_cost(SVE_COST);
 883   format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) ($2)" %}
 884   ins_encode %{
 885     __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ $2, as_PRegister($pg$$reg),
 886          as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 887   %}
 888   ins_pipe(pipe_slow);
 889 %}')dnl
 890 dnl
 891 // vector fmla - predicated
 892 VFMLA_PREDICATE(F, S)
 893 VFMLA_PREDICATE(D, D)
 894 
 895 dnl
 896 dnl VFMLS($1           $2    $3         )
 897 dnl VFMLS(name_suffix, size, min_vec_len)
 898 define(`VFMLS', `
 899 // dst_src1 = dst_src1 + -src2 * src3
 900 // dst_src1 = dst_src1 + src2 * -src3
 901 instruct vfmls$1(vReg dst_src1, vReg src2, vReg src3) %{
 902   predicate(UseFMA && UseSVE > 0);
 903   match(Set dst_src1 (FmaV$1 dst_src1 (Binary (NegV$1 src2) src3)));
 904   match(Set dst_src1 (FmaV$1 dst_src1 (Binary src2 (NegV$1 src3))));
 905   ins_cost(SVE_COST);
 906   format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
 907   ins_encode %{
 908     __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ $2,
 909          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 910   %}
 911   ins_pipe(pipe_slow);
 912 %}')dnl
 913 dnl
 914 // vector fmls
 915 VFMLS(F, S, 4)
 916 VFMLS(D, D, 2)
 917 
 918 dnl
 919 dnl VFNMLA($1           $2    $3         )
 920 dnl VFNMLA(name_suffix, size, min_vec_len)
 921 define(`VFNMLA', `
 922 // dst_src1 = -dst_src1 + -src2 * src3
 923 // dst_src1 = -dst_src1 + src2 * -src3
 924 instruct vfnmla$1(vReg dst_src1, vReg src2, vReg src3) %{
 925   predicate(UseFMA && UseSVE > 0);
 926   match(Set dst_src1 (FmaV$1 (NegV$1 dst_src1) (Binary (NegV$1 src2) src3)));
 927   match(Set dst_src1 (FmaV$1 (NegV$1 dst_src1) (Binary src2 (NegV$1 src3))));
 928   ins_cost(SVE_COST);
 929   format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
 930   ins_encode %{
 931     __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ $2,
 932          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 933   %}
 934   ins_pipe(pipe_slow);
 935 %}')dnl
 936 dnl
 937 // vector fnmla
 938 VFNMLA(F, S, 4)
 939 VFNMLA(D, D, 2)
 940 
 941 dnl
 942 dnl VFNMLS($1           $2    $3         )
 943 dnl VFNMLS(name_suffix, size, min_vec_len)
 944 define(`VFNMLS', `
 945 // dst_src1 = -dst_src1 + src2 * src3
 946 instruct vfnmls$1(vReg dst_src1, vReg src2, vReg src3) %{
 947   predicate(UseFMA && UseSVE > 0);
 948   match(Set dst_src1 (FmaV$1 (NegV$1 dst_src1) (Binary src2 src3)));
 949   ins_cost(SVE_COST);
 950   format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
 951   ins_encode %{
 952     __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ $2,
 953          ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 954   %}
 955   ins_pipe(pipe_slow);
 956 %}')dnl
 957 dnl
 958 // vector fnmls
 959 VFNMLS(F, S, 4)
 960 VFNMLS(D, D, 2)
 961 
 962 dnl
 963 dnl VMLA($1           $2    $3         )
 964 dnl VMLA(name_suffix, size, min_vec_len)
 965 define(`VMLA', `
 966 // dst_src1 = dst_src1 + src2 * src3
 967 instruct vmla$1(vReg dst_src1, vReg src2, vReg src3)
 968 %{
 969   predicate(UseSVE > 0);
 970   match(Set dst_src1 (AddV$1 dst_src1 (MulV$1 src2 src3)));
 971   ins_cost(SVE_COST);
 972   format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) ($2)" %}
 973   ins_encode %{
 974     __ sve_mla(as_FloatRegister($dst_src1$$reg), __ $2,
 975       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
 976   %}
 977   ins_pipe(pipe_slow);
 978 %}')dnl
 979 dnl
 980 // vector mla
 981 VMLA(B, B, 16)
 982 VMLA(S, H, 8)
 983 VMLA(I, S, 4)
 984 VMLA(L, D, 2)
 985 
 986 dnl
 987 dnl VMLS($1           $2    $3         )
 988 dnl VMLS(name_suffix, size, min_vec_len)
 989 define(`VMLS', `
 990 // dst_src1 = dst_src1 - src2 * src3
 991 instruct vmls$1(vReg dst_src1, vReg src2, vReg src3)
 992 %{
 993   predicate(UseSVE > 0);
 994   match(Set dst_src1 (SubV$1 dst_src1 (MulV$1 src2 src3)));
 995   ins_cost(SVE_COST);
 996   format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) ($2)" %}
 997   ins_encode %{
 998     __ sve_mls(as_FloatRegister($dst_src1$$reg), __ $2,
 999       ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1000   %}
1001   ins_pipe(pipe_slow);
1002 %}')dnl
1003 dnl
1004 // vector mls
1005 VMLS(B, B, 16)
1006 VMLS(S, H, 8)
1007 VMLS(I, S, 4)
1008 VMLS(L, D, 2)
1009 
1010 dnl
1011 dnl BINARY_OP_TRUE_PREDICATE($1,        $2,      $3,   $4,          $5  )
1012 dnl BINARY_OP_TRUE_PREDICATE(insn_name, op_name, size, min_vec_len, insn)
1013 define(`BINARY_OP_TRUE_PREDICATE', `
1014 instruct $1(vReg dst_src1, vReg src2) %{
1015   predicate(UseSVE > 0);
1016   match(Set dst_src1 ($2 dst_src1 src2));
1017   ins_cost(SVE_COST);
1018   format %{ "$5 $dst_src1, $dst_src1, $src2\t # vector (sve) ($3)" %}
1019   ins_encode %{
1020     __ $5(as_FloatRegister($dst_src1$$reg), __ $3,
1021          ptrue, as_FloatRegister($src2$$reg));
1022   %}
1023   ins_pipe(pipe_slow);
1024 %}')dnl
1025 dnl
1026 // vector mul
1027 BINARY_OP_TRUE_PREDICATE(vmulB, MulVB, B, 16, sve_mul)
1028 BINARY_OP_TRUE_PREDICATE(vmulS, MulVS, H, 8,  sve_mul)
1029 BINARY_OP_TRUE_PREDICATE(vmulI, MulVI, S, 4,  sve_mul)
1030 BINARY_OP_TRUE_PREDICATE(vmulL, MulVL, D, 2,  sve_mul)
1031 BINARY_OP_UNPREDICATE(vmulF, MulVF, S, 4, sve_fmul)
1032 BINARY_OP_UNPREDICATE(vmulD, MulVD, D, 2, sve_fmul)
1033 
1034 // vector mul - predicated
1035 BINARY_OP_PREDICATE(vmulB, MulVB, B, sve_mul)
1036 BINARY_OP_PREDICATE(vmulS, MulVS, H, sve_mul)
1037 BINARY_OP_PREDICATE(vmulI, MulVI, S, sve_mul)
1038 BINARY_OP_PREDICATE(vmulL, MulVL, D, sve_mul)
1039 BINARY_OP_PREDICATE(vmulF, MulVF, S, sve_fmul)
1040 BINARY_OP_PREDICATE(vmulD, MulVD, D, sve_fmul)
1041 
1042 // vector fneg
1043 UNARY_OP_TRUE_PREDICATE(vnegF, NegVF, S, sve_fneg)
1044 UNARY_OP_TRUE_PREDICATE(vnegD, NegVD, D, sve_fneg)
1045 
1046 // vector fneg - predicated
1047 UNARY_OP_PREDICATE(vnegF, NegVF, S, sve_fneg)
1048 UNARY_OP_PREDICATE(vnegD, NegVD, D, sve_fneg)
1049 
1050 // popcount vector
1051 
1052 instruct vpopcountI(vReg dst, vReg src) %{
1053   predicate(UseSVE > 0);
1054   match(Set dst (PopCountVI src));
1055   format %{ "sve_cnt $dst, $src\t# vector (sve) (S)\n\t" %}
1056   ins_encode %{
1057      __ sve_cnt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg));
1058   %}
1059   ins_pipe(pipe_slow);
1060 %}
1061 
1062 // vector mask compare
1063 
1064 instruct vmaskcmp(pRegGov dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %{
1065   predicate(UseSVE > 0);
1066   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
1067   effect(KILL cr);
1068   ins_cost(SVE_COST);
1069   format %{ "sve_cmp $dst, $src1, $src2\t# vector mask cmp (sve)" %}
1070   ins_encode %{
1071     BasicType bt = Matcher::vector_element_basic_type(this);
1072     __ sve_compare(as_PRegister($dst$$reg), bt, ptrue, as_FloatRegister($src1$$reg),
1073                    as_FloatRegister($src2$$reg), (int)$cond$$constant);
1074   %}
1075   ins_pipe(pipe_slow);
1076 %}
1077 
1078 instruct vmaskcmp_masked(pRegGov dst, vReg src1, vReg src2, immI cond, pRegGov pg, rFlagsReg cr) %{
1079   predicate(UseSVE > 0);
1080   match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond pg)));
1081   effect(KILL cr);
1082   ins_cost(SVE_COST);
1083   format %{ "sve_cmp $dst, $pg, $src1, $src2\t# vector mask cmp (sve)" %}
1084   ins_encode %{
1085     BasicType bt = Matcher::vector_element_basic_type(this);
1086     __ sve_compare(as_PRegister($dst$$reg), bt, as_PRegister($pg$$reg), as_FloatRegister($src1$$reg),
1087                    as_FloatRegister($src2$$reg), (int)$cond$$constant);
1088   %}
1089   ins_pipe(pipe_slow);
1090 %}
1091 
1092 // vector blend
1093 
1094 instruct vblend(vReg dst, vReg src1, vReg src2, pRegGov pg) %{
1095   predicate(UseSVE > 0);
1096   match(Set dst (VectorBlend (Binary src1 src2) pg));
1097   ins_cost(SVE_COST);
1098   format %{ "sve_sel $dst, $pg, $src2, $src1\t# vector blend (sve)" %}
1099   ins_encode %{
1100     Assembler::SIMD_RegVariant size =
1101                __ elemType_to_regVariant(Matcher::vector_element_basic_type(this));
1102     __ sve_sel(as_FloatRegister($dst$$reg), size, as_PRegister($pg$$reg),
1103                as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
1104   %}
1105   ins_pipe(pipe_slow);
1106 %}
1107 
1108 // vector load mask
1109 
1110 instruct vloadmaskB(pRegGov dst, vReg src, rFlagsReg cr) %{
1111   predicate(UseSVE > 0 &&
1112             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
1113   match(Set dst (VectorLoadMask src));
1114   effect(KILL cr);
1115   ins_cost(SVE_COST);
1116   format %{ "vloadmaskB $dst, $src\t# vector load mask (sve) (B)" %}
1117   ins_encode %{
1118     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ B,
1119                ptrue, as_FloatRegister($src$$reg), 0);
1120   %}
1121   ins_pipe(pipe_slow);
1122 %}
1123 
1124 instruct vloadmask_extend(pRegGov dst, vReg src, vReg tmp, rFlagsReg cr) %{
1125   predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() != T_BYTE);
1126   match(Set dst (VectorLoadMask src));
1127   effect(TEMP tmp, KILL cr);
1128   ins_cost(3 * SVE_COST);
1129   format %{ "vloadmask $dst, $src\t# vector load mask (sve) (H/S/D)" %}
1130   ins_encode %{
1131     BasicType bt = Matcher::vector_element_basic_type(this);
1132     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1133     __ sve_vector_extend(as_FloatRegister($tmp$$reg), size, as_FloatRegister($src$$reg), __ B);
1134     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
1135   %}
1136   ins_pipe(pipe_slow);
1137 %}
1138 
1139 // vector store mask
1140 
1141 instruct vstoremaskB(vReg dst, pRegGov src, immI_1 size) %{
1142   predicate(UseSVE > 0);
1143   match(Set dst (VectorStoreMask src size));
1144   ins_cost(SVE_COST);
1145   format %{ "vstoremask $dst, $src\t# vector store mask (sve) (B)" %}
1146   ins_encode %{
1147     __ sve_cpy(as_FloatRegister($dst$$reg), __ B, as_PRegister($src$$reg), 1, false);
1148   %}
1149   ins_pipe(pipe_slow);
1150 %}
1151 
1152 instruct vstoremask_narrow(vReg dst, pRegGov src, vReg tmp, immI_gt_1 size) %{
1153   predicate(UseSVE > 0);
1154   match(Set dst (VectorStoreMask src size));
1155   effect(TEMP_DEF dst, TEMP tmp);
1156   ins_cost(3 * SVE_COST);
1157   format %{ "vstoremask $dst, $src\t# vector store mask (sve) (H/S/D)" %}
1158   ins_encode %{
1159     Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant((int)$size$$constant);
1160     __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($src$$reg), 1, false);
1161     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
1162                          as_FloatRegister($dst$$reg), size, as_FloatRegister($tmp$$reg));
1163   %}
1164   ins_pipe(pipe_slow);
1165 %}
1166 
1167 // Combine LoadVector+VectorLoadMask when the vector element type is not T_BYTE
1168 
1169 instruct vloadmask_loadV(pRegGov dst, indirect mem, vReg tmp, rFlagsReg cr) %{
1170   predicate(UseSVE > 0 &&
1171             n->as_Vector()->length_in_bytes() == MaxVectorSize &&
1172             type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
1173   match(Set dst (VectorLoadMask (LoadVector mem)));
1174   effect(TEMP tmp, KILL cr);
1175   ins_cost(3 * SVE_COST);
1176   format %{ "sve_ld1b $tmp, $mem\n\t"
1177             "sve_cmpne $dst, $tmp, 0\t# load vector mask (sve) (H/S/D)" %}
1178   ins_encode %{
1179     // Load mask values which are boolean type, and extend them to the
1180     // expected vector element type. Convert the vector to predicate.
1181     BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
1182     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($tmp$$reg),
1183                           ptrue, T_BOOLEAN, to_vect_bt, $mem->opcode(),
1184                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1185     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ elemType_to_regVariant(to_vect_bt),
1186                ptrue, as_FloatRegister($tmp$$reg), 0);
1187   %}
1188   ins_pipe(pipe_slow);
1189 %}
1190 
1191 instruct vloadmask_loadV_partial(pRegGov dst, indirect mem, vReg vtmp, pRegGov ptmp, rFlagsReg cr) %{
1192   predicate(UseSVE > 0 &&
1193             n->as_Vector()->length_in_bytes() > 16 &&
1194             n->as_Vector()->length_in_bytes() < MaxVectorSize &&
1195             type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
1196   match(Set dst (VectorLoadMask (LoadVector mem)));
1197   effect(TEMP vtmp, TEMP ptmp, KILL cr);
1198   ins_cost(6 * SVE_COST);
1199   format %{ "vloadmask_loadV $dst, $mem\t# load vector mask partial (sve) (H/S/D)" %}
1200   ins_encode %{
1201     // Load valid mask values which are boolean type, and extend them to the
1202     // expected vector element type. Convert the vector to predicate.
1203     BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
1204     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(to_vect_bt);
1205     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this));
1206     loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($vtmp$$reg),
1207                           as_PRegister($ptmp$$reg), T_BOOLEAN, to_vect_bt, $mem->opcode(),
1208                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1209     __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($vtmp$$reg), 0);
1210   %}
1211   ins_pipe(pipe_slow);
1212 %}
1213 
1214 // Combine VectorStoreMask+StoreVector when the vector element type is not T_BYTE
1215 
1216 instruct storeV_vstoremask(indirect mem, pRegGov src, vReg tmp, immI_gt_1 esize) %{
1217   predicate(UseSVE > 0 &&
1218             Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
1219   match(Set mem (StoreVector mem (VectorStoreMask src esize)));
1220   effect(TEMP tmp);
1221   ins_cost(3 * SVE_COST);
1222   format %{ "sve_cpy $tmp, $src, 1\n\t"
1223             "sve_st1b $tmp, $mem\t# store vector mask (sve) (H/S/D)" %}
1224   ins_encode %{
1225     BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
1226     assert(type2aelembytes(from_vect_bt) == (int)$esize$$constant, "unsupported type.");
1227     Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
1228     __ sve_cpy(as_FloatRegister($tmp$$reg), size, as_PRegister($src$$reg), 1, false);
1229     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($tmp$$reg),
1230                           ptrue, T_BOOLEAN, from_vect_bt, $mem->opcode(),
1231                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1232   %}
1233   ins_pipe(pipe_slow);
1234 %}
1235 
1236 instruct storeV_vstoremask_partial(indirect mem, pRegGov src, vReg vtmp,
1237                                    immI_gt_1 esize, pRegGov ptmp, rFlagsReg cr) %{
1238   predicate(UseSVE > 0 &&
1239             n->as_StoreVector()->memory_size() > 16 &&
1240             type2aelembytes(n->as_StoreVector()->vect_type()->element_basic_type()) > 1 &&
1241             Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
1242   match(Set mem (StoreVector mem (VectorStoreMask src esize)));
1243   effect(TEMP vtmp, TEMP ptmp, KILL cr);
1244   format %{ "storeV_vstoremask $src, $mem\t# store vector mask partial (sve) (H/S/D)" %}
1245   ins_cost(6 * SVE_COST);
1246   ins_encode %{
1247     // Convert the valid src predicate to vector, and store the vector
1248     // elements as boolean values.
1249     BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
1250     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(from_vect_bt);
1251     __ sve_cpy(as_FloatRegister($vtmp$$reg), size, as_PRegister($src$$reg), 1, false);
1252     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
1253     loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($vtmp$$reg),
1254                           as_PRegister($ptmp$$reg), T_BOOLEAN, from_vect_bt, $mem->opcode(),
1255                           as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1256   %}
1257   ins_pipe(pipe_slow);
1258 %}
1259 dnl
1260 dnl REDUCE_I($1,        $2     )
1261 dnl REDUCE_I(insn_name, op_name)
1262 define(`REDUCE_I', `
1263 instruct reduce_$1I(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
1264   ifelse($2, AddReductionVI,
1265        `predicate(UseSVE > 0 &&
1266             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);',
1267        `predicate(UseSVE > 0 &&
1268             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1269             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);')
1270   match(Set dst ($2 src1 src2));
1271   effect(TEMP_DEF dst, TEMP tmp);
1272   ins_cost(SVE_COST);
1273   format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction (sve) (may extend)" %}
1274   ins_encode %{
1275     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1276     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1277                            $src1$$Register, as_FloatRegister($src2$$reg),
1278                            ptrue, as_FloatRegister($tmp$$reg));
1279   %}
1280   ins_pipe(pipe_slow);
1281 %}')dnl
1282 dnl
1283 dnl
1284 dnl REDUCE_L($1,        $2    )
1285 dnl REDUCE_L(insn_name, op_name)
1286 define(`REDUCE_L', `
1287 instruct reduce_$1L(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
1288   ifelse($2, AddReductionVL,
1289        `predicate(UseSVE > 0 &&
1290             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);',
1291        `predicate(UseSVE > 0 &&
1292             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1293             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);')
1294   match(Set dst ($2 src1 src2));
1295   effect(TEMP_DEF dst, TEMP tmp);
1296   ins_cost(SVE_COST);
1297   format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction (sve)" %}
1298   ins_encode %{
1299     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1300                            $src1$$Register, as_FloatRegister($src2$$reg),
1301                            ptrue, as_FloatRegister($tmp$$reg));
1302   %}
1303   ins_pipe(pipe_slow);
1304 %}')dnl
1305 dnl
1306 dnl REDUCE_I_PARTIAL($1,        $2     )
1307 dnl REDUCE_I_PARTIAL(insn_name, op_name)
1308 define(`REDUCE_I_PARTIAL', `
1309 instruct reduce_$1I_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1310                              pRegGov ptmp, rFlagsReg cr) %{
1311   ifelse($2, AddReductionVI,
1312        `predicate(UseSVE > 0 &&
1313             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);',
1314        `predicate(UseSVE > 0 &&
1315             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1316             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);')
1317   match(Set dst ($2 src1 src2));
1318   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1319   ins_cost(2 * SVE_COST);
1320   format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction partial (sve) (may extend)" %}
1321   ins_encode %{
1322     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1323     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1324     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1325                           Matcher::vector_length(this, $src2));
1326     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1327                            $src1$$Register, as_FloatRegister($src2$$reg),
1328                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1329   %}
1330   ins_pipe(pipe_slow);
1331 %}')dnl
1332 dnl
1333 dnl REDUCE_L_PARTIAL($1,        $2    )
1334 dnl REDUCE_L_PARTIAL(insn_name, op_name)
1335 define(`REDUCE_L_PARTIAL', `
1336 instruct reduce_$1L_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1337                              pRegGov ptmp, rFlagsReg cr) %{
1338   ifelse($2, AddReductionVL,
1339        `predicate(UseSVE > 0 &&
1340             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);',
1341        `predicate(UseSVE > 0 &&
1342             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1343             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);')
1344   match(Set dst ($2 src1 src2));
1345   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1346   ins_cost(2 * SVE_COST);
1347   format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction partial (sve)" %}
1348   ins_encode %{
1349     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1350                           Matcher::vector_length(this, $src2));
1351     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1352                            $src1$$Register, as_FloatRegister($src2$$reg),
1353                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1354   %}
1355   ins_pipe(pipe_slow);
1356 %}')dnl
1357 dnl
1358 dnl REDUCE_ADDF($1,        $2,      $3,      $4  )
1359 dnl REDUCE_ADDF(insn_name, op_name, reg_dst, size)
1360 define(`REDUCE_ADDF', `
1361 instruct reduce_$1($3 src1_dst, vReg src2) %{
1362   predicate(UseSVE > 0 &&
1363             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1364   match(Set src1_dst ($2 src1_dst src2));
1365   ins_cost(SVE_COST);
1366   format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) ($4)" %}
1367   ins_encode %{
1368     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1369          ptrue, as_FloatRegister($src2$$reg));
1370   %}
1371   ins_pipe(pipe_slow);
1372 %}')dnl
1373 dnl
1374 dnl
1375 dnl REDUCE_ADDF_PARTIAL($1,        $2,     $3,      $4  )
1376 dnl REDUCE_ADDF_PARTIAL(insn_name, suffix, reg_dst, size)
1377 define(`REDUCE_ADDF_PARTIAL', `
1378 instruct reduce_$1_partial($3 src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
1379   predicate(UseSVE > 0 &&
1380             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1381   match(Set src1_dst ($2 src1_dst src2));
1382   ins_cost(SVE_COST);
1383   effect(TEMP ptmp, KILL cr);
1384   format %{ "sve_reduce_$1 $src1_dst, $src1_dst, $src2\t# $1 reduction partial (sve) ($4)" %}
1385   ins_encode %{
1386     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
1387                           Matcher::vector_length(this, $src2));
1388     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1389                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1390   %}
1391   ins_pipe(pipe_slow);
1392 %}')dnl
1393 dnl
1394 dnl
1395 dnl REDUCE_I_PREDICATE($1,        $2     )
1396 dnl REDUCE_I_PREDICATE(insn_name, op_name)
1397 define(`REDUCE_I_PREDICATE', `
1398 instruct reduce_$1I_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
1399   ifelse($2, AddReductionVI,
1400        `predicate(UseSVE > 0 &&
1401             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);',
1402        `predicate(UseSVE > 0 &&
1403             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1404             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);')
1405   match(Set dst ($2 (Binary src1 src2) pg));
1406   effect(TEMP_DEF dst, TEMP tmp);
1407   ins_cost(SVE_COST);
1408   format %{ "sve_reduce_$1I $dst, $src1, $pg, $src2\t# $1I reduction predicated (sve) (may extend)" %}
1409   ins_encode %{
1410     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1411     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1412                            $src1$$Register, as_FloatRegister($src2$$reg),
1413                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
1414   %}
1415   ins_pipe(pipe_slow);
1416 %}')dnl
1417 dnl
1418 dnl REDUCE_L_PREDICATE($1,        $2    )
1419 dnl REDUCE_L_PREDICATE(insn_name, op_name)
1420 define(`REDUCE_L_PREDICATE', `
1421 instruct reduce_$1L_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
1422   ifelse($2, AddReductionVL,
1423        `predicate(UseSVE > 0 &&
1424             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);',
1425        `predicate(UseSVE > 0 &&
1426             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1427             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);')
1428   match(Set dst ($2 (Binary src1 src2) pg));
1429   effect(TEMP_DEF dst, TEMP tmp);
1430   ins_cost(SVE_COST);
1431   format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated (sve)" %}
1432   ins_encode %{
1433     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1434                            $src1$$Register, as_FloatRegister($src2$$reg),
1435                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
1436   %}
1437   ins_pipe(pipe_slow);
1438 %}')dnl
1439 dnl
1440 dnl REDUCE_I_PREDICATE_PARTIAL($1,        $2     )
1441 dnl REDUCE_I_PREDICATE_PARTIAL(insn_name, op_name)
1442 define(`REDUCE_I_PREDICATE_PARTIAL', `
1443 instruct reduce_$1I_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1444                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
1445   ifelse($2, AddReductionVI,
1446        `predicate(UseSVE > 0 &&
1447             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);',
1448        `predicate(UseSVE > 0 &&
1449             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1450             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);')
1451   match(Set dst ($2 (Binary src1 src2) pg));
1452   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1453   ins_cost(3 * SVE_COST);
1454   format %{ "sve_reduce_$1I $dst, $src1, $pg, $src2\t# $1I reduction predicated partial (sve) (may extend)" %}
1455   ins_encode %{
1456     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1457     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1458     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1459                           Matcher::vector_length(this, $src2));
1460     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1461                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1462     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1463                            $src1$$Register, as_FloatRegister($src2$$reg),
1464                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1465   %}
1466   ins_pipe(pipe_slow);
1467 %}')dnl
1468 dnl
1469 dnl REDUCE_L_PREDICATE_PARTIAL($1,        $2    )
1470 dnl REDUCE_L_PREDICATE_PARTIAL(insn_name, op_name)
1471 define(`REDUCE_L_PREDICATE_PARTIAL', `
1472 instruct reduce_$1L_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1473                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
1474   ifelse($2, AddReductionVL,
1475        `predicate(UseSVE > 0 &&
1476             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);',
1477        `predicate(UseSVE > 0 &&
1478             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1479             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);')
1480   match(Set dst ($2 (Binary src1 src2) pg));
1481   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1482   ins_cost(3 * SVE_COST);
1483   format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated partial (sve)" %}
1484   ins_encode %{
1485     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1486                           Matcher::vector_length(this, $src2));
1487     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1488                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1489     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1490                            $src1$$Register, as_FloatRegister($src2$$reg),
1491                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1492   %}
1493   ins_pipe(pipe_slow);
1494 %}')dnl
1495 dnl
1496 dnl REDUCE_ADDF_PREDICATE($1,        $2,      $3,      $4  )
1497 dnl REDUCE_ADDF_PREDICATE(insn_name, op_name, reg_dst, size)
1498 define(`REDUCE_ADDF_PREDICATE', `
1499 instruct reduce_$1_masked($3 src1_dst, vReg src2, pRegGov pg) %{
1500   predicate(UseSVE > 0 &&
1501             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1502   match(Set src1_dst ($2 (Binary src1_dst src2) pg));
1503   ins_cost(SVE_COST);
1504   format %{ "sve_reduce_$1 $src1_dst, $pg, $src2\t# $1 reduction predicated (sve)" %}
1505   ins_encode %{
1506     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1507                  as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1508   %}
1509   ins_pipe(pipe_slow);
1510 %}')dnl
1511 dnl
1512 dnl REDUCE_ADDF_PREDICATE_PARTIAL($1,        $2,      $3,      $4  )
1513 dnl REDUCE_ADDF_PREDICATE_PARTIAL(insn_name, op_name, reg_dst, size)
1514 define(`REDUCE_ADDF_PREDICATE_PARTIAL', `
1515 instruct reduce_$1_masked_partial($3 src1_dst, vReg src2, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
1516   predicate(UseSVE > 0 &&
1517             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1518   match(Set src1_dst ($2 (Binary src1_dst src2) pg));
1519   effect(TEMP ptmp, KILL cr);
1520   ins_cost(SVE_COST);
1521   format %{ "sve_reduce_$1 $src1_dst, $pg, $src2\t# $1 reduction predicated partial (sve)" %}
1522   ins_encode %{
1523     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
1524                           Matcher::vector_length(this, $src2));
1525     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1526                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1527     __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1528                  as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1529   %}
1530   ins_pipe(pipe_slow);
1531 %}')dnl
1532 dnl
1533 
1534 // vector add reduction
1535 REDUCE_I(add, AddReductionVI)
1536 REDUCE_L(add, AddReductionVL)
1537 REDUCE_ADDF(addF, AddReductionVF, vRegF, S)
1538 REDUCE_ADDF(addD, AddReductionVD, vRegD, D)
1539 REDUCE_I_PARTIAL(add, AddReductionVI)
1540 REDUCE_L_PARTIAL(add, AddReductionVL)
1541 REDUCE_ADDF_PARTIAL(addF, AddReductionVF, vRegF, S)
1542 REDUCE_ADDF_PARTIAL(addD, AddReductionVD, vRegD, D)
1543 
1544 // vector add reduction - predicated
1545 REDUCE_I_PREDICATE(add, AddReductionVI)
1546 REDUCE_L_PREDICATE(add, AddReductionVL)
1547 REDUCE_ADDF_PREDICATE(addF, AddReductionVF, vRegF, S)
1548 REDUCE_ADDF_PREDICATE(addD, AddReductionVD, vRegD, D)
1549 REDUCE_I_PREDICATE_PARTIAL(add, AddReductionVI)
1550 REDUCE_L_PREDICATE_PARTIAL(add, AddReductionVL)
1551 REDUCE_ADDF_PREDICATE_PARTIAL(addF, AddReductionVF, vRegF, S)
1552 REDUCE_ADDF_PREDICATE_PARTIAL(addD, AddReductionVD, vRegD, D)
1553 
1554 // vector and reduction
1555 REDUCE_I(and, AndReductionV)
1556 REDUCE_L(and, AndReductionV)
1557 REDUCE_I_PARTIAL(and, AndReductionV)
1558 REDUCE_L_PARTIAL(and, AndReductionV)
1559 
1560 // vector and reduction - predicated
1561 REDUCE_I_PREDICATE(and, AndReductionV)
1562 REDUCE_L_PREDICATE(and, AndReductionV)
1563 REDUCE_I_PREDICATE_PARTIAL(and, AndReductionV)
1564 REDUCE_L_PREDICATE_PARTIAL(and, AndReductionV)
1565 
1566 // vector or reduction
1567 REDUCE_I(or, OrReductionV)
1568 REDUCE_L(or, OrReductionV)
1569 REDUCE_I_PARTIAL(or, OrReductionV)
1570 REDUCE_L_PARTIAL(or, OrReductionV)
1571 
1572 // vector or reduction - predicated
1573 REDUCE_I_PREDICATE(or, OrReductionV)
1574 REDUCE_L_PREDICATE(or, OrReductionV)
1575 REDUCE_I_PREDICATE_PARTIAL(or, OrReductionV)
1576 REDUCE_L_PREDICATE_PARTIAL(or, OrReductionV)
1577 
1578 // vector xor reduction
1579 REDUCE_I(eor, XorReductionV)
1580 REDUCE_L(eor, XorReductionV)
1581 REDUCE_I_PARTIAL(eor, XorReductionV)
1582 REDUCE_L_PARTIAL(eor, XorReductionV)
1583 
1584 // vector xor reduction - predicated
1585 REDUCE_I_PREDICATE(eor, XorReductionV)
1586 REDUCE_L_PREDICATE(eor, XorReductionV)
1587 REDUCE_I_PREDICATE_PARTIAL(eor, XorReductionV)
1588 REDUCE_L_PREDICATE_PARTIAL(eor, XorReductionV)
1589 
1590 dnl
1591 dnl REDUCE_MAXMIN_I($1,        $2     )
1592 dnl REDUCE_MAXMIN_I(insn_name, op_name)
1593 define(`REDUCE_MAXMIN_I', `
1594 instruct reduce_$1I(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
1595   predicate(UseSVE > 0 &&
1596             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1597             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1598             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
1599   match(Set dst ($2 src1 src2));
1600   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1601   ins_cost(SVE_COST);
1602   format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction (sve)" %}
1603   ins_encode %{
1604     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1605     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1606                            $src1$$Register, as_FloatRegister($src2$$reg),
1607                            ptrue, as_FloatRegister($tmp$$reg));
1608   %}
1609   ins_pipe(pipe_slow);
1610 %}')dnl
1611 dnl
1612 dnl REDUCE_MAXMIN_L($1,        $2     )
1613 dnl REDUCE_MAXMIN_L(insn_name, op_name)
1614 define(`REDUCE_MAXMIN_L', `
1615 instruct reduce_$1L(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
1616   predicate(UseSVE > 0 &&
1617             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1618             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1619   match(Set dst ($2 src1 src2));
1620   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1621   ins_cost(SVE_COST);
1622   format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction (sve)" %}
1623   ins_encode %{
1624     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1625                            $src1$$Register, as_FloatRegister($src2$$reg),
1626                            ptrue, as_FloatRegister($tmp$$reg));
1627   %}
1628   ins_pipe(pipe_slow);
1629 %}')dnl
1630 dnl
1631 dnl REDUCE_MAXMIN_I_PARTIAL($1     , $2     )
1632 dnl REDUCE_MAXMIN_I_PARTIAL(min_max, op_name)
1633 define(`REDUCE_MAXMIN_I_PARTIAL', `
1634 instruct reduce_$1I_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1635                              pRegGov ptmp, rFlagsReg cr) %{
1636   predicate(UseSVE > 0 &&
1637             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1638             n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1639             is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
1640   match(Set dst ($2 src1 src2));
1641   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1642   ins_cost(2 * SVE_COST);
1643   format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction partial (sve)" %}
1644   ins_encode %{
1645     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1646     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1647     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1648                           Matcher::vector_length(this, $src2));
1649     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1650                            $src1$$Register, as_FloatRegister($src2$$reg),
1651                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1652   %}
1653   ins_pipe(pipe_slow);
1654 %}')dnl
1655 dnl
1656 dnl REDUCE_MAXMIN_L_PARTIAL($1     , $2     )
1657 dnl REDUCE_MAXMIN_L_PARTIAL(min_max, op_name)
1658 define(`REDUCE_MAXMIN_L_PARTIAL', `
1659 instruct reduce_$1L_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1660                              pRegGov ptmp, rFlagsReg cr) %{
1661   predicate(UseSVE > 0 &&
1662             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1663             n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1664   match(Set dst ($2 src1 src2));
1665   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1666   ins_cost(2 * SVE_COST);
1667   format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction  partial (sve)" %}
1668   ins_encode %{
1669     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1670                           Matcher::vector_length(this, $src2));
1671     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1672                            $src1$$Register, as_FloatRegister($src2$$reg),
1673                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1674   %}
1675   ins_pipe(pipe_slow);
1676 %}')dnl
1677 dnl
1678 dnl REDUCE_MAXMIN_I_PREDICATE($1     , $2     )
1679 dnl REDUCE_MAXMIN_I_PREDICATE(min_max, op_name)
1680 define(`REDUCE_MAXMIN_I_PREDICATE', `
1681 instruct reduce_$1I_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp,
1682                            pRegGov pg, rFlagsReg cr) %{
1683   predicate(UseSVE > 0 &&
1684             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1685             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1686             is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
1687   match(Set dst ($2 (Binary src1 src2) pg));
1688   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1689   ins_cost(SVE_COST);
1690   format %{ "sve_reduce_$1I $dst, $src1, $pg, $src2\t# $1I reduction predicated (sve)" %}
1691   ins_encode %{
1692     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1693     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1694                            $src1$$Register, as_FloatRegister($src2$$reg),
1695                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
1696   %}
1697   ins_pipe(pipe_slow);
1698 %}')dnl
1699 dnl
1700 dnl REDUCE_MAXMIN_L_PREDICATE($1     , $2     )
1701 dnl REDUCE_MAXMIN_L_PREDICATE(min_max, op_name)
1702 define(`REDUCE_MAXMIN_L_PREDICATE', `
1703 instruct reduce_$1L_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp,
1704                           pRegGov pg, rFlagsReg cr) %{
1705   predicate(UseSVE > 0 &&
1706             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1707             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1708   match(Set dst ($2 (Binary src1 src2) pg));
1709   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1710   ins_cost(SVE_COST);
1711   format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated (sve)" %}
1712   ins_encode %{
1713     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1714                            $src1$$Register, as_FloatRegister($src2$$reg),
1715                            as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
1716   %}
1717   ins_pipe(pipe_slow);
1718 %}')dnl
1719 dnl
1720 dnl REDUCE_MAXMIN_I_PREDICATE_PARTIAL($1     , $2     )
1721 dnl REDUCE_MAXMIN_I_PREDICATE_PARTIAL(min_max, op_name)
1722 define(`REDUCE_MAXMIN_I_PREDICATE_PARTIAL', `
1723 instruct reduce_$1I_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1724                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
1725   predicate(UseSVE > 0 &&
1726             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1727             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1728             is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
1729   match(Set dst ($2 (Binary src1 src2) pg));
1730   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1731   ins_cost(3 * SVE_COST);
1732   format %{ "sve_reduce_$1I $dst, $src1, $pg, $src2\t# $1I reduction predicated partial (sve)" %}
1733   ins_encode %{
1734     BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1735     Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1736     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1737                           Matcher::vector_length(this, $src2));
1738     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1739                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1740     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1741                            $src1$$Register, as_FloatRegister($src2$$reg),
1742                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1743   %}
1744   ins_pipe(pipe_slow);
1745 %}')dnl
1746 dnl
1747 dnl REDUCE_MAXMIN_L_PREDICATE_PARTIAL($1     , $2     )
1748 dnl REDUCE_MAXMIN_L_PREDICATE_PARTIAL(min_max, op_name)
1749 define(`REDUCE_MAXMIN_L_PREDICATE_PARTIAL', `
1750 instruct reduce_$1L_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1751                                   pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
1752   predicate(UseSVE > 0 &&
1753             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1754             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1755   match(Set dst ($2 (Binary src1 src2) pg));
1756   effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1757   ins_cost(3 * SVE_COST);
1758   format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated partial (sve)" %}
1759   ins_encode %{
1760     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1761                           Matcher::vector_length(this, $src2));
1762     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1763                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1764     __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1765                            $src1$$Register, as_FloatRegister($src2$$reg),
1766                            as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1767   %}
1768   ins_pipe(pipe_slow);
1769 %}')dnl
1770 dnl
1771 dnl REDUCE_FMINMAX($1,      $2,          $3,           $4,   $5         )
1772 dnl REDUCE_FMINMAX(min_max, name_suffix, element_type, size, reg_src_dst)
1773 define(`REDUCE_FMINMAX', `
1774 instruct reduce_$1$2($5 dst, $5 src1, vReg src2) %{
1775   predicate(UseSVE > 0 &&
1776             n->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1777             n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1778   match(Set dst (translit($1, `m', `M')ReductionV src1 src2));
1779   ins_cost(INSN_COST);
1780   effect(TEMP_DEF dst);
1781   format %{ "sve_reduce_$1$2 $dst, $src1, $src2\t# $1$2 reduction (sve)" %}
1782   ins_encode %{
1783     __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src2$$reg));
1784     __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1785   %}
1786   ins_pipe(pipe_slow);
1787 %}')dnl
1788 dnl
1789 dnl REDUCE_FMINMAX_PARTIAL($1,      $2,          $3,           $4,   $5         )
1790 dnl REDUCE_FMINMAX_PARTIAL(min_max, name_suffix, element_type, size, reg_src_dst)
1791 define(`REDUCE_FMINMAX_PARTIAL', `
1792 instruct reduce_$1$2_partial($5 dst, $5 src1, vReg src2,
1793                              pRegGov ptmp, rFlagsReg cr) %{
1794   predicate(UseSVE > 0 &&
1795             n->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1796             n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1797   match(Set dst (translit($1, `m', `M')ReductionV src1 src2));
1798   ins_cost(INSN_COST);
1799   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
1800   format %{ "sve_reduce_$1$2 $dst, $src1, $src2\t# $1$2 reduction partial (sve)" %}
1801   ins_encode %{
1802     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
1803                           Matcher::vector_length(this, $src2));
1804     __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1805     __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1806   %}
1807   ins_pipe(pipe_slow);
1808 %}')dnl
1809 dnl
1810 dnl REDUCE_FMINMAX_PREDICATE($1,      $2,          $3,           $4,   $5         )
1811 dnl REDUCE_FMINMAX_PREDICATE(min_max, name_suffix, element_type, size, reg_src_dst)
1812 define(`REDUCE_FMINMAX_PREDICATE', `
1813 instruct reduce_$1$2_masked($5 dst, $5 src1, vReg src2, pRegGov pg) %{
1814   predicate(UseSVE > 0 &&
1815             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1816             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1817   match(Set dst (translit($1, `m', `M')ReductionV (Binary src1 src2) pg));
1818   ins_cost(SVE_COST);
1819   effect(TEMP_DEF dst);
1820   format %{ "sve_reduce_$1$2 $dst, $src1, $pg, $src2\t# $1$2 reduction predicated (sve)" %}
1821   ins_encode %{
1822     __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1823     __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1824   %}
1825   ins_pipe(pipe_slow);
1826 %}')dnl
1827 dnl
1828 dnl REDUCE_FMINMAX_PREDICATE_PARTIAL($1,      $2,          $3,           $4,   $5         )
1829 dnl REDUCE_FMINMAX_PREDICATE_PARTIAL(min_max, name_suffix, element_type, size, reg_src_dst)
1830 define(`REDUCE_FMINMAX_PREDICATE_PARTIAL', `
1831 instruct reduce_$1$2_masked_partial($5 dst, $5 src1, vReg src2, pRegGov pg,
1832                                     pRegGov ptmp, rFlagsReg cr) %{
1833   predicate(UseSVE > 0 &&
1834             n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1835             n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1836   match(Set dst (translit($1, `m', `M')ReductionV (Binary src1 src2) pg));
1837   effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
1838   ins_cost(3 * SVE_COST);
1839   format %{ "sve_reduce_$1$2 $dst, $src1, $pg, $src2\t# $1$2 reduction predicated partial (sve)" %}
1840   ins_encode %{
1841     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
1842                           Matcher::vector_length(this, $src2));
1843     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1844                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1845     __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4,
1846                as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1847     __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1848   %}
1849   ins_pipe(pipe_slow);
1850 %}')dnl
1851 // vector max reduction
1852 REDUCE_MAXMIN_I(max, MaxReductionV)
1853 REDUCE_MAXMIN_L(max, MaxReductionV)
1854 REDUCE_MAXMIN_I_PARTIAL(max, MaxReductionV)
1855 REDUCE_MAXMIN_L_PARTIAL(max, MaxReductionV)
1856 REDUCE_FMINMAX(max, F, T_FLOAT,  S, vRegF)
1857 REDUCE_FMINMAX_PARTIAL(max, F, T_FLOAT,  S, vRegF)
1858 REDUCE_FMINMAX(max, D, T_DOUBLE, D, vRegD)
1859 REDUCE_FMINMAX_PARTIAL(max, D, T_DOUBLE, D, vRegD)
1860 
1861 // vector max reduction - predicated
1862 REDUCE_MAXMIN_I_PREDICATE(max, MaxReductionV)
1863 REDUCE_MAXMIN_L_PREDICATE(max, MaxReductionV)
1864 REDUCE_MAXMIN_I_PREDICATE_PARTIAL(max, MaxReductionV)
1865 REDUCE_MAXMIN_L_PREDICATE_PARTIAL(max, MaxReductionV)
1866 REDUCE_FMINMAX_PREDICATE(max, F, T_FLOAT,  S, vRegF)
1867 REDUCE_FMINMAX_PREDICATE(max, D, T_DOUBLE, D, vRegD)
1868 REDUCE_FMINMAX_PREDICATE_PARTIAL(max, F, T_FLOAT,  S, vRegF)
1869 REDUCE_FMINMAX_PREDICATE_PARTIAL(max, D, T_DOUBLE, D, vRegD)
1870 
1871 // vector min reduction
1872 REDUCE_MAXMIN_I(min, MinReductionV)
1873 REDUCE_MAXMIN_L(min, MinReductionV)
1874 REDUCE_MAXMIN_I_PARTIAL(min, MinReductionV)
1875 REDUCE_MAXMIN_L_PARTIAL(min, MinReductionV)
1876 REDUCE_FMINMAX(min, F, T_FLOAT,  S, vRegF)
1877 REDUCE_FMINMAX_PARTIAL(min, F, T_FLOAT,  S, vRegF)
1878 REDUCE_FMINMAX(min, D, T_DOUBLE, D, vRegD)
1879 REDUCE_FMINMAX_PARTIAL(min, D, T_DOUBLE, D, vRegD)
1880 
1881 // vector min reduction - predicated
1882 REDUCE_MAXMIN_I_PREDICATE(min, MinReductionV)
1883 REDUCE_MAXMIN_L_PREDICATE(min, MinReductionV)
1884 REDUCE_MAXMIN_I_PREDICATE_PARTIAL(min, MinReductionV)
1885 REDUCE_MAXMIN_L_PREDICATE_PARTIAL(min, MinReductionV)
1886 REDUCE_FMINMAX_PREDICATE(min, F, T_FLOAT,  S, vRegF)
1887 REDUCE_FMINMAX_PREDICATE(min, D, T_DOUBLE, D, vRegD)
1888 REDUCE_FMINMAX_PREDICATE_PARTIAL(min, F, T_FLOAT,  S, vRegF)
1889 REDUCE_FMINMAX_PREDICATE_PARTIAL(min, D, T_DOUBLE, D, vRegD)
1890 
1891 // vector Math.rint, floor, ceil
1892 
1893 instruct vroundD(vReg dst, vReg src, immI rmode) %{
1894   predicate(UseSVE > 0 &&
1895             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
1896   match(Set dst (RoundDoubleModeV src rmode));
1897   format %{ "sve_frint $dst, $src, $rmode\t# vector (sve) (D)" %}
1898   ins_encode %{
1899     switch ($rmode$$constant) {
1900       case RoundDoubleModeNode::rmode_rint:
1901         __ sve_frintn(as_FloatRegister($dst$$reg), __ D,
1902              ptrue, as_FloatRegister($src$$reg));
1903         break;
1904       case RoundDoubleModeNode::rmode_floor:
1905         __ sve_frintm(as_FloatRegister($dst$$reg), __ D,
1906              ptrue, as_FloatRegister($src$$reg));
1907         break;
1908       case RoundDoubleModeNode::rmode_ceil:
1909         __ sve_frintp(as_FloatRegister($dst$$reg), __ D,
1910              ptrue, as_FloatRegister($src$$reg));
1911         break;
1912     }
1913   %}
1914   ins_pipe(pipe_slow);
1915 %}
1916 dnl
1917 dnl REPLICATE($1,        $2,      $3,      $4,   $5         )
1918 dnl REPLICATE(insn_name, op_name, reg_src, size, min_vec_len)
1919 define(`REPLICATE', `
1920 instruct $1(vReg dst, $3 src) %{
1921   predicate(UseSVE > 0);
1922   match(Set dst ($2 src));
1923   ins_cost(SVE_COST);
1924   format %{ "sve_dup  $dst, $src\t# vector (sve) ($4)" %}
1925   ins_encode %{
1926     __ sve_dup(as_FloatRegister($dst$$reg), __ $4, as_Register($src$$reg));
1927   %}
1928   ins_pipe(pipe_slow);
1929 %}')dnl
1930 dnl
1931 dnl REPLICATE_IMM8($1,        $2,      $3,       $4,   $5         )
1932 dnl REPLICATE_IMM8(insn_name, op_name, imm_type, size, min_vec_len)
1933 define(`REPLICATE_IMM8', `
1934 instruct $1(vReg dst, $3 con) %{
1935   predicate(UseSVE > 0);
1936   match(Set dst ($2 con));
1937   ins_cost(SVE_COST);
1938   format %{ "sve_dup  $dst, $con\t# vector (sve) ($4)" %}
1939   ins_encode %{
1940     __ sve_dup(as_FloatRegister($dst$$reg), __ $4, $con$$constant);
1941   %}
1942   ins_pipe(pipe_slow);
1943 %}')dnl
1944 dnl
1945 dnl FREPLICATE($1,        $2,      $3,        $4)
1946 dnl FREPLICATE(insn_name, op_name, reg_src, size)
1947 define(`FREPLICATE', `
1948 instruct $1(vReg dst, $3 src) %{
1949   predicate(UseSVE > 0);
1950   match(Set dst ($2 src));
1951   ins_cost(SVE_COST);
1952   format %{ "sve_cpy  $dst, $src\t# vector (sve) ($4)" %}
1953   ins_encode %{
1954     __ sve_cpy(as_FloatRegister($dst$$reg), __ $4,
1955          ptrue, as_FloatRegister($src$$reg));
1956   %}
1957   ins_pipe(pipe_slow);
1958 %}')dnl
1959 
1960 // vector replicate
1961 REPLICATE(replicateB, ReplicateB, iRegIorL2I, B, 16)
1962 REPLICATE(replicateS, ReplicateS, iRegIorL2I, H, 8)
1963 REPLICATE(replicateI, ReplicateI, iRegIorL2I, S, 4)
1964 REPLICATE(replicateL, ReplicateL, iRegL,      D, 2)
1965 REPLICATE_IMM8(replicateB_imm8, ReplicateB, immI8,        B, 16)
1966 REPLICATE_IMM8(replicateS_imm8, ReplicateS, immI8_shift8, H, 8)
1967 REPLICATE_IMM8(replicateI_imm8, ReplicateI, immI8_shift8, S, 4)
1968 REPLICATE_IMM8(replicateL_imm8, ReplicateL, immL8_shift8, D, 2)
1969 FREPLICATE(replicateF, ReplicateF, vRegF, S, 4)
1970 FREPLICATE(replicateD, ReplicateD, vRegD, D, 2)
1971 dnl
1972 dnl VSHIFT_TRUE_PREDICATE($1,        $2,      $3,   $4,          $5  )
1973 dnl VSHIFT_TRUE_PREDICATE(insn_name, op_name, size, min_vec_len, insn)
1974 define(`VSHIFT_TRUE_PREDICATE', `
1975 instruct $1(vReg dst, vReg shift) %{
1976   predicate(UseSVE > 0);
1977   match(Set dst ($2 dst shift));
1978   ins_cost(SVE_COST);
1979   format %{ "$5 $dst, $dst, $shift\t# vector (sve) ($3)" %}
1980   ins_encode %{
1981     __ $5(as_FloatRegister($dst$$reg), __ $3,
1982          ptrue, as_FloatRegister($shift$$reg));
1983   %}
1984   ins_pipe(pipe_slow);
1985 %}')dnl
1986 dnl
1987 dnl VSHIFT_IMM_UNPREDICATED($1,        $2,      $3,       $4,   $5,          $6  )
1988 dnl VSHIFT_IMM_UNPREDICATED(insn_name, op_name, op_name2, size, min_vec_len, insn)
1989 define(`VSHIFT_IMM_UNPREDICATED', `
1990 instruct $1(vReg dst, vReg src, immI shift) %{
1991   predicate(UseSVE > 0);
1992   match(Set dst ($2 src ($3 shift)));
1993   ins_cost(SVE_COST);
1994   format %{ "$6 $dst, $src, $shift\t# vector (sve) ($4)" %}
1995   ins_encode %{
1996     int con = (int)$shift$$constant;dnl
1997 ifelse(eval(index(`$1', `vasr') == 0 || index(`$1', `vlsr') == 0), 1, `
1998     if (con == 0) {
1999       __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
2000            as_FloatRegister($src$$reg));
2001       return;
2002     }')dnl
2003 ifelse(eval(index(`$1', `vasr') == 0), 1, `ifelse(eval(index(`$4', `B') == 0), 1, `
2004     if (con >= 8) con = 7;')ifelse(eval(index(`$4', `H') == 0), 1, `
2005     if (con >= 16) con = 15;')')dnl
2006 ifelse(eval(index(`$1', `vlsl') == 0  || index(`$1', `vlsr') == 0), 1, `ifelse(eval(index(`$4', `B') == 0), 1, `
2007     if (con >= 8) {
2008       __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
2009            as_FloatRegister($src$$reg));
2010       return;
2011     }')ifelse(eval(index(`$4', `H') == 0), 1, `
2012     if (con >= 16) {
2013       __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
2014            as_FloatRegister($src$$reg));
2015       return;
2016     }')')
2017     __ $6(as_FloatRegister($dst$$reg), __ $4,
2018          as_FloatRegister($src$$reg), con);
2019   %}
2020   ins_pipe(pipe_slow);
2021 %}')dnl
2022 dnl
2023 dnl VSHIFT_COUNT($1,        $2,   $3,          $4  )
2024 dnl VSHIFT_COUNT(insn_name, size, min_vec_len, type)
2025 define(`VSHIFT_COUNT', `
2026 instruct $1(vReg dst, iRegIorL2I cnt) %{
2027   predicate(UseSVE > 0 &&
2028             ELEMENT_SHORT_CHAR($4, n));
2029   match(Set dst (LShiftCntV cnt));
2030   match(Set dst (RShiftCntV cnt));
2031   format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) ($2)" %}
2032   ins_encode %{
2033     __ sve_dup(as_FloatRegister($dst$$reg), __ $2, as_Register($cnt$$reg));
2034   %}
2035   ins_pipe(pipe_slow);
2036 %}')dnl
2037 
2038 // vector shift
2039 VSHIFT_TRUE_PREDICATE(vasrB, RShiftVB,  B, 16, sve_asr)
2040 VSHIFT_TRUE_PREDICATE(vasrS, RShiftVS,  H,  8, sve_asr)
2041 VSHIFT_TRUE_PREDICATE(vasrI, RShiftVI,  S,  4, sve_asr)
2042 VSHIFT_TRUE_PREDICATE(vasrL, RShiftVL,  D,  2, sve_asr)
2043 VSHIFT_TRUE_PREDICATE(vlslB, LShiftVB,  B, 16, sve_lsl)
2044 VSHIFT_TRUE_PREDICATE(vlslS, LShiftVS,  H,  8, sve_lsl)
2045 VSHIFT_TRUE_PREDICATE(vlslI, LShiftVI,  S,  4, sve_lsl)
2046 VSHIFT_TRUE_PREDICATE(vlslL, LShiftVL,  D,  2, sve_lsl)
2047 VSHIFT_TRUE_PREDICATE(vlsrB, URShiftVB, B, 16, sve_lsr)
2048 VSHIFT_TRUE_PREDICATE(vlsrS, URShiftVS, H,  8, sve_lsr)
2049 VSHIFT_TRUE_PREDICATE(vlsrI, URShiftVI, S,  4, sve_lsr)
2050 VSHIFT_TRUE_PREDICATE(vlsrL, URShiftVL, D,  2, sve_lsr)
2051 VSHIFT_IMM_UNPREDICATED(vasrB_imm, RShiftVB,  RShiftCntV, B, 16, sve_asr)
2052 VSHIFT_IMM_UNPREDICATED(vasrS_imm, RShiftVS,  RShiftCntV, H,  8, sve_asr)
2053 VSHIFT_IMM_UNPREDICATED(vasrI_imm, RShiftVI,  RShiftCntV, S,  4, sve_asr)
2054 VSHIFT_IMM_UNPREDICATED(vasrL_imm, RShiftVL,  RShiftCntV, D,  2, sve_asr)
2055 VSHIFT_IMM_UNPREDICATED(vlsrB_imm, URShiftVB, RShiftCntV, B, 16, sve_lsr)
2056 VSHIFT_IMM_UNPREDICATED(vlsrS_imm, URShiftVS, RShiftCntV, H,  8, sve_lsr)
2057 VSHIFT_IMM_UNPREDICATED(vlsrI_imm, URShiftVI, RShiftCntV, S,  4, sve_lsr)
2058 VSHIFT_IMM_UNPREDICATED(vlsrL_imm, URShiftVL, RShiftCntV, D,  2, sve_lsr)
2059 VSHIFT_IMM_UNPREDICATED(vlslB_imm, LShiftVB,  LShiftCntV, B, 16, sve_lsl)
2060 VSHIFT_IMM_UNPREDICATED(vlslS_imm, LShiftVS,  LShiftCntV, H,  8, sve_lsl)
2061 VSHIFT_IMM_UNPREDICATED(vlslI_imm, LShiftVI,  LShiftCntV, S,  4, sve_lsl)
2062 VSHIFT_IMM_UNPREDICATED(vlslL_imm, LShiftVL,  LShiftCntV, D,  2, sve_lsl)
2063 VSHIFT_COUNT(vshiftcntB, B, 16, T_BYTE)
2064 VSHIFT_COUNT(vshiftcntS, H,  8, T_SHORT)
2065 VSHIFT_COUNT(vshiftcntI, S,  4, T_INT)
2066 VSHIFT_COUNT(vshiftcntL, D,  2, T_LONG)
2067 
2068 // vector shift - predicated
2069 BINARY_OP_PREDICATE(vasrB, RShiftVB,  B, sve_asr)
2070 BINARY_OP_PREDICATE(vasrS, RShiftVS,  H, sve_asr)
2071 BINARY_OP_PREDICATE(vasrI, RShiftVI,  S, sve_asr)
2072 BINARY_OP_PREDICATE(vasrL, RShiftVL,  D, sve_asr)
2073 BINARY_OP_PREDICATE(vlslB, LShiftVB,  B, sve_lsl)
2074 BINARY_OP_PREDICATE(vlslS, LShiftVS,  H, sve_lsl)
2075 BINARY_OP_PREDICATE(vlslI, LShiftVI,  S, sve_lsl)
2076 BINARY_OP_PREDICATE(vlslL, LShiftVL,  D, sve_lsl)
2077 BINARY_OP_PREDICATE(vlsrB, URShiftVB, B, sve_lsr)
2078 BINARY_OP_PREDICATE(vlsrS, URShiftVS, H, sve_lsr)
2079 BINARY_OP_PREDICATE(vlsrI, URShiftVI, S, sve_lsr)
2080 BINARY_OP_PREDICATE(vlsrL, URShiftVL, D, sve_lsr)
2081 dnl
2082 dnl VSHIFT_IMM_PREDICATED($1,        $2,      $3,       $4,   $5,   $6  )
2083 dnl VSHIFT_IMM_PREDICATED(insn_name, op_name, op_name2, type, size, insn)
2084 define(`VSHIFT_IMM_PREDICATED', `
2085 instruct $1_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
2086   predicate(UseSVE > 0);
2087   match(Set dst_src ($2 (Binary dst_src ($3 shift)) pg));
2088   ins_cost(SVE_COST);
2089   format %{ "$6 $dst_src, $pg, $dst_src, $shift\t# vector (sve) ($4)" %}
2090   ins_encode %{
2091     int con = (int)$shift$$constant;
2092     assert(con ifelse(index(`$1', `vlsl'), 0, `>=', `>') 0 && con < $5, "invalid shift immediate");
2093     __ $6(as_FloatRegister($dst_src$$reg), __ $4, as_PRegister($pg$$reg), con);
2094   %}
2095   ins_pipe(pipe_slow);
2096 %}')dnl
2097 dnl
2098 VSHIFT_IMM_PREDICATED(vasrB, RShiftVB,  RShiftCntV, B, 8,  sve_asr)
2099 VSHIFT_IMM_PREDICATED(vasrS, RShiftVS,  RShiftCntV, H, 16, sve_asr)
2100 VSHIFT_IMM_PREDICATED(vasrI, RShiftVI,  RShiftCntV, S, 32, sve_asr)
2101 VSHIFT_IMM_PREDICATED(vasrL, RShiftVL,  RShiftCntV, D, 64, sve_asr)
2102 VSHIFT_IMM_PREDICATED(vlsrB, URShiftVB, RShiftCntV, B, 8,  sve_lsr)
2103 VSHIFT_IMM_PREDICATED(vlsrS, URShiftVS, RShiftCntV, H, 16, sve_lsr)
2104 VSHIFT_IMM_PREDICATED(vlsrI, URShiftVI, RShiftCntV, S, 32, sve_lsr)
2105 VSHIFT_IMM_PREDICATED(vlsrL, URShiftVL, RShiftCntV, D, 64, sve_lsr)
2106 VSHIFT_IMM_PREDICATED(vlslB, LShiftVB,  LShiftCntV, B, 8,  sve_lsl)
2107 VSHIFT_IMM_PREDICATED(vlslS, LShiftVS,  LShiftCntV, H, 16, sve_lsl)
2108 VSHIFT_IMM_PREDICATED(vlslI, LShiftVI,  LShiftCntV, S, 32, sve_lsl)
2109 VSHIFT_IMM_PREDICATED(vlslL, LShiftVL,  LShiftCntV, D, 64, sve_lsl)
2110 
2111 // vector sqrt
2112 UNARY_OP_TRUE_PREDICATE(vsqrtF, SqrtVF, S, sve_fsqrt)
2113 UNARY_OP_TRUE_PREDICATE(vsqrtD, SqrtVD, D, sve_fsqrt)
2114 
2115 // vector sqrt - predicated
2116 UNARY_OP_PREDICATE(vsqrtF, SqrtVF, S, sve_fsqrt)
2117 UNARY_OP_PREDICATE(vsqrtD, SqrtVD, D, sve_fsqrt)
2118 
2119 // vector sub
2120 BINARY_OP_UNPREDICATE(vsubB, SubVB, B, 16, sve_sub)
2121 BINARY_OP_UNPREDICATE(vsubS, SubVS, H, 8, sve_sub)
2122 BINARY_OP_UNPREDICATE(vsubI, SubVI, S, 4, sve_sub)
2123 BINARY_OP_UNPREDICATE(vsubL, SubVL, D, 2, sve_sub)
2124 BINARY_OP_UNPREDICATE(vsubF, SubVF, S, 4, sve_fsub)
2125 BINARY_OP_UNPREDICATE(vsubD, SubVD, D, 2, sve_fsub)
2126 
2127 // vector sub - predicated
2128 BINARY_OP_PREDICATE(vsubB, SubVB, B, sve_sub)
2129 BINARY_OP_PREDICATE(vsubS, SubVS, H, sve_sub)
2130 BINARY_OP_PREDICATE(vsubI, SubVI, S, sve_sub)
2131 BINARY_OP_PREDICATE(vsubL, SubVL, D, sve_sub)
2132 BINARY_OP_PREDICATE(vsubF, SubVF, S, sve_fsub)
2133 BINARY_OP_PREDICATE(vsubD, SubVD, D, sve_fsub)
2134 
2135 // ------------------------------ Vector mask cast --------------------------
2136 
2137 instruct vmaskcast(pRegGov dst_src) %{
2138   predicate(UseSVE > 0 &&
2139             n->bottom_type()->is_vect()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
2140             n->bottom_type()->is_vect()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
2141   match(Set dst_src (VectorMaskCast dst_src));
2142   ins_cost(0);
2143   format %{ "vmaskcast $dst_src\t# empty (sve)" %}
2144   ins_encode %{
2145     // empty
2146   %}
2147   ins_pipe(pipe_class_empty);
2148 %}
2149 
2150 instruct vmaskcast_extend(pRegGov dst, pReg src)
2151 %{
2152   predicate(UseSVE > 0 &&
2153             (Matcher::vector_length_in_bytes(n) == 2 * Matcher::vector_length_in_bytes(n->in(1)) ||
2154              Matcher::vector_length_in_bytes(n) == 4 * Matcher::vector_length_in_bytes(n->in(1)) ||
2155              Matcher::vector_length_in_bytes(n) == 8 * Matcher::vector_length_in_bytes(n->in(1))));
2156   match(Set dst (VectorMaskCast src));
2157   ins_cost(SVE_COST * 3);
2158   format %{ "sve_vmaskcast_extend  $dst, $src\t# extend predicate $src" %}
2159   ins_encode %{
2160     __ sve_vmaskcast_extend(as_PRegister($dst$$reg), as_PRegister($src$$reg),
2161                             Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
2162   %}
2163   ins_pipe(pipe_slow);
2164 %}
2165 
2166 instruct vmaskcast_narrow(pRegGov dst, pReg src)
2167 %{
2168   predicate(UseSVE > 0 &&
2169             (Matcher::vector_length_in_bytes(n) * 2 == Matcher::vector_length_in_bytes(n->in(1)) ||
2170              Matcher::vector_length_in_bytes(n) * 4 == Matcher::vector_length_in_bytes(n->in(1)) ||
2171              Matcher::vector_length_in_bytes(n) * 8 == Matcher::vector_length_in_bytes(n->in(1))));
2172   match(Set dst (VectorMaskCast src));
2173   ins_cost(SVE_COST * 3);
2174   format %{ "sve_vmaskcast_narrow  $dst, $src\t# narrow predicate $src" %}
2175   ins_encode %{
2176     __ sve_vmaskcast_narrow(as_PRegister($dst$$reg), as_PRegister($src$$reg),
2177                             Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
2178   %}
2179   ins_pipe(pipe_slow);
2180 %}
2181 dnl
2182 
2183 // ------------------------------ Vector cast -------------------------------
2184 dnl
2185 dnl
2186 define(`VECTOR_CAST_X2X', `
2187 instruct vcvt$1to$2`'(vReg dst, vReg src)
2188 %{
2189   predicate(UseSVE > 0 &&
2190             n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
2191   match(Set dst (VectorCast$1`'2X src));
2192   ins_cost(SVE_COST);
2193   format %{ "sve_vectorcast_$5  $dst, $src\t# convert $1 to $2 vector" %}
2194   ins_encode %{
2195     __ sve_$3(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src$$reg), __ $4);
2196   %}
2197   ins_pipe(pipe_slow);
2198 %}')dnl
2199 
2200 dnl
2201 dnl Start of vector cast rules
2202 dnl
2203 instruct vcvtBtoX_extend(vReg dst, vReg src)
2204 %{
2205   predicate(UseSVE > 0);
2206   match(Set dst (VectorCastB2X src));
2207   ins_cost(2 * SVE_COST);
2208   format %{ "sve_vectorcast_b2x  $dst, $src\t# convert B to X vector (extend)" %}
2209   ins_encode %{
2210     BasicType to_bt = Matcher::vector_element_basic_type(this);
2211     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2212     __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ B);
2213     if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
2214       __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
2215     }
2216   %}
2217   ins_pipe(pipe_slow);
2218 %}
2219 
2220 instruct vcvtStoB(vReg dst, vReg src, vReg tmp)
2221 %{
2222   predicate(UseSVE > 0 &&
2223             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2224   match(Set dst (VectorCastS2X src));
2225   effect(TEMP tmp);
2226   ins_cost(2 * SVE_COST);
2227   format %{ "sve_vectorcast_s2b  $dst, $src\t# convert H to B vector" %}
2228   ins_encode %{
2229     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
2230                          as_FloatRegister($src$$reg), __ H, as_FloatRegister($tmp$$reg));
2231   %}
2232   ins_pipe(pipe_slow);
2233 %}
2234 
2235 instruct vcvtStoX_extend(vReg dst, vReg src)
2236 %{
2237   predicate(UseSVE > 0 &&
2238             type2aelembytes(Matcher::vector_element_basic_type(n)) > 2);
2239   match(Set dst (VectorCastS2X src));
2240   ins_cost(2 * SVE_COST);
2241   format %{ "sve_vectorcast_s2x  $dst, $src\t# convert H to X vector (extend)" %}
2242   ins_encode %{
2243     BasicType to_bt = Matcher::vector_element_basic_type(this);
2244     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2245     __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ H);
2246     if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
2247       __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
2248     }
2249   %}
2250   ins_pipe(pipe_slow);
2251 %}
2252 
2253 instruct vcvtItoB(vReg dst, vReg src, vReg tmp)
2254 %{
2255   predicate(UseSVE > 0 &&
2256             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2257   match(Set dst (VectorCastI2X src));
2258   effect(TEMP_DEF dst, TEMP tmp);
2259   ins_cost(3 * SVE_COST);
2260   format %{ "sve_vectorcast_i2b  $dst, $src\t# convert I to B vector" %}
2261   ins_encode %{
2262     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
2263                          as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
2264   %}
2265   ins_pipe(pipe_slow);
2266 %}
2267 
2268 instruct vcvtItoS(vReg dst, vReg src, vReg tmp)
2269 %{
2270   predicate(UseSVE > 0 &&
2271             n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
2272   match(Set dst (VectorCastI2X src));
2273   effect(TEMP tmp);
2274   ins_cost(2 * SVE_COST);
2275   format %{ "sve_vectorcast_i2s $dst, $src\t# convert I to H vector" %}
2276   ins_encode %{
2277     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ H,
2278                          as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
2279   %}
2280   ins_pipe(pipe_slow);
2281 %}
2282 
2283 instruct vcvtItoL(vReg dst, vReg src)
2284 %{
2285   predicate(UseSVE > 0 &&
2286             n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2287   match(Set dst (VectorCastI2X src));
2288   ins_cost(SVE_COST);
2289   format %{ "sve_vectorcast_i2l  $dst, $src\t# convert I to L vector" %}
2290   ins_encode %{
2291     __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
2292   %}
2293   ins_pipe(pipe_slow);
2294 %}
2295 dnl
2296 dnl vcvtItoF
2297 VECTOR_CAST_X2X(I, F, scvtf, S, i2f)
2298 
2299 instruct vcvtItoD(vReg dst, vReg src)
2300 %{
2301   predicate(UseSVE > 0 &&
2302             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
2303   match(Set dst (VectorCastI2X src));
2304   ins_cost(2 * SVE_COST);
2305   format %{ "sve_vectorcast_i2d  $dst, $src\t# convert I to D vector" %}
2306   ins_encode %{
2307     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
2308     __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ D);
2309   %}
2310   ins_pipe(pipe_slow);
2311 %}
2312 
2313 instruct vcvtLtoX_narrow(vReg dst, vReg src, vReg tmp)
2314 %{
2315   predicate(UseSVE > 0 && is_integral_type(Matcher::vector_element_basic_type(n)));
2316   match(Set dst (VectorCastL2X src));
2317   effect(TEMP_DEF dst, TEMP tmp);
2318   ins_cost(2 * SVE_COST);
2319   format %{ "sve_vectorcast_l2x  $dst, $src\t# convert L to B/H/S vector (narrow)" %}
2320   ins_encode %{
2321     BasicType to_bt = Matcher::vector_element_basic_type(this);
2322     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2323     __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
2324                          as_FloatRegister($src$$reg), __ D, as_FloatRegister($tmp$$reg));
2325   %}
2326   ins_pipe(pipe_slow);
2327 %}
2328 
2329 instruct vcvtLtoF(vReg dst, vReg src, vReg tmp)
2330 %{
2331   predicate(UseSVE > 0 &&
2332             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2333   match(Set dst (VectorCastL2X src));
2334   effect(TEMP_DEF dst, TEMP tmp);
2335   ins_cost(3 * SVE_COST);
2336   format %{ "sve_vectorcast_l2f  $dst, $src\t# convert L to F vector" %}
2337   ins_encode %{
2338     __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
2339     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
2340                          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
2341 
2342   %}
2343   ins_pipe(pipe_slow);
2344 %}
2345 dnl
2346 dnl vcvtLtoD
2347 VECTOR_CAST_X2X(L, D, scvtf, D, l2d)
2348 
2349 instruct vcvtFtoX_narrow(vReg dst, vReg src, vReg tmp)
2350 %{
2351   predicate(UseSVE > 0 &&
2352             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2353              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT));
2354   match(Set dst (VectorCastF2X src));
2355   effect(TEMP_DEF dst, TEMP tmp);
2356   ins_cost(3 * SVE_COST);
2357   format %{ "sve_vectorcast_f2x  $dst, $src\t# convert F to B/H vector" %}
2358   ins_encode %{
2359     BasicType to_bt = Matcher::vector_element_basic_type(this);
2360     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2361     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
2362     __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
2363                          as_FloatRegister($dst$$reg), __ S, as_FloatRegister($tmp$$reg));
2364   %}
2365   ins_pipe(pipe_slow);
2366 %}
2367 
2368 instruct vcvtFtoI(vReg dst, vReg src)
2369 %{
2370   predicate(UseSVE > 0 &&
2371             (n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2372   match(Set dst (VectorCastF2X src));
2373   ins_cost(SVE_COST);
2374   format %{ "sve_vectorcast_f2x  $dst, $src\t# convert F to I vector" %}
2375   ins_encode %{
2376     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
2377   %}
2378   ins_pipe(pipe_slow);
2379 %}
2380 
2381 instruct vcvtFtoL(vReg dst, vReg src)
2382 %{
2383   predicate(UseSVE > 0 &&
2384             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
2385   match(Set dst (VectorCastF2X src));
2386   ins_cost(SVE_COST * 2);
2387   format %{ "sve_vectorcast_f2x  $dst, $src\t# convert F to L vector" %}
2388   ins_encode %{
2389     __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
2390     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);
2391   %}
2392   ins_pipe(pipe_slow);
2393 %}
2394 
2395 instruct vcvtFtoD(vReg dst, vReg src)
2396 %{
2397   predicate(UseSVE > 0 &&
2398             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
2399   match(Set dst (VectorCastF2X src));
2400   ins_cost(2 * SVE_COST);
2401   format %{ "sve_vectorcast_f2d  $dst, $dst\t# convert F to D vector" %}
2402   ins_encode %{
2403     __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
2404     __ sve_fcvt(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);
2405   %}
2406   ins_pipe(pipe_slow);
2407 %}
2408 
2409 instruct vcvtDtoX_narrow(vReg dst, vReg src, vReg tmp)
2410 %{
2411   predicate(UseSVE > 0 &&
2412             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2413              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2414              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2415   match(Set dst (VectorCastD2X src));
2416   effect(TEMP_DEF dst, TEMP tmp);
2417   ins_cost(3 * SVE_COST);
2418   format %{ "sve_vectorcast_d2x  $dst, $src\t# convert D to X vector (narrow)" %}
2419   ins_encode %{
2420     BasicType to_bt = Matcher::vector_element_basic_type(this);
2421     Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2422     __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
2423     __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
2424                          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
2425   %}
2426   ins_pipe(pipe_slow);
2427 %}
2428 dnl
2429 dnl vcvtDtoL
2430 VECTOR_CAST_X2X(D, L, fcvtzs, D, d2l)
2431 
2432 instruct vcvtDtoF(vReg dst, vReg src, vReg tmp)
2433 %{
2434   predicate(UseSVE > 0 &&
2435             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2436   match(Set dst (VectorCastD2X src));
2437   effect(TEMP_DEF dst, TEMP tmp);
2438   ins_cost(3 * SVE_COST);
2439   format %{ "sve_vectorcast_d2f  $dst, S, $dst\t# convert D to F vector" %}
2440   ins_encode %{
2441     __ sve_fcvt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
2442     __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
2443                          as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
2444   %}
2445   ins_pipe(pipe_slow);
2446 %}
2447 
2448 dnl
2449 dnl
2450 // ------------------------------ Vector extract ---------------------------------
2451 define(`VECTOR_EXTRACT_SXT', `
2452 instruct extract$1`'($2 dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
2453 %{
2454   predicate(UseSVE > 0);
2455   match(Set dst (Extract$1 src idx));
2456   effect(TEMP pgtmp, KILL cr);
2457   ins_cost(2 * SVE_COST);
2458   format %{ "sve_extract $dst, $3, $pgtmp, $src, $idx\n\t"
2459             "sbfmw $dst, $dst, 0U, $5\t# extract from vector($1)" %}
2460   ins_encode %{
2461     __ sve_extract(as_$4($dst$$reg), __ $3, as_PRegister($pgtmp$$reg),
2462                    as_FloatRegister($src$$reg), (int)($idx$$constant));
2463     __ sbfmw(as_$4($dst$$reg), as_$4($dst$$reg), 0U, $5);
2464   %}
2465   ins_pipe(pipe_slow);
2466 %}')dnl
2467 dnl                $1 $2         $3 $4        $5
2468 VECTOR_EXTRACT_SXT(B, iRegINoSp, B, Register, 7U)
2469 VECTOR_EXTRACT_SXT(S, iRegINoSp, H, Register, 15U)
2470 
2471 dnl
2472 define(`VECTOR_EXTRACT', `
2473 instruct extract$1`'($2 dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
2474 %{
2475   predicate(UseSVE > 0);
2476   match(Set dst (Extract$1 src idx));
2477   effect(TEMP pgtmp, KILL cr);
2478   ins_cost(2 * SVE_COST);
2479   format %{ "sve_extract $dst, $3, $pgtmp, $src, $idx\t# extract from vector($1)" %}
2480   ins_encode %{
2481     __ sve_extract(as_$4($dst$$reg), __ $3, as_PRegister($pgtmp$$reg),
2482                    as_FloatRegister($src$$reg), (int)($idx$$constant));
2483   %}
2484   ins_pipe(pipe_slow);
2485 %}')dnl
2486 dnl            $1 $2         $3 $4
2487 VECTOR_EXTRACT(I, iRegINoSp, S, Register)
2488 VECTOR_EXTRACT(L, iRegLNoSp, D, Register)
2489 VECTOR_EXTRACT(F, vRegF,     S, FloatRegister)
2490 VECTOR_EXTRACT(D, vRegD,     D, FloatRegister)
2491 
2492 // ------------------------------- VectorTest ----------------------------------
2493 
2494 instruct vtest_alltrue(iRegINoSp dst, pRegGov src1, pRegGov src2, pReg ptmp, rFlagsReg cr)
2495 %{
2496   predicate(UseSVE > 0 &&
2497             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2498             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
2499   match(Set dst (VectorTest src1 src2));
2500   effect(TEMP ptmp, KILL cr);
2501   ins_cost(SVE_COST);
2502   format %{ "sve_eors $ptmp, $src1, $src2\t# $src2 is all true mask\n"
2503             "csetw $dst, EQ\t# VectorTest (sve) - alltrue" %}
2504   ins_encode %{
2505     __ sve_eors(as_PRegister($ptmp$$reg), ptrue,
2506                 as_PRegister($src1$$reg), as_PRegister($src2$$reg));
2507     __ csetw(as_Register($dst$$reg), Assembler::EQ);
2508   %}
2509   ins_pipe(pipe_slow);
2510 %}
2511 
2512 instruct vtest_anytrue(iRegINoSp dst, pRegGov src1, pRegGov src2, rFlagsReg cr)
2513 %{
2514   predicate(UseSVE > 0 &&
2515             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2516             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
2517   match(Set dst (VectorTest src1 src2));
2518   effect(KILL cr);
2519   ins_cost(SVE_COST);
2520   format %{ "sve_ptest $src1\n\t"
2521             "csetw $dst, NE\t# VectorTest (sve) - anytrue" %}
2522   ins_encode %{
2523     // "src2" is not used for sve.
2524     __ sve_ptest(ptrue, as_PRegister($src1$$reg));
2525     __ csetw(as_Register($dst$$reg), Assembler::NE);
2526   %}
2527   ins_pipe(pipe_slow);
2528 %}
2529 dnl
2530 dnl
2531 dnl VTEST_PARTIAL($1,      $2,   $3,   $4  )
2532 dnl VTEST_PARTIAL(op_name, pred, inst, cond)
2533 define(`VTEST_PARTIAL', `
2534 instruct vtest_$1_partial`'(iRegINoSp dst, pRegGov src1, pRegGov src2, pRegGov ptmp, rFlagsReg cr)
2535 %{
2536   predicate(UseSVE > 0 &&
2537             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2538             static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::$2);
2539   match(Set dst (VectorTest src1 src2));
2540   effect(TEMP ptmp, KILL cr);
2541   ins_cost(SVE_COST);
2542   format %{ "vtest_$1_partial $dst, $src1, $src2\t# VectorTest partial (sve) - $1" %}
2543   ins_encode %{
2544     BasicType bt = Matcher::vector_element_basic_type(this, $src1);
2545     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2546     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size,
2547                           Matcher::vector_length(this, $src1));
2548     __ $3(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2549           as_PRegister($src1$$reg), as_PRegister($src2$$reg));
2550     __ csetw(as_Register($dst$$reg), Assembler::$4);
2551   %}
2552   ins_pipe(pipe_slow);
2553 %}')dnl
2554 dnl
2555 VTEST_PARTIAL(alltrue, overflow, sve_eors, EQ)
2556 VTEST_PARTIAL(anytrue, ne,       sve_ands, NE)
2557 
2558 // ------------------------------ Vector insert ---------------------------------
2559 
2560 instruct insertI_small(vReg dst, vReg src, iRegIorL2I val, immI idx, pRegGov pgtmp, rFlagsReg cr)
2561 %{
2562   predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
2563             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2564              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2565              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2566   match(Set dst (VectorInsert (Binary src val) idx));
2567   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
2568   ins_cost(4 * SVE_COST);
2569   format %{ "sve_index $dst, -16, 1\t# (B/H/S)\n\t"
2570             "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2571             "sve_orr $dst, $src, $src\n\t"
2572             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
2573   ins_encode %{
2574     BasicType bt = Matcher::vector_element_basic_type(this, $src);
2575     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2576     __ sve_index(as_FloatRegister($dst$$reg), size, -16, 1);
2577     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
2578                as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2579     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2580     __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
2581   %}
2582   ins_pipe(pipe_slow);
2583 %}
2584 
2585 instruct insertF_small(vReg dst, vReg src, vRegF val, immI idx, pRegGov pgtmp, rFlagsReg cr)
2586 %{
2587   predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
2588             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2589   match(Set dst (VectorInsert (Binary src val) idx));
2590   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
2591   ins_cost(4 * SVE_COST);
2592   format %{ "sve_index $dst, S, -16, 1\n\t"
2593             "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2594             "sve_orr $dst, $src, $src\n\t"
2595             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
2596   ins_encode %{
2597     __ sve_index(as_FloatRegister($dst$$reg), __ S, -16, 1);
2598     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
2599                as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2600     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2601     __ sve_cpy(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
2602   %}
2603   ins_pipe(pipe_slow);
2604 %}
2605 
2606 instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr)
2607 %{
2608   predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
2609             (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2610              n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2611              n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2612   match(Set dst (VectorInsert (Binary src val) idx));
2613   effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
2614   ins_cost(5 * SVE_COST);
2615   format %{ "sve_index $tmp1, 0, 1\t# (B/H/S)\n\t"
2616             "sve_dup $dst, $idx\t# (B/H/S)\n\t"
2617             "sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
2618             "sve_orr $dst, $src, $src\n\t"
2619             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
2620   ins_encode %{
2621     BasicType bt = Matcher::vector_element_basic_type(this, $src);
2622     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2623     __ sve_index(as_FloatRegister($tmp1$$reg), size, 0, 1);
2624     __ sve_dup(as_FloatRegister($dst$$reg), size, (int)($idx$$constant));
2625     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
2626                as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
2627     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2628     __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
2629   %}
2630   ins_pipe(pipe_slow);
2631 %}
2632 dnl
2633 dnl
2634 define(`VECTOR_INSERT_D', `
2635 instruct insert$1`'(vReg dst, vReg src, $2 val, immI idx, pRegGov pgtmp, rFlagsReg cr)
2636 %{
2637   predicate(UseSVE > 0 &&
2638             n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($1));
2639   match(Set dst (VectorInsert (Binary src val) idx));
2640   effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
2641   ins_cost(4 * SVE_COST);
2642   format %{ "sve_index $dst, $3, -16, 1\n\t"
2643             "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2644             "sve_orr $dst, $src, $src\n\t"
2645             "sve_cpy $dst, $pgtmp, $val\t# insert into vector ($1)" %}
2646   ins_encode %{
2647     __ sve_index(as_FloatRegister($dst$$reg), __ $3, -16, 1);
2648     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ $3, ptrue,
2649                as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2650     __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2651     __ sve_cpy(as_FloatRegister($dst$$reg), __ $3, as_PRegister($pgtmp$$reg), as_$4($val$$reg));
2652   %}
2653   ins_pipe(pipe_slow);
2654 %}')dnl
2655 dnl             $1 $2     $3 $4
2656 VECTOR_INSERT_D(L, iRegL, D, Register)
2657 VECTOR_INSERT_D(D, vRegD, D, FloatRegister)
2658 
2659 instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr)
2660 %{
2661   predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
2662             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2663   match(Set dst (VectorInsert (Binary src val) idx));
2664   effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
2665   ins_cost(5 * SVE_COST);
2666   format %{ "sve_index $tmp1, S, 0, 1\n\t"
2667             "sve_dup $dst, S, $idx\n\t"
2668             "sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
2669             "sve_orr $dst, $src, $src\n\t"
2670             "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
2671   ins_encode %{
2672     __ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
2673     __ sve_dup(as_FloatRegister($dst$$reg), __ S, (int)($idx$$constant));
2674     __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
2675                as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
2676     __ sve_orr(as_FloatRegister($dst$$reg),
2677                as_FloatRegister($src$$reg),
2678                as_FloatRegister($src$$reg));
2679     __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
2680                as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
2681   %}
2682   ins_pipe(pipe_slow);
2683 %}
2684 
2685 // ------------------------------ Vector shuffle -------------------------------
2686 
2687 instruct loadshuffle(vReg dst, vReg src) %{
2688   predicate(UseSVE > 0);
2689   match(Set dst (VectorLoadShuffle src));
2690   ins_cost(SVE_COST);
2691   format %{ "sve_loadshuffle $dst, $src\t# vector load shuffle (B/H/S/D)" %}
2692   ins_encode %{
2693     BasicType bt = Matcher::vector_element_basic_type(this);
2694     if (bt == T_BYTE) {
2695       if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
2696         __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
2697                    as_FloatRegister($src$$reg));
2698       }
2699     } else {
2700       __ sve_vector_extend(as_FloatRegister($dst$$reg),  __ elemType_to_regVariant(bt),
2701                            as_FloatRegister($src$$reg), __ B);
2702     }
2703   %}
2704   ins_pipe(pipe_slow);
2705 %}
2706 
2707 // ------------------------------ Vector rearrange -------------------------------
2708 
2709 instruct rearrange(vReg dst, vReg src, vReg shuffle)
2710 %{
2711   predicate(UseSVE > 0);
2712   match(Set dst (VectorRearrange src shuffle));
2713   ins_cost(SVE_COST);
2714   format %{ "sve_tbl $dst, $src, $shuffle\t# vector rearrange" %}
2715   ins_encode %{
2716     BasicType bt = Matcher::vector_element_basic_type(this, $src);
2717     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2718     __ sve_tbl(as_FloatRegister($dst$$reg), size,
2719                as_FloatRegister($src$$reg), as_FloatRegister($shuffle$$reg));
2720   %}
2721   ins_pipe(pipe_slow);
2722 %}
2723 
2724 // ------------------------------ Vector Load Gather ---------------------------------
2725 
2726 instruct gatherI(vReg dst, indirect mem, vReg idx) %{
2727   predicate(UseSVE > 0 &&
2728             n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
2729             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2730              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2731   match(Set dst (LoadVectorGather mem idx));
2732   ins_cost(SVE_COST);
2733   format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (S)" %}
2734   ins_encode %{
2735     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), ptrue,
2736                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2737   %}
2738   ins_pipe(pipe_slow);
2739 %}
2740 
2741 instruct gatherL(vReg dst, indirect mem, vReg idx) %{
2742   predicate(UseSVE > 0 &&
2743             n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
2744             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2745              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2746   match(Set dst (LoadVectorGather mem idx));
2747   ins_cost(2 * SVE_COST);
2748   format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (D)" %}
2749   ins_encode %{
2750     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2751     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), ptrue, as_Register($mem$$base),
2752                        as_FloatRegister($idx$$reg));
2753   %}
2754   ins_pipe(pipe_slow);
2755 %}
2756 
2757 // ------------------------------ Vector Load Gather Partial-------------------------------
2758 
2759 instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2760   predicate(UseSVE > 0 &&
2761             n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
2762             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2763              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2764   match(Set dst (LoadVectorGather mem idx));
2765   effect(TEMP ptmp, KILL cr);
2766   ins_cost(2 * SVE_COST + INSN_COST);
2767   format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (S)" %}
2768   ins_encode %{
2769     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this));
2770     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
2771                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2772   %}
2773   ins_pipe(pipe_slow);
2774 %}
2775 
2776 instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2777   predicate(UseSVE > 0 &&
2778             n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
2779             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2780              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2781   match(Set dst (LoadVectorGather mem idx));
2782   effect(TEMP ptmp, KILL cr);
2783   ins_cost(3 * SVE_COST + INSN_COST);
2784   format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (D)" %}
2785   ins_encode %{
2786     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2787                           Matcher::vector_length(this));
2788     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2789     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
2790                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2791   %}
2792   ins_pipe(pipe_slow);
2793 %}
2794 
2795 // ------------------------------ Vector Load Gather Predicated -------------------------------
2796 
2797 instruct gatherI_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
2798   predicate(UseSVE > 0 &&
2799             n->as_LoadVector()->memory_size() == MaxVectorSize &&
2800             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2801              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2802   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
2803   ins_cost(SVE_COST);
2804   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (S)" %}
2805   ins_encode %{
2806     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
2807                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2808   %}
2809   ins_pipe(pipe_slow);
2810 %}
2811 
2812 instruct gatherL_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
2813   predicate(UseSVE > 0 &&
2814             n->as_LoadVector()->memory_size() == MaxVectorSize &&
2815             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2816              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2817   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
2818   ins_cost(2 * SVE_COST);
2819   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (D)" %}
2820   ins_encode %{
2821     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2822     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
2823                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2824   %}
2825   ins_pipe(pipe_slow);
2826 %}
2827 
2828 // ------------------------------ Vector Load Gather Predicated Partial -------------------------------
2829 
2830 instruct gatherI_masked_partial(vReg dst, indirect mem, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2831   predicate(UseSVE > 0 &&
2832             n->as_LoadVector()->memory_size() < MaxVectorSize &&
2833             (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2834              n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2835   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
2836   effect(TEMP ptmp, KILL cr);
2837   ins_cost(3 * SVE_COST);
2838   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated partial (S)" %}
2839   ins_encode %{
2840     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2841                           Matcher::vector_length(this));
2842     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2843                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2844     __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
2845                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2846   %}
2847   ins_pipe(pipe_slow);
2848 %}
2849 
2850 instruct gatherL_masked_partial(vReg dst, indirect mem, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2851   predicate(UseSVE > 0 &&
2852             n->as_LoadVector()->memory_size() < MaxVectorSize &&
2853             (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2854              n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2855   match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
2856   effect(TEMP ptmp, KILL cr);
2857   ins_cost(4 * SVE_COST);
2858   format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated partial (D)" %}
2859   ins_encode %{
2860     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this));
2861     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2862                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2863     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2864     __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
2865                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
2866   %}
2867   ins_pipe(pipe_slow);
2868 %}
2869 
2870 // ------------------------------ Vector Store Scatter -------------------------------
2871 
2872 instruct scatterI(indirect mem, vReg src, vReg idx) %{
2873   predicate(UseSVE > 0 &&
2874             n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
2875             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2876              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2877   match(Set mem (StoreVectorScatter mem (Binary src idx)));
2878   ins_cost(SVE_COST);
2879   format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (S)" %}
2880   ins_encode %{
2881     __ sve_st1w_scatter(as_FloatRegister($src$$reg), ptrue,
2882                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2883   %}
2884   ins_pipe(pipe_slow);
2885 %}
2886 
2887 instruct scatterL(indirect mem, vReg src, vReg idx) %{
2888   predicate(UseSVE > 0 &&
2889             n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
2890             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2891              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2892   match(Set mem (StoreVectorScatter mem (Binary src idx)));
2893   ins_cost(2 * SVE_COST);
2894   format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (D)" %}
2895   ins_encode %{
2896     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2897     __ sve_st1d_scatter(as_FloatRegister($src$$reg), ptrue,
2898                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2899   %}
2900   ins_pipe(pipe_slow);
2901 %}
2902 
2903 // ------------------------------ Vector Store Scatter Partial -------------------------------
2904 
2905 instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2906   predicate(UseSVE > 0 &&
2907             n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
2908             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2909              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2910   match(Set mem (StoreVectorScatter mem (Binary src idx)));
2911   effect(TEMP ptmp, KILL cr);
2912   ins_cost(2 * SVE_COST + INSN_COST);
2913   format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (S)" %}
2914   ins_encode %{
2915     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2916                           Matcher::vector_length(this, $src));
2917     __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
2918                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2919   %}
2920   ins_pipe(pipe_slow);
2921 %}
2922 
2923 instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2924   predicate(UseSVE > 0 &&
2925             n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
2926             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2927              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2928   match(Set mem (StoreVectorScatter mem (Binary src idx)));
2929   effect(TEMP ptmp, KILL cr);
2930   ins_cost(3 * SVE_COST + INSN_COST);
2931   format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (D)" %}
2932   ins_encode %{
2933     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2934                           Matcher::vector_length(this, $src));
2935     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2936     __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
2937                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2938   %}
2939   ins_pipe(pipe_slow);
2940 %}
2941 
2942 // ------------------------------ Vector Store Scatter Predicated -------------------------------
2943 
2944 instruct scatterI_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
2945   predicate(UseSVE > 0 &&
2946             n->as_StoreVector()->memory_size() == MaxVectorSize &&
2947             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2948              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2949   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
2950   ins_cost(SVE_COST);
2951   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicate (S)" %}
2952   ins_encode %{
2953     __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
2954                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2955   %}
2956   ins_pipe(pipe_slow);
2957 %}
2958 
2959 instruct scatterL_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
2960   predicate(UseSVE > 0 &&
2961             n->as_StoreVector()->memory_size() == MaxVectorSize &&
2962             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2963              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2964   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
2965   ins_cost(2 * SVE_COST);
2966   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated (D)" %}
2967   ins_encode %{
2968     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2969     __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
2970                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2971   %}
2972   ins_pipe(pipe_slow);
2973 %}
2974 
2975 // ------------------------------ Vector Store Scatter Predicated Partial -------------------------------
2976 
2977 instruct scatterI_masked_partial(indirect mem, vReg src, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2978   predicate(UseSVE > 0 &&
2979             n->as_StoreVector()->memory_size() < MaxVectorSize &&
2980             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2981              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2982   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
2983   effect(TEMP ptmp, KILL cr);
2984   ins_cost(3 * SVE_COST);
2985   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated partial (S)" %}
2986   ins_encode %{
2987     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2988                           Matcher::vector_length(this, $src));
2989     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2990                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2991     __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
2992                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
2993   %}
2994   ins_pipe(pipe_slow);
2995 %}
2996 
2997 instruct scatterL_masked_partial(indirect mem, vReg src, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2998   predicate(UseSVE > 0 &&
2999             n->as_StoreVector()->memory_size() < MaxVectorSize &&
3000             (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
3001              n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
3002   match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
3003   effect(TEMP ptmp, KILL cr);
3004   ins_cost(4 * SVE_COST);
3005   format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated partial (D)" %}
3006   ins_encode %{
3007     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
3008                           Matcher::vector_length(this, $src));
3009     __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
3010                as_PRegister($pg$$reg), as_PRegister($pg$$reg));
3011     __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
3012     __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
3013                         as_Register($mem$$base), as_FloatRegister($idx$$reg));
3014   %}
3015   ins_pipe(pipe_slow);
3016 %}
3017 
3018 // ------------------------------ Vector Load Const -------------------------------
3019 
3020 instruct loadconB(vReg dst, immI0 src) %{
3021   predicate(UseSVE > 0 &&
3022             n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
3023   match(Set dst (VectorLoadConst src));
3024   ins_cost(SVE_COST);
3025   format %{ "sve_index $dst, 0, 1\t# generate iota indices" %}
3026   ins_encode %{
3027     __ sve_index(as_FloatRegister($dst$$reg), __ B, 0, 1);
3028   %}
3029   ins_pipe(pipe_slow);
3030 %}
3031 
3032 // Intrisics for String.indexOf(char)
3033 
3034 dnl
3035 define(`STRING_INDEXOF_CHAR', `
3036 instruct string$1_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
3037                                   iRegI_R0 result, vReg ztmp1, vReg ztmp2,
3038                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
3039 %{
3040   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
3041   predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::$1));
3042   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
3043 
3044   format %{ "String$2 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
3045 
3046   ins_encode %{
3047     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
3048                                as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
3049                                as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), $3 /* isL */);
3050   %}
3051   ins_pipe(pipe_class_memory);
3052 %}')dnl
3053 dnl                 $1 $2      $3
3054 STRING_INDEXOF_CHAR(L, Latin1, true)
3055 STRING_INDEXOF_CHAR(U, UTF16,  false)
3056 
3057 // Intrisics for String.compareTo()
3058 
3059 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
3060 // these string_compare variants as NEON register type for convenience so that the prototype of
3061 // string_compare can be shared with all variants.
3062 
3063 dnl
3064 define(`STRING_COMPARETO', `
3065 instruct string_compare$1_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
3066                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
3067                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
3068                               pRegGov_P1 pgtmp2, rFlagsReg cr)
3069 %{
3070   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::$1));
3071   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
3072   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
3073          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
3074 
3075   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
3076   ins_encode %{
3077     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
3078     __ string_compare($str1$$Register, $str2$$Register,
3079                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
3080                       $tmp1$$Register, $tmp2$$Register,
3081                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
3082                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
3083                       StrIntrinsicNode::$1);
3084   %}
3085   ins_pipe(pipe_class_memory);
3086 %}')dnl
3087 dnl              $1
3088 STRING_COMPARETO(LL)
3089 STRING_COMPARETO(LU)
3090 STRING_COMPARETO(UL)
3091 STRING_COMPARETO(UU)
3092 
3093 // ---------------------------- Vector mask reductions ---------------------------
3094 instruct vmask_truecount(iRegINoSp dst, pReg src) %{
3095   predicate(UseSVE > 0 &&
3096             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3097   match(Set dst (VectorMaskTrueCount src));
3098   ins_cost(SVE_COST);
3099   format %{ "vmask_truecount $dst, $src\t# vector mask truecount (sve)" %}
3100   ins_encode %{
3101     BasicType bt = Matcher::vector_element_basic_type(this, $src);
3102     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3103     __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($src$$reg));
3104   %}
3105   ins_pipe(pipe_slow);
3106 %}
3107 
3108 instruct vmask_firsttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
3109   predicate(UseSVE > 0 &&
3110             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3111   match(Set dst (VectorMaskFirstTrue src));
3112   effect(TEMP ptmp);
3113   ins_cost(2 * SVE_COST);
3114   format %{ "vmask_firsttrue $dst, $src\t# vector mask firsttrue (sve)" %}
3115   ins_encode %{
3116     BasicType bt = Matcher::vector_element_basic_type(this, $src);
3117     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3118     __ sve_brkb(as_PRegister($ptmp$$reg), ptrue, as_PRegister($src$$reg), false);
3119     __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($ptmp$$reg));
3120   %}
3121   ins_pipe(pipe_slow);
3122 %}
3123 
3124 instruct vmask_lasttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
3125   predicate(UseSVE > 0 &&
3126             n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3127   match(Set dst (VectorMaskLastTrue src));
3128   effect(TEMP ptmp);
3129   ins_cost(3 * SVE_COST);
3130   format %{ "vmask_lasttrue $dst, $src\t# vector mask lasttrue (sve)" %}
3131   ins_encode %{
3132     BasicType bt = Matcher::vector_element_basic_type(this, $src);
3133     __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($src$$reg), as_PRegister($ptmp$$reg));
3134   %}
3135   ins_pipe(pipe_slow);
3136 %}
3137 
3138 instruct vmask_truecount_partial(iRegINoSp dst, pReg src, pReg ptmp, rFlagsReg cr) %{
3139   predicate(UseSVE > 0 &&
3140             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3141   match(Set dst (VectorMaskTrueCount src));
3142   effect(TEMP ptmp, KILL cr);
3143   ins_cost(2 * SVE_COST);
3144   format %{ "vmask_truecount_partial $dst, $src\t# vector mask truecount partial (sve)" %}
3145   ins_encode %{
3146     BasicType bt = Matcher::vector_element_basic_type(this, $src);
3147     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3148     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
3149     __ sve_cntp($dst$$Register, size, as_PRegister($ptmp$$reg), as_PRegister($src$$reg));
3150   %}
3151   ins_pipe(pipe_slow);
3152 %}
3153 
3154 instruct vmask_firsttrue_partial(iRegINoSp dst, pReg src, pReg ptmp1, pReg ptmp2, rFlagsReg cr) %{
3155   predicate(UseSVE > 0 &&
3156             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3157   match(Set dst (VectorMaskFirstTrue src));
3158   effect(TEMP ptmp1, TEMP ptmp2, KILL cr);
3159   ins_cost(3 * SVE_COST);
3160   format %{ "vmask_firsttrue_partial $dst, $src\t# vector mask firsttrue partial (sve)" %}
3161   ins_encode %{
3162     BasicType bt = Matcher::vector_element_basic_type(this, $src);
3163     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3164     __ sve_whilelo_zr_imm(as_PRegister($ptmp1$$reg), size,
3165                           Matcher::vector_length(this, $src));
3166     __ sve_brkb(as_PRegister($ptmp2$$reg), as_PRegister($ptmp1$$reg), as_PRegister($src$$reg), false);
3167     __ sve_cntp($dst$$Register, size, as_PRegister($ptmp1$$reg), as_PRegister($ptmp2$$reg));
3168   %}
3169   ins_pipe(pipe_slow);
3170 %}
3171 
3172 instruct vmask_lasttrue_partial(iRegINoSp dst, pReg src, pReg ptmp, rFlagsReg cr) %{
3173   predicate(UseSVE > 0 &&
3174             n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3175   match(Set dst (VectorMaskLastTrue src));
3176   effect(TEMP ptmp, KILL cr);
3177   ins_cost(5 * SVE_COST);
3178   format %{ "vmask_lasttrue_partial $dst, $src\t# vector mask lasttrue partial (sve)" %}
3179   ins_encode %{
3180     BasicType bt = Matcher::vector_element_basic_type(this, $src);
3181     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3182     __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
3183     __ sve_and(as_PRegister($ptmp$$reg), ptrue, as_PRegister($ptmp$$reg), as_PRegister($src$$reg));
3184     __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg));
3185   %}
3186   ins_pipe(pipe_slow);
3187 %}
3188 
3189 instruct vmask_tolong(iRegLNoSp dst, pReg src, vReg vtmp1, vReg vtmp2, pRegGov pgtmp, rFlagsReg cr) %{
3190   predicate(UseSVE > 0 &&
3191             n->in(1)->bottom_type()->is_vect()->length() <= 64);
3192   match(Set dst (VectorMaskToLong src));
3193   effect(TEMP vtmp1, TEMP vtmp2, TEMP pgtmp, KILL cr);
3194   ins_cost(13 * SVE_COST);
3195   format %{ "vmask_tolong $dst, $src\t# vector mask tolong (sve)" %}
3196   ins_encode %{
3197     __ sve_vmask_tolong(as_Register($dst$$reg), as_PRegister($src$$reg),
3198                         Matcher::vector_element_basic_type(this, $src),
3199                         Matcher::vector_length(this, $src),
3200                         as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg),
3201                         as_PRegister($pgtmp$$reg));
3202   %}
3203   ins_pipe(pipe_slow);
3204 %}dnl
3205 
3206 // ---------------------------- Vector mask generation ---------------------------
3207 instruct vmask_gen(pRegGov pg, iRegL len, rFlagsReg cr) %{
3208   predicate(UseSVE > 0);
3209   match(Set pg (VectorMaskGen len));
3210   effect(KILL cr);
3211   ins_cost(SVE_COST);
3212   format %{ "sve_whilelo $pg, zr, $len\t # sve" %}
3213   ins_encode %{
3214     BasicType bt = Matcher::vector_element_basic_type(this);
3215     Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3216     __ sve_whilelo(as_PRegister($pg$$reg), size, zr, as_Register($len$$reg));
3217   %}
3218   ins_pipe(pipe_slow);
3219 %}