1 // 2 // Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. 3 // Copyright (c) 2020, 2022, Arm Limited. All rights reserved. 4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 // 6 // This code is free software; you can redistribute it and/or modify it 7 // under the terms of the GNU General Public License version 2 only, as 8 // published by the Free Software Foundation. 9 // 10 // This code is distributed in the hope that it will be useful, but WITHOUT 11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 // version 2 for more details (a copy is included in the LICENSE file that 14 // accompanied this code). 15 // 16 // You should have received a copy of the GNU General Public License version 17 // 2 along with this work; if not, write to the Free Software Foundation, 18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 // 20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 // or visit www.oracle.com if you need additional information or have any 22 // questions. 23 // 24 // 25 26 // This file is automatically generated by running "m4 aarch64_sve_ad.m4". Do not edit ---- 27 28 // AArch64 SVE Architecture Description File 29 30 31 // 4 bit signed offset -- for predicated load/store 32 33 operand vmemA_immIOffset4() 34 %{ 35 // (esize / msize) = 1 36 predicate(Address::offset_ok_for_sve_immed(n->get_int(), 4, 37 Matcher::scalable_vector_reg_size(T_BYTE))); 38 match(ConI); 39 40 op_cost(0); 41 format %{ %} 42 interface(CONST_INTER); 43 %} 44 45 operand vmemA_immLOffset4() 46 %{ 47 // (esize / msize) = 1 48 predicate(Address::offset_ok_for_sve_immed(n->get_long(), 4, 49 Matcher::scalable_vector_reg_size(T_BYTE))); 50 match(ConL); 51 52 op_cost(0); 53 format %{ %} 54 interface(CONST_INTER); 55 %} 56 57 operand vmemA_indOffI4(iRegP reg, vmemA_immIOffset4 off) 58 %{ 59 constraint(ALLOC_IN_RC(ptr_reg)); 60 match(AddP reg off); 61 op_cost(0); 62 format %{ "[$reg, $off]" %} 63 interface(MEMORY_INTER) %{ 64 base($reg); 65 index(0xffffffff); 66 scale(0x0); 67 disp($off); 68 %} 69 %} 70 71 operand vmemA_indOffL4(iRegP reg, vmemA_immLOffset4 off) 72 %{ 73 constraint(ALLOC_IN_RC(ptr_reg)); 74 match(AddP reg off); 75 op_cost(0); 76 format %{ "[$reg, $off]" %} 77 interface(MEMORY_INTER) %{ 78 base($reg); 79 index(0xffffffff); 80 scale(0x0); 81 disp($off); 82 %} 83 %} 84 85 // The indOff of vmemA is valid only when the vector element (load to/store from) 86 // size equals to memory element (load from/store to) size. 87 opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4); 88 89 source_hpp %{ 90 bool op_sve_supported(int opcode, int vlen, BasicType bt); 91 bool masked_op_sve_supported(int opcode, int vlen, BasicType bt); 92 %} 93 94 source %{ 95 96 typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T, 97 PRegister Pg, const Address &adr); 98 99 // Predicated load/store, with optional ptrue to all elements of given predicate register. 100 static void loadStoreA_predicated(C2_MacroAssembler masm, bool is_store, FloatRegister reg, 101 PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt, 102 int opcode, Register base, int index, int size, int disp) { 103 sve_mem_insn_predicate insn; 104 int mesize = type2aelembytes(mem_elem_bt); 105 if (index == -1) { 106 assert(size == 0, "unsupported address mode: scale size = %d", size); 107 switch(mesize) { 108 case 1: 109 insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b; 110 break; 111 case 2: 112 insn = is_store ? &C2_MacroAssembler::sve_st1h : &C2_MacroAssembler::sve_ld1h; 113 break; 114 case 4: 115 insn = is_store ? &C2_MacroAssembler::sve_st1w : &C2_MacroAssembler::sve_ld1w; 116 break; 117 case 8: 118 insn = is_store ? &C2_MacroAssembler::sve_st1d : &C2_MacroAssembler::sve_ld1d; 119 break; 120 default: 121 assert(false, "unsupported"); 122 ShouldNotReachHere(); 123 } 124 int imm4 = disp / mesize / Matcher::scalable_vector_reg_size(vector_elem_bt); 125 (masm.*insn)(reg, Assembler::elemType_to_regVariant(vector_elem_bt), pg, Address(base, imm4)); 126 } else { 127 assert(false, "unimplemented"); 128 ShouldNotReachHere(); 129 } 130 } 131 132 bool op_sve_supported(int opcode, int vlen, BasicType bt) { 133 int length_in_bytes = vlen * type2aelembytes(bt); 134 switch (opcode) { 135 case Op_MulAddVS2VI: 136 // No multiply reduction instructions 137 case Op_MulReductionVD: 138 case Op_MulReductionVF: 139 case Op_MulReductionVI: 140 case Op_MulReductionVL: 141 // Others 142 case Op_ExtractC: 143 case Op_ExtractUB: 144 return false; 145 // Vector API specific 146 case Op_VectorLoadShuffle: 147 case Op_VectorRearrange: 148 return vlen >= 4 && length_in_bytes <= MaxVectorSize; 149 case Op_LoadVector: 150 case Op_StoreVector: 151 return Matcher::vector_size_supported(bt, vlen); 152 case Op_ExpandV: 153 if (UseSVE < 2 || is_subword_type(bt)) return false; 154 default: 155 break; 156 } 157 // By default, we only support vector operations with no less than 8 bytes and 2 elements. 158 return 8 <= length_in_bytes && length_in_bytes <= MaxVectorSize && vlen >= 2; 159 } 160 161 bool masked_op_sve_supported(int opcode, int vlen, BasicType bt) { 162 if (opcode == Op_VectorRearrange) { 163 return false; 164 } 165 return op_sve_supported(opcode, vlen, bt); 166 } 167 %} 168 169 definitions %{ 170 int_def SVE_COST (200, 200); 171 %} 172 173 174 // All SVE instructions 175 176 // vector load/store 177 178 // Unpredicated vector load/store 179 instruct loadV(vReg dst, vmemA mem) %{ 180 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() >= 16 && 181 n->as_LoadVector()->memory_size() == MaxVectorSize); 182 match(Set dst (LoadVector mem)); 183 ins_cost(4 * SVE_COST); 184 format %{ "sve_ldr $dst, $mem\t# vector (sve)" %} 185 ins_encode %{ 186 FloatRegister dst_reg = as_FloatRegister($dst$$reg); 187 BasicType bt = Matcher::vector_element_basic_type(this); 188 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg, ptrue, 189 bt, bt, $mem->opcode(), 190 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); 191 %} 192 ins_pipe(pipe_slow); 193 %} 194 195 instruct storeV(vReg src, vmemA mem) %{ 196 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() >= 16 && 197 n->as_StoreVector()->memory_size() == MaxVectorSize); 198 match(Set mem (StoreVector mem src)); 199 ins_cost(4 * SVE_COST); 200 format %{ "sve_str $mem, $src\t# vector (sve)" %} 201 ins_encode %{ 202 FloatRegister src_reg = as_FloatRegister($src$$reg); 203 BasicType bt = Matcher::vector_element_basic_type(this, $src); 204 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg, ptrue, 205 bt, bt, $mem->opcode(), 206 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); 207 %} 208 ins_pipe(pipe_slow); 209 %} 210 211 // Load Vector (16 bits) 212 instruct loadV2_vreg(vReg dst, vmem2 mem) 213 %{ 214 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 2); 215 match(Set dst (LoadVector mem)); 216 ins_cost(4 * INSN_COST); 217 format %{ "ldrh $dst,$mem\t# vector (16 bits)" %} 218 ins_encode( aarch64_enc_ldrvH(dst, mem) ); 219 ins_pipe(vload_reg_mem64); 220 %} 221 222 // Store Vector (16 bits) 223 instruct storeV2_vreg(vReg src, vmem2 mem) 224 %{ 225 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 2); 226 match(Set mem (StoreVector mem src)); 227 ins_cost(4 * INSN_COST); 228 format %{ "strh $mem,$src\t# vector (16 bits)" %} 229 ins_encode( aarch64_enc_strvH(src, mem) ); 230 ins_pipe(vstore_reg_mem64); 231 %} 232 233 // Load Vector (32 bits) 234 instruct loadV4_vreg(vReg dst, vmem4 mem) 235 %{ 236 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 4); 237 match(Set dst (LoadVector mem)); 238 ins_cost(4 * INSN_COST); 239 format %{ "ldrs $dst,$mem\t# vector (32 bits)" %} 240 ins_encode( aarch64_enc_ldrvS(dst, mem) ); 241 ins_pipe(vload_reg_mem64); 242 %} 243 244 // Store Vector (32 bits) 245 instruct storeV4_vreg(vReg src, vmem4 mem) 246 %{ 247 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 4); 248 match(Set mem (StoreVector mem src)); 249 ins_cost(4 * INSN_COST); 250 format %{ "strs $mem,$src\t# vector (32 bits)" %} 251 ins_encode( aarch64_enc_strvS(src, mem) ); 252 ins_pipe(vstore_reg_mem64); 253 %} 254 255 // Load Vector (64 bits) 256 instruct loadV8_vreg(vReg dst, vmem8 mem) 257 %{ 258 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 8); 259 match(Set dst (LoadVector mem)); 260 ins_cost(4 * INSN_COST); 261 format %{ "ldrd $dst,$mem\t# vector (64 bits)" %} 262 ins_encode( aarch64_enc_ldrvD(dst, mem) ); 263 ins_pipe(vload_reg_mem64); 264 %} 265 266 // Store Vector (64 bits) 267 instruct storeV8_vreg(vReg src, vmem8 mem) 268 %{ 269 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 8); 270 match(Set mem (StoreVector mem src)); 271 ins_cost(4 * INSN_COST); 272 format %{ "strd $mem,$src\t# vector (64 bits)" %} 273 ins_encode( aarch64_enc_strvD(src, mem) ); 274 ins_pipe(vstore_reg_mem64); 275 %} 276 277 // Load Vector (128 bits) 278 instruct loadV16_vreg(vReg dst, vmem16 mem) 279 %{ 280 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 16); 281 match(Set dst (LoadVector mem)); 282 ins_cost(4 * INSN_COST); 283 format %{ "ldrq $dst,$mem\t# vector (128 bits)" %} 284 ins_encode( aarch64_enc_ldrvQ(dst, mem) ); 285 ins_pipe(vload_reg_mem128); 286 %} 287 288 // Store Vector (128 bits) 289 instruct storeV16_vreg(vReg src, vmem16 mem) 290 %{ 291 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 16); 292 match(Set mem (StoreVector mem src)); 293 ins_cost(4 * INSN_COST); 294 format %{ "strq $mem,$src\t# vector (128 bits)" %} 295 ins_encode( aarch64_enc_strvQ(src, mem) ); 296 ins_pipe(vstore_reg_mem128); 297 %} 298 299 // Predicated vector load/store, based on the vector length of the node. 300 // Only load/store values in the range of the memory_size. This is needed 301 // when the memory_size is lower than the hardware supported max vector size. 302 // And this might happen for Vector API mask vector load/store. 303 instruct loadV_partial(vReg dst, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{ 304 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() > 16 && 305 n->as_LoadVector()->memory_size() < MaxVectorSize); 306 match(Set dst (LoadVector mem)); 307 effect(TEMP pgtmp, KILL cr); 308 ins_cost(6 * SVE_COST); 309 format %{ "sve_ptrue $pgtmp, vector_length\n\t" 310 "sve_ldr $dst, $pgtmp, $mem\t# load vector partial" %} 311 ins_encode %{ 312 BasicType bt = Matcher::vector_element_basic_type(this); 313 __ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt), 314 Matcher::vector_length(this)); 315 FloatRegister dst_reg = as_FloatRegister($dst$$reg); 316 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg, 317 as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(), 318 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); 319 %} 320 ins_pipe(pipe_slow); 321 %} 322 323 instruct storeV_partial(vReg src, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{ 324 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() > 16 && 325 n->as_StoreVector()->memory_size() < MaxVectorSize); 326 match(Set mem (StoreVector mem src)); 327 effect(TEMP pgtmp, KILL cr); 328 ins_cost(5 * SVE_COST); 329 format %{ "sve_ptrue $pgtmp, vector_length\n\t" 330 "sve_str $src, $pgtmp, $mem\t# store vector partial" %} 331 ins_encode %{ 332 BasicType bt = Matcher::vector_element_basic_type(this, $src); 333 __ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt), 334 Matcher::vector_length(this, $src)); 335 FloatRegister src_reg = as_FloatRegister($src$$reg); 336 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg, 337 as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(), 338 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); 339 %} 340 ins_pipe(pipe_slow); 341 %} 342 343 // vector load/store - predicated 344 345 instruct loadV_masked(vReg dst, vmemA mem, pRegGov pg) %{ 346 predicate(UseSVE > 0); 347 match(Set dst (LoadVectorMasked mem pg)); 348 ins_cost(4 * SVE_COST); 349 format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated (sve)" %} 350 ins_encode %{ 351 BasicType bt = Matcher::vector_element_basic_type(this); 352 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg), 353 as_PRegister($pg$$reg), bt, bt, $mem->opcode(), 354 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); 355 %} 356 ins_pipe(pipe_slow); 357 %} 358 359 instruct storeV_masked(vReg src, vmemA mem, pRegGov pg) %{ 360 predicate(UseSVE > 0); 361 match(Set mem (StoreVectorMasked mem (Binary src pg))); 362 ins_cost(4 * SVE_COST); 363 format %{ "sve_str $mem, $pg, $src\t# store vector predicated (sve)" %} 364 ins_encode %{ 365 BasicType bt = Matcher::vector_element_basic_type(this, $src); 366 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg), 367 as_PRegister($pg$$reg), bt, bt, $mem->opcode(), 368 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); 369 %} 370 ins_pipe(pipe_slow); 371 %} 372 373 // mask logical and/or/xor 374 375 instruct vmask_and(pRegGov pd, pRegGov pn, pRegGov pm) %{ 376 predicate(UseSVE > 0); 377 match(Set pd (AndVMask pn pm)); 378 ins_cost(SVE_COST); 379 format %{ "sve_and $pd, $pn, $pm\t# predicate (sve)" %} 380 ins_encode %{ 381 __ sve_and(as_PRegister($pd$$reg), ptrue, 382 as_PRegister($pn$$reg), as_PRegister($pm$$reg)); 383 %} 384 ins_pipe(pipe_slow); 385 %} 386 387 instruct vmask_or(pRegGov pd, pRegGov pn, pRegGov pm) %{ 388 predicate(UseSVE > 0); 389 match(Set pd (OrVMask pn pm)); 390 ins_cost(SVE_COST); 391 format %{ "sve_orr $pd, $pn, $pm\t# predicate (sve)" %} 392 ins_encode %{ 393 __ sve_orr(as_PRegister($pd$$reg), ptrue, 394 as_PRegister($pn$$reg), as_PRegister($pm$$reg)); 395 %} 396 ins_pipe(pipe_slow); 397 %} 398 399 instruct vmask_xor(pRegGov pd, pRegGov pn, pRegGov pm) %{ 400 predicate(UseSVE > 0); 401 match(Set pd (XorVMask pn pm)); 402 ins_cost(SVE_COST); 403 format %{ "sve_eor $pd, $pn, $pm\t# predicate (sve)" %} 404 ins_encode %{ 405 __ sve_eor(as_PRegister($pd$$reg), ptrue, 406 as_PRegister($pn$$reg), as_PRegister($pm$$reg)); 407 %} 408 ins_pipe(pipe_slow); 409 %} 410 411 // mask logical and_not 412 413 instruct vmask_and_notI(pRegGov pd, pRegGov pn, pRegGov pm, immI_M1 m1) %{ 414 predicate(UseSVE > 0); 415 match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1)))); 416 ins_cost(SVE_COST); 417 format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) (B/H/S)" %} 418 ins_encode %{ 419 __ sve_bic(as_PRegister($pd$$reg), ptrue, 420 as_PRegister($pn$$reg), as_PRegister($pm$$reg)); 421 %} 422 ins_pipe(pipe_slow); 423 %} 424 425 instruct vmask_and_notL(pRegGov pd, pRegGov pn, pRegGov pm, immL_M1 m1) %{ 426 predicate(UseSVE > 0); 427 match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1)))); 428 ins_cost(SVE_COST); 429 format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) (D)" %} 430 ins_encode %{ 431 __ sve_bic(as_PRegister($pd$$reg), ptrue, 432 as_PRegister($pn$$reg), as_PRegister($pm$$reg)); 433 %} 434 ins_pipe(pipe_slow); 435 %} 436 437 // vector reinterpret 438 439 instruct reinterpret(vReg dst) %{ 440 predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() == 441 n->in(1)->bottom_type()->is_vect()->length_in_bytes()); // src == dst 442 match(Set dst (VectorReinterpret dst)); 443 ins_cost(0); 444 format %{ "# reinterpret $dst\t# do nothing" %} 445 ins_encode %{ 446 // empty 447 %} 448 ins_pipe(pipe_class_empty); 449 %} 450 451 instruct reinterpretResize(vReg dst, vReg src, pRegGov pgtmp, rFlagsReg cr) %{ 452 predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() != 453 n->in(1)->bottom_type()->is_vect()->length_in_bytes()); // src != dst 454 match(Set dst (VectorReinterpret src)); 455 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr); 456 ins_cost(3 * SVE_COST); 457 format %{ "reinterpretResize $dst, $src\t# vector (sve)" %} 458 ins_encode %{ 459 uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src); 460 uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this); 461 uint length_in_bytes_resize = length_in_bytes_src < length_in_bytes_dst ? 462 length_in_bytes_src : length_in_bytes_dst; 463 assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize, 464 "invalid vector length"); 465 __ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ B, length_in_bytes_resize); 466 __ sve_dup(as_FloatRegister($dst$$reg), __ B, 0); 467 __ sve_sel(as_FloatRegister($dst$$reg), __ B, as_PRegister($pgtmp$$reg), 468 as_FloatRegister($src$$reg), as_FloatRegister($dst$$reg)); 469 %} 470 ins_pipe(pipe_slow); 471 %} 472 473 // vector mask reinterpret 474 475 instruct vmask_reinterpret_same_esize(pRegGov dst_src) %{ 476 predicate(UseSVE > 0 && 477 n->as_Vector()->length() == n->in(1)->bottom_type()->is_vect()->length() && 478 n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes()); 479 match(Set dst_src (VectorReinterpret dst_src)); 480 ins_cost(0); 481 format %{ "# vmask_reinterpret $dst_src\t# do nothing" %} 482 ins_encode %{ 483 // empty 484 %} 485 ins_pipe(pipe_class_empty); 486 %} 487 488 instruct vmask_reinterpret_diff_esize(pRegGov dst, pRegGov src, vReg tmp, rFlagsReg cr) %{ 489 predicate(UseSVE > 0 && 490 n->as_Vector()->length() != n->in(1)->bottom_type()->is_vect()->length() && 491 n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes()); 492 match(Set dst (VectorReinterpret src)); 493 effect(TEMP tmp, KILL cr); 494 ins_cost(2 * SVE_COST); 495 format %{ "# vmask_reinterpret $dst, $src\t# vector (sve)" %} 496 ins_encode %{ 497 BasicType from_bt = Matcher::vector_element_basic_type(this, $src); 498 Assembler::SIMD_RegVariant from_size = __ elemType_to_regVariant(from_bt); 499 BasicType to_bt = Matcher::vector_element_basic_type(this); 500 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt); 501 __ sve_cpy(as_FloatRegister($tmp$$reg), from_size, as_PRegister($src$$reg), -1, false); 502 __ sve_cmp(Assembler::EQ, as_PRegister($dst$$reg), to_size, ptrue, as_FloatRegister($tmp$$reg), -1); 503 %} 504 ins_pipe(pipe_slow); 505 %} 506 507 // vector abs 508 509 instruct vabsB(vReg dst, vReg src) %{ 510 predicate(UseSVE > 0 && 511 !n->as_Vector()->is_predicated_vector()); 512 match(Set dst (AbsVB src)); 513 ins_cost(SVE_COST); 514 format %{ "sve_abs $dst, $src\t# vector (sve) (B)" %} 515 ins_encode %{ 516 __ sve_abs(as_FloatRegister($dst$$reg), __ B, 517 ptrue, as_FloatRegister($src$$reg)); 518 %} 519 ins_pipe(pipe_slow); 520 %} 521 522 instruct vabsS(vReg dst, vReg src) %{ 523 predicate(UseSVE > 0 && 524 !n->as_Vector()->is_predicated_vector()); 525 match(Set dst (AbsVS src)); 526 ins_cost(SVE_COST); 527 format %{ "sve_abs $dst, $src\t# vector (sve) (H)" %} 528 ins_encode %{ 529 __ sve_abs(as_FloatRegister($dst$$reg), __ H, 530 ptrue, as_FloatRegister($src$$reg)); 531 %} 532 ins_pipe(pipe_slow); 533 %} 534 535 instruct vabsI(vReg dst, vReg src) %{ 536 predicate(UseSVE > 0 && 537 !n->as_Vector()->is_predicated_vector()); 538 match(Set dst (AbsVI src)); 539 ins_cost(SVE_COST); 540 format %{ "sve_abs $dst, $src\t# vector (sve) (S)" %} 541 ins_encode %{ 542 __ sve_abs(as_FloatRegister($dst$$reg), __ S, 543 ptrue, as_FloatRegister($src$$reg)); 544 %} 545 ins_pipe(pipe_slow); 546 %} 547 548 instruct vabsL(vReg dst, vReg src) %{ 549 predicate(UseSVE > 0 && 550 !n->as_Vector()->is_predicated_vector()); 551 match(Set dst (AbsVL src)); 552 ins_cost(SVE_COST); 553 format %{ "sve_abs $dst, $src\t# vector (sve) (D)" %} 554 ins_encode %{ 555 __ sve_abs(as_FloatRegister($dst$$reg), __ D, 556 ptrue, as_FloatRegister($src$$reg)); 557 %} 558 ins_pipe(pipe_slow); 559 %} 560 561 instruct vabsF(vReg dst, vReg src) %{ 562 predicate(UseSVE > 0 && 563 !n->as_Vector()->is_predicated_vector()); 564 match(Set dst (AbsVF src)); 565 ins_cost(SVE_COST); 566 format %{ "sve_fabs $dst, $src\t# vector (sve) (S)" %} 567 ins_encode %{ 568 __ sve_fabs(as_FloatRegister($dst$$reg), __ S, 569 ptrue, as_FloatRegister($src$$reg)); 570 %} 571 ins_pipe(pipe_slow); 572 %} 573 574 instruct vabsD(vReg dst, vReg src) %{ 575 predicate(UseSVE > 0 && 576 !n->as_Vector()->is_predicated_vector()); 577 match(Set dst (AbsVD src)); 578 ins_cost(SVE_COST); 579 format %{ "sve_fabs $dst, $src\t# vector (sve) (D)" %} 580 ins_encode %{ 581 __ sve_fabs(as_FloatRegister($dst$$reg), __ D, 582 ptrue, as_FloatRegister($src$$reg)); 583 %} 584 ins_pipe(pipe_slow); 585 %} 586 587 // vector abs - predicated 588 589 instruct vabsB_masked(vReg dst_src, pRegGov pg) %{ 590 predicate(UseSVE > 0); 591 match(Set dst_src (AbsVB dst_src pg)); 592 ins_cost(SVE_COST); 593 format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (B)" %} 594 ins_encode %{ 595 __ sve_abs(as_FloatRegister($dst_src$$reg), __ B, 596 as_PRegister($pg$$reg), 597 as_FloatRegister($dst_src$$reg)); 598 %} 599 ins_pipe(pipe_slow); 600 %} 601 602 instruct vabsS_masked(vReg dst_src, pRegGov pg) %{ 603 predicate(UseSVE > 0); 604 match(Set dst_src (AbsVS dst_src pg)); 605 ins_cost(SVE_COST); 606 format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (H)" %} 607 ins_encode %{ 608 __ sve_abs(as_FloatRegister($dst_src$$reg), __ H, 609 as_PRegister($pg$$reg), 610 as_FloatRegister($dst_src$$reg)); 611 %} 612 ins_pipe(pipe_slow); 613 %} 614 615 instruct vabsI_masked(vReg dst_src, pRegGov pg) %{ 616 predicate(UseSVE > 0); 617 match(Set dst_src (AbsVI dst_src pg)); 618 ins_cost(SVE_COST); 619 format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (S)" %} 620 ins_encode %{ 621 __ sve_abs(as_FloatRegister($dst_src$$reg), __ S, 622 as_PRegister($pg$$reg), 623 as_FloatRegister($dst_src$$reg)); 624 %} 625 ins_pipe(pipe_slow); 626 %} 627 628 instruct vabsL_masked(vReg dst_src, pRegGov pg) %{ 629 predicate(UseSVE > 0); 630 match(Set dst_src (AbsVL dst_src pg)); 631 ins_cost(SVE_COST); 632 format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (D)" %} 633 ins_encode %{ 634 __ sve_abs(as_FloatRegister($dst_src$$reg), __ D, 635 as_PRegister($pg$$reg), 636 as_FloatRegister($dst_src$$reg)); 637 %} 638 ins_pipe(pipe_slow); 639 %} 640 641 instruct vabsF_masked(vReg dst_src, pRegGov pg) %{ 642 predicate(UseSVE > 0); 643 match(Set dst_src (AbsVF dst_src pg)); 644 ins_cost(SVE_COST); 645 format %{ "sve_fabs $dst_src, $pg, $dst_src\t# vector (sve) (S)" %} 646 ins_encode %{ 647 __ sve_fabs(as_FloatRegister($dst_src$$reg), __ S, 648 as_PRegister($pg$$reg), 649 as_FloatRegister($dst_src$$reg)); 650 %} 651 ins_pipe(pipe_slow); 652 %} 653 654 instruct vabsD_masked(vReg dst_src, pRegGov pg) %{ 655 predicate(UseSVE > 0); 656 match(Set dst_src (AbsVD dst_src pg)); 657 ins_cost(SVE_COST); 658 format %{ "sve_fabs $dst_src, $pg, $dst_src\t# vector (sve) (D)" %} 659 ins_encode %{ 660 __ sve_fabs(as_FloatRegister($dst_src$$reg), __ D, 661 as_PRegister($pg$$reg), 662 as_FloatRegister($dst_src$$reg)); 663 %} 664 ins_pipe(pipe_slow); 665 %} 666 667 // vector add 668 669 instruct vaddB(vReg dst, vReg src1, vReg src2) %{ 670 predicate(UseSVE > 0); 671 match(Set dst (AddVB src1 src2)); 672 ins_cost(SVE_COST); 673 format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (B)" %} 674 ins_encode %{ 675 __ sve_add(as_FloatRegister($dst$$reg), __ B, 676 as_FloatRegister($src1$$reg), 677 as_FloatRegister($src2$$reg)); 678 %} 679 ins_pipe(pipe_slow); 680 %} 681 682 instruct vaddS(vReg dst, vReg src1, vReg src2) %{ 683 predicate(UseSVE > 0); 684 match(Set dst (AddVS src1 src2)); 685 ins_cost(SVE_COST); 686 format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (H)" %} 687 ins_encode %{ 688 __ sve_add(as_FloatRegister($dst$$reg), __ H, 689 as_FloatRegister($src1$$reg), 690 as_FloatRegister($src2$$reg)); 691 %} 692 ins_pipe(pipe_slow); 693 %} 694 695 instruct vaddI(vReg dst, vReg src1, vReg src2) %{ 696 predicate(UseSVE > 0); 697 match(Set dst (AddVI src1 src2)); 698 ins_cost(SVE_COST); 699 format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (S)" %} 700 ins_encode %{ 701 __ sve_add(as_FloatRegister($dst$$reg), __ S, 702 as_FloatRegister($src1$$reg), 703 as_FloatRegister($src2$$reg)); 704 %} 705 ins_pipe(pipe_slow); 706 %} 707 708 instruct vaddL(vReg dst, vReg src1, vReg src2) %{ 709 predicate(UseSVE > 0); 710 match(Set dst (AddVL src1 src2)); 711 ins_cost(SVE_COST); 712 format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (D)" %} 713 ins_encode %{ 714 __ sve_add(as_FloatRegister($dst$$reg), __ D, 715 as_FloatRegister($src1$$reg), 716 as_FloatRegister($src2$$reg)); 717 %} 718 ins_pipe(pipe_slow); 719 %} 720 721 instruct vaddF(vReg dst, vReg src1, vReg src2) %{ 722 predicate(UseSVE > 0); 723 match(Set dst (AddVF src1 src2)); 724 ins_cost(SVE_COST); 725 format %{ "sve_fadd $dst, $src1, $src2\t # vector (sve) (S)" %} 726 ins_encode %{ 727 __ sve_fadd(as_FloatRegister($dst$$reg), __ S, 728 as_FloatRegister($src1$$reg), 729 as_FloatRegister($src2$$reg)); 730 %} 731 ins_pipe(pipe_slow); 732 %} 733 734 instruct vaddD(vReg dst, vReg src1, vReg src2) %{ 735 predicate(UseSVE > 0); 736 match(Set dst (AddVD src1 src2)); 737 ins_cost(SVE_COST); 738 format %{ "sve_fadd $dst, $src1, $src2\t # vector (sve) (D)" %} 739 ins_encode %{ 740 __ sve_fadd(as_FloatRegister($dst$$reg), __ D, 741 as_FloatRegister($src1$$reg), 742 as_FloatRegister($src2$$reg)); 743 %} 744 ins_pipe(pipe_slow); 745 %} 746 747 // vector add - predicated 748 749 instruct vaddB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 750 predicate(UseSVE > 0); 751 match(Set dst_src1 (AddVB (Binary dst_src1 src2) pg)); 752 ins_cost(SVE_COST); 753 format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %} 754 ins_encode %{ 755 __ sve_add(as_FloatRegister($dst_src1$$reg), __ B, 756 as_PRegister($pg$$reg), 757 as_FloatRegister($src2$$reg)); 758 %} 759 ins_pipe(pipe_slow); 760 %} 761 762 instruct vaddS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 763 predicate(UseSVE > 0); 764 match(Set dst_src1 (AddVS (Binary dst_src1 src2) pg)); 765 ins_cost(SVE_COST); 766 format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %} 767 ins_encode %{ 768 __ sve_add(as_FloatRegister($dst_src1$$reg), __ H, 769 as_PRegister($pg$$reg), 770 as_FloatRegister($src2$$reg)); 771 %} 772 ins_pipe(pipe_slow); 773 %} 774 775 instruct vaddI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 776 predicate(UseSVE > 0); 777 match(Set dst_src1 (AddVI (Binary dst_src1 src2) pg)); 778 ins_cost(SVE_COST); 779 format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %} 780 ins_encode %{ 781 __ sve_add(as_FloatRegister($dst_src1$$reg), __ S, 782 as_PRegister($pg$$reg), 783 as_FloatRegister($src2$$reg)); 784 %} 785 ins_pipe(pipe_slow); 786 %} 787 788 instruct vaddL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 789 predicate(UseSVE > 0); 790 match(Set dst_src1 (AddVL (Binary dst_src1 src2) pg)); 791 ins_cost(SVE_COST); 792 format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %} 793 ins_encode %{ 794 __ sve_add(as_FloatRegister($dst_src1$$reg), __ D, 795 as_PRegister($pg$$reg), 796 as_FloatRegister($src2$$reg)); 797 %} 798 ins_pipe(pipe_slow); 799 %} 800 801 instruct vaddF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 802 predicate(UseSVE > 0); 803 match(Set dst_src1 (AddVF (Binary dst_src1 src2) pg)); 804 ins_cost(SVE_COST); 805 format %{ "sve_fadd $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %} 806 ins_encode %{ 807 __ sve_fadd(as_FloatRegister($dst_src1$$reg), __ S, 808 as_PRegister($pg$$reg), 809 as_FloatRegister($src2$$reg)); 810 %} 811 ins_pipe(pipe_slow); 812 %} 813 814 instruct vaddD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 815 predicate(UseSVE > 0); 816 match(Set dst_src1 (AddVD (Binary dst_src1 src2) pg)); 817 ins_cost(SVE_COST); 818 format %{ "sve_fadd $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %} 819 ins_encode %{ 820 __ sve_fadd(as_FloatRegister($dst_src1$$reg), __ D, 821 as_PRegister($pg$$reg), 822 as_FloatRegister($src2$$reg)); 823 %} 824 ins_pipe(pipe_slow); 825 %} 826 827 // vector add reg imm (unpredicated) 828 829 instruct vaddImmB(vReg dst_src, immBAddSubV con) %{ 830 predicate(UseSVE > 0); 831 match(Set dst_src (AddVB dst_src (ReplicateB con))); 832 ins_cost(SVE_COST); 833 format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) (B)" %} 834 ins_encode %{ 835 int32_t val = $con$$constant; 836 if (val > 0){ 837 __ sve_add(as_FloatRegister($dst_src$$reg), __ B, val); 838 } else if (val < 0){ 839 __ sve_sub(as_FloatRegister($dst_src$$reg), __ B, -val); 840 } 841 %} 842 ins_pipe(pipe_slow); 843 %} 844 845 instruct vaddImmS(vReg dst_src, immIAddSubV con) %{ 846 predicate(UseSVE > 0); 847 match(Set dst_src (AddVS dst_src (ReplicateS con))); 848 ins_cost(SVE_COST); 849 format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) (H)" %} 850 ins_encode %{ 851 int32_t val = $con$$constant; 852 if (val > 0){ 853 __ sve_add(as_FloatRegister($dst_src$$reg), __ H, val); 854 } else if (val < 0){ 855 __ sve_sub(as_FloatRegister($dst_src$$reg), __ H, -val); 856 } 857 %} 858 ins_pipe(pipe_slow); 859 %} 860 861 instruct vaddImmI(vReg dst_src, immIAddSubV con) %{ 862 predicate(UseSVE > 0); 863 match(Set dst_src (AddVI dst_src (ReplicateI con))); 864 ins_cost(SVE_COST); 865 format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) (S)" %} 866 ins_encode %{ 867 int32_t val = $con$$constant; 868 if (val > 0){ 869 __ sve_add(as_FloatRegister($dst_src$$reg), __ S, val); 870 } else if (val < 0){ 871 __ sve_sub(as_FloatRegister($dst_src$$reg), __ S, -val); 872 } 873 %} 874 ins_pipe(pipe_slow); 875 %} 876 877 instruct vaddImmL(vReg dst_src, immLAddSubV con) %{ 878 predicate(UseSVE > 0); 879 match(Set dst_src (AddVL dst_src (ReplicateL con))); 880 ins_cost(SVE_COST); 881 format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) (D)" %} 882 ins_encode %{ 883 int32_t val = $con$$constant; 884 if (val > 0){ 885 __ sve_add(as_FloatRegister($dst_src$$reg), __ D, val); 886 } else if (val < 0){ 887 __ sve_sub(as_FloatRegister($dst_src$$reg), __ D, -val); 888 } 889 %} 890 ins_pipe(pipe_slow); 891 %} 892 893 // vector binary op reg imm (unpredicated) 894 895 instruct vandB(vReg dst_src, immBLog con) %{ 896 predicate(UseSVE > 0); 897 match(Set dst_src (AndV dst_src (ReplicateB con))); 898 ins_cost(SVE_COST); 899 format %{ "sve_and $dst_src, $dst_src, $con\t # vector (sve) (B)" %} 900 ins_encode %{ 901 __ sve_and(as_FloatRegister($dst_src$$reg), __ B, 902 (uint64_t)($con$$constant)); 903 %} 904 ins_pipe(pipe_slow); 905 %} 906 907 instruct vandH(vReg dst_src, immSLog con) %{ 908 predicate(UseSVE > 0); 909 match(Set dst_src (AndV dst_src (ReplicateS con))); 910 ins_cost(SVE_COST); 911 format %{ "sve_and $dst_src, $dst_src, $con\t # vector (sve) (H)" %} 912 ins_encode %{ 913 __ sve_and(as_FloatRegister($dst_src$$reg), __ H, 914 (uint64_t)($con$$constant)); 915 %} 916 ins_pipe(pipe_slow); 917 %} 918 919 instruct vandS(vReg dst_src, immILog con) %{ 920 predicate(UseSVE > 0); 921 match(Set dst_src (AndV dst_src (ReplicateI con))); 922 ins_cost(SVE_COST); 923 format %{ "sve_and $dst_src, $dst_src, $con\t # vector (sve) (S)" %} 924 ins_encode %{ 925 __ sve_and(as_FloatRegister($dst_src$$reg), __ S, 926 (uint64_t)($con$$constant)); 927 %} 928 ins_pipe(pipe_slow); 929 %} 930 931 instruct vandD(vReg dst_src, immLLog con) %{ 932 predicate(UseSVE > 0); 933 match(Set dst_src (AndV dst_src (ReplicateL con))); 934 ins_cost(SVE_COST); 935 format %{ "sve_and $dst_src, $dst_src, $con\t # vector (sve) (D)" %} 936 ins_encode %{ 937 __ sve_and(as_FloatRegister($dst_src$$reg), __ D, 938 (uint64_t)($con$$constant)); 939 %} 940 ins_pipe(pipe_slow); 941 %} 942 943 instruct vorB(vReg dst_src, immBLog con) %{ 944 predicate(UseSVE > 0); 945 match(Set dst_src (OrV dst_src (ReplicateB con))); 946 ins_cost(SVE_COST); 947 format %{ "sve_orr $dst_src, $dst_src, $con\t # vector (sve) (B)" %} 948 ins_encode %{ 949 __ sve_orr(as_FloatRegister($dst_src$$reg), __ B, 950 (uint64_t)($con$$constant)); 951 %} 952 ins_pipe(pipe_slow); 953 %} 954 955 instruct vorH(vReg dst_src, immSLog con) %{ 956 predicate(UseSVE > 0); 957 match(Set dst_src (OrV dst_src (ReplicateS con))); 958 ins_cost(SVE_COST); 959 format %{ "sve_orr $dst_src, $dst_src, $con\t # vector (sve) (H)" %} 960 ins_encode %{ 961 __ sve_orr(as_FloatRegister($dst_src$$reg), __ H, 962 (uint64_t)($con$$constant)); 963 %} 964 ins_pipe(pipe_slow); 965 %} 966 967 instruct vorS(vReg dst_src, immILog con) %{ 968 predicate(UseSVE > 0); 969 match(Set dst_src (OrV dst_src (ReplicateI con))); 970 ins_cost(SVE_COST); 971 format %{ "sve_orr $dst_src, $dst_src, $con\t # vector (sve) (S)" %} 972 ins_encode %{ 973 __ sve_orr(as_FloatRegister($dst_src$$reg), __ S, 974 (uint64_t)($con$$constant)); 975 %} 976 ins_pipe(pipe_slow); 977 %} 978 979 instruct vorD(vReg dst_src, immLLog con) %{ 980 predicate(UseSVE > 0); 981 match(Set dst_src (OrV dst_src (ReplicateL con))); 982 ins_cost(SVE_COST); 983 format %{ "sve_orr $dst_src, $dst_src, $con\t # vector (sve) (D)" %} 984 ins_encode %{ 985 __ sve_orr(as_FloatRegister($dst_src$$reg), __ D, 986 (uint64_t)($con$$constant)); 987 %} 988 ins_pipe(pipe_slow); 989 %} 990 991 instruct vxorB(vReg dst_src, immBLog con) %{ 992 predicate(UseSVE > 0); 993 match(Set dst_src (XorV dst_src (ReplicateB con))); 994 ins_cost(SVE_COST); 995 format %{ "sve_eor $dst_src, $dst_src, $con\t # vector (sve) (B)" %} 996 ins_encode %{ 997 __ sve_eor(as_FloatRegister($dst_src$$reg), __ B, 998 (uint64_t)($con$$constant)); 999 %} 1000 ins_pipe(pipe_slow); 1001 %} 1002 1003 instruct vxorH(vReg dst_src, immSLog con) %{ 1004 predicate(UseSVE > 0); 1005 match(Set dst_src (XorV dst_src (ReplicateS con))); 1006 ins_cost(SVE_COST); 1007 format %{ "sve_eor $dst_src, $dst_src, $con\t # vector (sve) (H)" %} 1008 ins_encode %{ 1009 __ sve_eor(as_FloatRegister($dst_src$$reg), __ H, 1010 (uint64_t)($con$$constant)); 1011 %} 1012 ins_pipe(pipe_slow); 1013 %} 1014 1015 instruct vxorS(vReg dst_src, immILog con) %{ 1016 predicate(UseSVE > 0); 1017 match(Set dst_src (XorV dst_src (ReplicateI con))); 1018 ins_cost(SVE_COST); 1019 format %{ "sve_eor $dst_src, $dst_src, $con\t # vector (sve) (S)" %} 1020 ins_encode %{ 1021 __ sve_eor(as_FloatRegister($dst_src$$reg), __ S, 1022 (uint64_t)($con$$constant)); 1023 %} 1024 ins_pipe(pipe_slow); 1025 %} 1026 1027 instruct vxorD(vReg dst_src, immLLog con) %{ 1028 predicate(UseSVE > 0); 1029 match(Set dst_src (XorV dst_src (ReplicateL con))); 1030 ins_cost(SVE_COST); 1031 format %{ "sve_eor $dst_src, $dst_src, $con\t # vector (sve) (D)" %} 1032 ins_encode %{ 1033 __ sve_eor(as_FloatRegister($dst_src$$reg), __ D, 1034 (uint64_t)($con$$constant)); 1035 %} 1036 ins_pipe(pipe_slow); 1037 %} 1038 // vector and 1039 1040 instruct vand(vReg dst, vReg src1, vReg src2) %{ 1041 predicate(UseSVE > 0); 1042 match(Set dst (AndV src1 src2)); 1043 ins_cost(SVE_COST); 1044 format %{ "sve_and $dst, $src1, $src2\t# vector (sve)" %} 1045 ins_encode %{ 1046 __ sve_and(as_FloatRegister($dst$$reg), 1047 as_FloatRegister($src1$$reg), 1048 as_FloatRegister($src2$$reg)); 1049 %} 1050 ins_pipe(pipe_slow); 1051 %} 1052 1053 // vector or 1054 1055 instruct vor(vReg dst, vReg src1, vReg src2) %{ 1056 predicate(UseSVE > 0); 1057 match(Set dst (OrV src1 src2)); 1058 ins_cost(SVE_COST); 1059 format %{ "sve_orr $dst, $src1, $src2\t# vector (sve)" %} 1060 ins_encode %{ 1061 __ sve_orr(as_FloatRegister($dst$$reg), 1062 as_FloatRegister($src1$$reg), 1063 as_FloatRegister($src2$$reg)); 1064 %} 1065 ins_pipe(pipe_slow); 1066 %} 1067 1068 // vector xor 1069 1070 instruct vxor(vReg dst, vReg src1, vReg src2) %{ 1071 predicate(UseSVE > 0); 1072 match(Set dst (XorV src1 src2)); 1073 ins_cost(SVE_COST); 1074 format %{ "sve_eor $dst, $src1, $src2\t# vector (sve)" %} 1075 ins_encode %{ 1076 __ sve_eor(as_FloatRegister($dst$$reg), 1077 as_FloatRegister($src1$$reg), 1078 as_FloatRegister($src2$$reg)); 1079 %} 1080 ins_pipe(pipe_slow); 1081 %} 1082 1083 // vector and - predicated 1084 1085 instruct vand_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 1086 predicate(UseSVE > 0); 1087 match(Set dst_src1 (AndV (Binary dst_src1 src2) pg)); 1088 ins_cost(SVE_COST); 1089 format %{ "sve_and $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %} 1090 ins_encode %{ 1091 BasicType bt = Matcher::vector_element_basic_type(this); 1092 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 1093 __ sve_and(as_FloatRegister($dst_src1$$reg), size, 1094 as_PRegister($pg$$reg), 1095 as_FloatRegister($src2$$reg)); 1096 %} 1097 ins_pipe(pipe_slow); 1098 %} 1099 1100 // vector or - predicated 1101 1102 instruct vor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 1103 predicate(UseSVE > 0); 1104 match(Set dst_src1 (OrV (Binary dst_src1 src2) pg)); 1105 ins_cost(SVE_COST); 1106 format %{ "sve_orr $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %} 1107 ins_encode %{ 1108 BasicType bt = Matcher::vector_element_basic_type(this); 1109 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 1110 __ sve_orr(as_FloatRegister($dst_src1$$reg), size, 1111 as_PRegister($pg$$reg), 1112 as_FloatRegister($src2$$reg)); 1113 %} 1114 ins_pipe(pipe_slow); 1115 %} 1116 1117 // vector xor - predicated 1118 1119 instruct vxor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 1120 predicate(UseSVE > 0); 1121 match(Set dst_src1 (XorV (Binary dst_src1 src2) pg)); 1122 ins_cost(SVE_COST); 1123 format %{ "sve_eor $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %} 1124 ins_encode %{ 1125 BasicType bt = Matcher::vector_element_basic_type(this); 1126 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 1127 __ sve_eor(as_FloatRegister($dst_src1$$reg), size, 1128 as_PRegister($pg$$reg), 1129 as_FloatRegister($src2$$reg)); 1130 %} 1131 ins_pipe(pipe_slow); 1132 %} 1133 1134 // vector not 1135 1136 instruct vnotI(vReg dst, vReg src, immI_M1 m1) %{ 1137 predicate(UseSVE > 0); 1138 match(Set dst (XorV src (ReplicateB m1))); 1139 match(Set dst (XorV src (ReplicateS m1))); 1140 match(Set dst (XorV src (ReplicateI m1))); 1141 ins_cost(SVE_COST); 1142 format %{ "sve_not $dst, $src\t# vector (sve) B/H/S" %} 1143 ins_encode %{ 1144 __ sve_not(as_FloatRegister($dst$$reg), __ D, 1145 ptrue, as_FloatRegister($src$$reg)); 1146 %} 1147 ins_pipe(pipe_slow); 1148 %} 1149 1150 instruct vnotL(vReg dst, vReg src, immL_M1 m1) %{ 1151 predicate(UseSVE > 0); 1152 match(Set dst (XorV src (ReplicateL m1))); 1153 ins_cost(SVE_COST); 1154 format %{ "sve_not $dst, $src\t# vector (sve) D" %} 1155 ins_encode %{ 1156 __ sve_not(as_FloatRegister($dst$$reg), __ D, 1157 ptrue, as_FloatRegister($src$$reg)); 1158 %} 1159 ins_pipe(pipe_slow); 1160 %} 1161 1162 // vector not - predicated 1163 1164 instruct vnotI_masked(vReg dst, vReg src, immI_M1 m1, pRegGov pg) %{ 1165 predicate(UseSVE > 0); 1166 match(Set dst (XorV (Binary src (ReplicateB m1)) pg)); 1167 match(Set dst (XorV (Binary src (ReplicateS m1)) pg)); 1168 match(Set dst (XorV (Binary src (ReplicateI m1)) pg)); 1169 ins_cost(SVE_COST); 1170 format %{ "sve_not $dst, $pg, $src\t# vector (sve) B/H/S" %} 1171 ins_encode %{ 1172 BasicType bt = Matcher::vector_element_basic_type(this); 1173 __ sve_not(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt), 1174 as_PRegister($pg$$reg), as_FloatRegister($src$$reg)); 1175 %} 1176 ins_pipe(pipe_slow); 1177 %} 1178 1179 instruct vnotL_masked(vReg dst, vReg src, immL_M1 m1, pRegGov pg) %{ 1180 predicate(UseSVE > 0); 1181 match(Set dst (XorV (Binary src (ReplicateL m1)) pg)); 1182 ins_cost(SVE_COST); 1183 format %{ "sve_not $dst, $pg, $src\t# vector (sve) D" %} 1184 ins_encode %{ 1185 BasicType bt = Matcher::vector_element_basic_type(this); 1186 __ sve_not(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt), 1187 as_PRegister($pg$$reg), as_FloatRegister($src$$reg)); 1188 %} 1189 ins_pipe(pipe_slow); 1190 %} 1191 1192 // vector and_not 1193 1194 instruct vand_notI(vReg dst, vReg src1, vReg src2, immI_M1 m1) %{ 1195 predicate(UseSVE > 0); 1196 match(Set dst (AndV src1 (XorV src2 (ReplicateB m1)))); 1197 match(Set dst (AndV src1 (XorV src2 (ReplicateS m1)))); 1198 match(Set dst (AndV src1 (XorV src2 (ReplicateI m1)))); 1199 ins_cost(SVE_COST); 1200 format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) B/H/S" %} 1201 ins_encode %{ 1202 __ sve_bic(as_FloatRegister($dst$$reg), 1203 as_FloatRegister($src1$$reg), 1204 as_FloatRegister($src2$$reg)); 1205 %} 1206 ins_pipe(pipe_slow); 1207 %} 1208 1209 instruct vand_notL(vReg dst, vReg src1, vReg src2, immL_M1 m1) %{ 1210 predicate(UseSVE > 0); 1211 match(Set dst (AndV src1 (XorV src2 (ReplicateL m1)))); 1212 ins_cost(SVE_COST); 1213 format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) D" %} 1214 ins_encode %{ 1215 __ sve_bic(as_FloatRegister($dst$$reg), 1216 as_FloatRegister($src1$$reg), 1217 as_FloatRegister($src2$$reg)); 1218 %} 1219 ins_pipe(pipe_slow); 1220 %} 1221 1222 // vector and_not - predicated 1223 1224 instruct vand_notI_masked(vReg dst_src1, vReg src2, immI_M1 m1, pRegGov pg) %{ 1225 predicate(UseSVE > 0); 1226 match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateB m1))) pg)); 1227 match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateS m1))) pg)); 1228 match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateI m1))) pg)); 1229 ins_cost(SVE_COST); 1230 format %{ "sve_bic $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) B/H/S" %} 1231 ins_encode %{ 1232 BasicType bt = Matcher::vector_element_basic_type(this); 1233 __ sve_bic(as_FloatRegister($dst_src1$$reg), __ elemType_to_regVariant(bt), 1234 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg)); 1235 %} 1236 ins_pipe(pipe_slow); 1237 %} 1238 1239 instruct vand_notL_masked(vReg dst_src1, vReg src2, immL_M1 m1, pRegGov pg) %{ 1240 predicate(UseSVE > 0); 1241 match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateL m1))) pg)); 1242 ins_cost(SVE_COST); 1243 format %{ "sve_bic $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) D" %} 1244 ins_encode %{ 1245 BasicType bt = Matcher::vector_element_basic_type(this); 1246 __ sve_bic(as_FloatRegister($dst_src1$$reg), __ elemType_to_regVariant(bt), 1247 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg)); 1248 %} 1249 ins_pipe(pipe_slow); 1250 %} 1251 1252 // vector float div 1253 1254 instruct vdivF(vReg dst_src1, vReg src2) %{ 1255 predicate(UseSVE > 0); 1256 match(Set dst_src1 (DivVF dst_src1 src2)); 1257 ins_cost(SVE_COST); 1258 format %{ "sve_fdiv $dst_src1, $dst_src1, $src2\t# vector (sve) (S)" %} 1259 ins_encode %{ 1260 __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ S, 1261 ptrue, as_FloatRegister($src2$$reg)); 1262 %} 1263 ins_pipe(pipe_slow); 1264 %} 1265 1266 instruct vdivD(vReg dst_src1, vReg src2) %{ 1267 predicate(UseSVE > 0); 1268 match(Set dst_src1 (DivVD dst_src1 src2)); 1269 ins_cost(SVE_COST); 1270 format %{ "sve_fdiv $dst_src1, $dst_src1, $src2\t# vector (sve) (D)" %} 1271 ins_encode %{ 1272 __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ D, 1273 ptrue, as_FloatRegister($src2$$reg)); 1274 %} 1275 ins_pipe(pipe_slow); 1276 %} 1277 1278 // vector float div - predicated 1279 1280 instruct vfdivF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 1281 predicate(UseSVE > 0); 1282 match(Set dst_src1 (DivVF (Binary dst_src1 src2) pg)); 1283 ins_cost(SVE_COST); 1284 format %{ "sve_fdiv $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %} 1285 ins_encode %{ 1286 __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ S, 1287 as_PRegister($pg$$reg), 1288 as_FloatRegister($src2$$reg)); 1289 %} 1290 ins_pipe(pipe_slow); 1291 %} 1292 1293 instruct vfdivD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 1294 predicate(UseSVE > 0); 1295 match(Set dst_src1 (DivVD (Binary dst_src1 src2) pg)); 1296 ins_cost(SVE_COST); 1297 format %{ "sve_fdiv $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %} 1298 ins_encode %{ 1299 __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ D, 1300 as_PRegister($pg$$reg), 1301 as_FloatRegister($src2$$reg)); 1302 %} 1303 ins_pipe(pipe_slow); 1304 %} 1305 1306 // vector min/max 1307 1308 instruct vmin(vReg dst_src1, vReg src2) %{ 1309 predicate(UseSVE > 0); 1310 match(Set dst_src1 (MinV dst_src1 src2)); 1311 ins_cost(SVE_COST); 1312 format %{ "sve_min $dst_src1, $dst_src1, $src2\t # vector (sve)" %} 1313 ins_encode %{ 1314 BasicType bt = Matcher::vector_element_basic_type(this); 1315 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 1316 if (is_floating_point_type(bt)) { 1317 __ sve_fmin(as_FloatRegister($dst_src1$$reg), size, 1318 ptrue, as_FloatRegister($src2$$reg)); 1319 } else { 1320 assert(is_integral_type(bt), "unsupported type"); 1321 __ sve_smin(as_FloatRegister($dst_src1$$reg), size, 1322 ptrue, as_FloatRegister($src2$$reg)); 1323 } 1324 %} 1325 ins_pipe(pipe_slow); 1326 %} 1327 1328 instruct vmax(vReg dst_src1, vReg src2) %{ 1329 predicate(UseSVE > 0); 1330 match(Set dst_src1 (MaxV dst_src1 src2)); 1331 ins_cost(SVE_COST); 1332 format %{ "sve_max $dst_src1, $dst_src1, $src2\t # vector (sve)" %} 1333 ins_encode %{ 1334 BasicType bt = Matcher::vector_element_basic_type(this); 1335 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 1336 if (is_floating_point_type(bt)) { 1337 __ sve_fmax(as_FloatRegister($dst_src1$$reg), size, 1338 ptrue, as_FloatRegister($src2$$reg)); 1339 } else { 1340 assert(is_integral_type(bt), "unsupported type"); 1341 __ sve_smax(as_FloatRegister($dst_src1$$reg), size, 1342 ptrue, as_FloatRegister($src2$$reg)); 1343 } 1344 %} 1345 ins_pipe(pipe_slow); 1346 %} 1347 1348 // vector min/max - predicated 1349 1350 instruct vmin_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 1351 predicate(UseSVE > 0); 1352 match(Set dst_src1 (MinV (Binary dst_src1 src2) pg)); 1353 ins_cost(SVE_COST); 1354 format %{ "sve_min $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %} 1355 ins_encode %{ 1356 BasicType bt = Matcher::vector_element_basic_type(this); 1357 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 1358 if (is_floating_point_type(bt)) { 1359 __ sve_fmin(as_FloatRegister($dst_src1$$reg), size, 1360 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg)); 1361 } else { 1362 assert(is_integral_type(bt), "unsupported type"); 1363 __ sve_smin(as_FloatRegister($dst_src1$$reg), size, 1364 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg)); 1365 } 1366 %} 1367 ins_pipe(pipe_slow); 1368 %} 1369 1370 instruct vmax_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 1371 predicate(UseSVE > 0); 1372 match(Set dst_src1 (MaxV (Binary dst_src1 src2) pg)); 1373 ins_cost(SVE_COST); 1374 format %{ "sve_max $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %} 1375 ins_encode %{ 1376 BasicType bt = Matcher::vector_element_basic_type(this); 1377 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 1378 if (is_floating_point_type(bt)) { 1379 __ sve_fmax(as_FloatRegister($dst_src1$$reg), size, 1380 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg)); 1381 } else { 1382 assert(is_integral_type(bt), "unsupported type"); 1383 __ sve_smax(as_FloatRegister($dst_src1$$reg), size, 1384 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg)); 1385 } 1386 %} 1387 ins_pipe(pipe_slow); 1388 %} 1389 1390 // vector fmla 1391 1392 // dst_src1 = dst_src1 + src2 * src3 1393 instruct vfmlaF(vReg dst_src1, vReg src2, vReg src3) %{ 1394 predicate(UseFMA && UseSVE > 0); 1395 match(Set dst_src1 (FmaVF dst_src1 (Binary src2 src3))); 1396 ins_cost(SVE_COST); 1397 format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %} 1398 ins_encode %{ 1399 __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ S, 1400 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1401 %} 1402 ins_pipe(pipe_slow); 1403 %} 1404 1405 // dst_src1 = dst_src1 + src2 * src3 1406 instruct vfmlaD(vReg dst_src1, vReg src2, vReg src3) %{ 1407 predicate(UseFMA && UseSVE > 0); 1408 match(Set dst_src1 (FmaVD dst_src1 (Binary src2 src3))); 1409 ins_cost(SVE_COST); 1410 format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %} 1411 ins_encode %{ 1412 __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ D, 1413 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1414 %} 1415 ins_pipe(pipe_slow); 1416 %} 1417 1418 // vector fmad - predicated 1419 1420 // dst_src1 = dst_src1 * src2 + src3 1421 instruct vfmadF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{ 1422 predicate(UseFMA && UseSVE > 0); 1423 match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary src3 pg))); 1424 ins_cost(SVE_COST); 1425 format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) (S)" %} 1426 ins_encode %{ 1427 __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg), 1428 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1429 %} 1430 ins_pipe(pipe_slow); 1431 %} 1432 1433 // dst_src1 = dst_src1 * src2 + src3 1434 instruct vfmadD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{ 1435 predicate(UseFMA && UseSVE > 0); 1436 match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary src3 pg))); 1437 ins_cost(SVE_COST); 1438 format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) (D)" %} 1439 ins_encode %{ 1440 __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg), 1441 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1442 %} 1443 ins_pipe(pipe_slow); 1444 %} 1445 1446 // vector fmls 1447 1448 // dst_src1 = dst_src1 + -src2 * src3 1449 // The NegVF must not be predicated. 1450 instruct vfmlsF1(vReg dst_src1, vReg src2, vReg src3) %{ 1451 predicate(UseFMA && UseSVE > 0 && 1452 !n->in(2)->in(1)->as_Vector()->is_predicated_vector()); 1453 match(Set dst_src1 (FmaVF dst_src1 (Binary (NegVF src2) src3))); 1454 ins_cost(SVE_COST); 1455 format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %} 1456 ins_encode %{ 1457 __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ S, 1458 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1459 %} 1460 ins_pipe(pipe_slow); 1461 %} 1462 1463 // dst_src1 = dst_src1 + src2 * -src3 1464 // The NegVF must not be predicated. 1465 instruct vfmlsF2(vReg dst_src1, vReg src2, vReg src3) %{ 1466 predicate(UseFMA && UseSVE > 0 && 1467 !n->in(2)->in(2)->as_Vector()->is_predicated_vector()); 1468 match(Set dst_src1 (FmaVF dst_src1 (Binary src2 (NegVF src3)))); 1469 ins_cost(SVE_COST); 1470 format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %} 1471 ins_encode %{ 1472 __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ S, 1473 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1474 %} 1475 ins_pipe(pipe_slow); 1476 %} 1477 1478 // dst_src1 = dst_src1 + -src2 * src3 1479 // The NegVD must not be predicated. 1480 instruct vfmlsD1(vReg dst_src1, vReg src2, vReg src3) %{ 1481 predicate(UseFMA && UseSVE > 0 && 1482 !n->in(2)->in(1)->as_Vector()->is_predicated_vector()); 1483 match(Set dst_src1 (FmaVD dst_src1 (Binary (NegVD src2) src3))); 1484 ins_cost(SVE_COST); 1485 format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %} 1486 ins_encode %{ 1487 __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ D, 1488 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1489 %} 1490 ins_pipe(pipe_slow); 1491 %} 1492 1493 // dst_src1 = dst_src1 + src2 * -src3 1494 // The NegVD must not be predicated. 1495 instruct vfmlsD2(vReg dst_src1, vReg src2, vReg src3) %{ 1496 predicate(UseFMA && UseSVE > 0 && 1497 !n->in(2)->in(2)->as_Vector()->is_predicated_vector()); 1498 match(Set dst_src1 (FmaVD dst_src1 (Binary src2 (NegVD src3)))); 1499 ins_cost(SVE_COST); 1500 format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %} 1501 ins_encode %{ 1502 __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ D, 1503 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1504 %} 1505 ins_pipe(pipe_slow); 1506 %} 1507 1508 // vector fmsb - predicated 1509 1510 // dst_src1 = dst_src1 * -src2 + src3 1511 // The NegVF must not be predicated. 1512 instruct vfmsbF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{ 1513 predicate(UseFMA && UseSVE > 0 && 1514 !n->in(1)->in(2)->as_Vector()->is_predicated_vector()); 1515 match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary src3 pg))); 1516 ins_cost(SVE_COST); 1517 format %{ "sve_fmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) (S)" %} 1518 ins_encode %{ 1519 __ sve_fmsb(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg), 1520 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1521 %} 1522 ins_pipe(pipe_slow); 1523 %} 1524 1525 // dst_src1 = dst_src1 * -src2 + src3 1526 // The NegVD must not be predicated. 1527 instruct vfmsbD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{ 1528 predicate(UseFMA && UseSVE > 0 && 1529 !n->in(1)->in(2)->as_Vector()->is_predicated_vector()); 1530 match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary src3 pg))); 1531 ins_cost(SVE_COST); 1532 format %{ "sve_fmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) (D)" %} 1533 ins_encode %{ 1534 __ sve_fmsb(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg), 1535 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1536 %} 1537 ins_pipe(pipe_slow); 1538 %} 1539 1540 // vector fnmla 1541 1542 // dst_src1 = -dst_src1 + -src2 * src3 1543 // The NegVF must not be predicated. 1544 instruct vfnmlaF1(vReg dst_src1, vReg src2, vReg src3) %{ 1545 predicate(UseFMA && UseSVE > 0 && 1546 !n->in(1)->as_Vector()->is_predicated_vector() && 1547 !n->in(2)->in(1)->as_Vector()->is_predicated_vector()); 1548 match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary (NegVF src2) src3))); 1549 ins_cost(SVE_COST); 1550 format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %} 1551 ins_encode %{ 1552 __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ S, 1553 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1554 %} 1555 ins_pipe(pipe_slow); 1556 %} 1557 1558 // dst_src1 = -dst_src1 + src2 * -src3 1559 // The NegVF must not be predicated. 1560 instruct vfnmlaF2(vReg dst_src1, vReg src2, vReg src3) %{ 1561 predicate(UseFMA && UseSVE > 0 && 1562 !n->in(1)->as_Vector()->is_predicated_vector() && 1563 !n->in(2)->in(2)->as_Vector()->is_predicated_vector()); 1564 match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 (NegVF src3)))); 1565 ins_cost(SVE_COST); 1566 format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %} 1567 ins_encode %{ 1568 __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ S, 1569 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1570 %} 1571 ins_pipe(pipe_slow); 1572 %} 1573 1574 // dst_src1 = -dst_src1 + -src2 * src3 1575 // The NegVD must not be predicated. 1576 instruct vfnmlaD1(vReg dst_src1, vReg src2, vReg src3) %{ 1577 predicate(UseFMA && UseSVE > 0 && 1578 !n->in(1)->as_Vector()->is_predicated_vector() && 1579 !n->in(2)->in(1)->as_Vector()->is_predicated_vector()); 1580 match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary (NegVD src2) src3))); 1581 ins_cost(SVE_COST); 1582 format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %} 1583 ins_encode %{ 1584 __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ D, 1585 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1586 %} 1587 ins_pipe(pipe_slow); 1588 %} 1589 1590 // dst_src1 = -dst_src1 + src2 * -src3 1591 // The NegVD must not be predicated. 1592 instruct vfnmlaD2(vReg dst_src1, vReg src2, vReg src3) %{ 1593 predicate(UseFMA && UseSVE > 0 && 1594 !n->in(1)->as_Vector()->is_predicated_vector() && 1595 !n->in(2)->in(2)->as_Vector()->is_predicated_vector()); 1596 match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 (NegVD src3)))); 1597 ins_cost(SVE_COST); 1598 format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %} 1599 ins_encode %{ 1600 __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ D, 1601 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1602 %} 1603 ins_pipe(pipe_slow); 1604 %} 1605 1606 // vector fnmad - predicated 1607 1608 // dst_src1 = -src3 + dst_src1 * -src2 1609 // The NegVF must not be predicated. 1610 instruct vfnmadF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{ 1611 predicate(UseFMA && UseSVE > 0 && 1612 !n->in(1)->in(2)->as_Vector()->is_predicated_vector() && 1613 !n->in(2)->in(1)->as_Vector()->is_predicated_vector()); 1614 match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary (NegVF src3) pg))); 1615 ins_cost(SVE_COST); 1616 format %{ "sve_fnmad $dst_src1, $pg, $src2, $src3\t # vector (sve) (S)" %} 1617 ins_encode %{ 1618 __ sve_fnmad(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg), 1619 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1620 %} 1621 ins_pipe(pipe_slow); 1622 %} 1623 1624 // dst_src1 = -src3 + dst_src1 * -src2 1625 // The NegVD must not be predicated. 1626 instruct vfnmadD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{ 1627 predicate(UseFMA && UseSVE > 0 && 1628 !n->in(1)->in(2)->as_Vector()->is_predicated_vector() && 1629 !n->in(2)->in(1)->as_Vector()->is_predicated_vector()); 1630 match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary (NegVD src3) pg))); 1631 ins_cost(SVE_COST); 1632 format %{ "sve_fnmad $dst_src1, $pg, $src2, $src3\t # vector (sve) (D)" %} 1633 ins_encode %{ 1634 __ sve_fnmad(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg), 1635 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1636 %} 1637 ins_pipe(pipe_slow); 1638 %} 1639 1640 // vector fnmls 1641 1642 // dst_src1 = -dst_src1 + src2 * src3 1643 // The NegVF must not be predicated. 1644 instruct vfnmlsF(vReg dst_src1, vReg src2, vReg src3) %{ 1645 predicate(UseFMA && UseSVE > 0 && 1646 !n->in(1)->as_Vector()->is_predicated_vector()); 1647 match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 src3))); 1648 ins_cost(SVE_COST); 1649 format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %} 1650 ins_encode %{ 1651 __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ S, 1652 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1653 %} 1654 ins_pipe(pipe_slow); 1655 %} 1656 1657 // dst_src1 = -dst_src1 + src2 * src3 1658 // The NegVD must not be predicated. 1659 instruct vfnmlsD(vReg dst_src1, vReg src2, vReg src3) %{ 1660 predicate(UseFMA && UseSVE > 0 && 1661 !n->in(1)->as_Vector()->is_predicated_vector()); 1662 match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 src3))); 1663 ins_cost(SVE_COST); 1664 format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %} 1665 ins_encode %{ 1666 __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ D, 1667 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1668 %} 1669 ins_pipe(pipe_slow); 1670 %} 1671 1672 // vector fnmsb - predicated 1673 1674 // dst_src1 = -src3 + dst_src1 * src2 1675 // The NegVF must not be predicated. 1676 instruct vfnmsbF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{ 1677 predicate(UseFMA && UseSVE > 0 && 1678 !n->in(2)->in(1)->as_Vector()->is_predicated_vector()); 1679 match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary (NegVF src3) pg))); 1680 ins_cost(SVE_COST); 1681 format %{ "sve_fnmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) (S)" %} 1682 ins_encode %{ 1683 __ sve_fnmsb(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg), 1684 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1685 %} 1686 ins_pipe(pipe_slow); 1687 %} 1688 1689 // dst_src1 = -src3 + dst_src1 * src2 1690 // The NegVD must not be predicated. 1691 instruct vfnmsbD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{ 1692 predicate(UseFMA && UseSVE > 0 && 1693 !n->in(2)->in(1)->as_Vector()->is_predicated_vector()); 1694 match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary (NegVD src3) pg))); 1695 ins_cost(SVE_COST); 1696 format %{ "sve_fnmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) (D)" %} 1697 ins_encode %{ 1698 __ sve_fnmsb(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg), 1699 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1700 %} 1701 ins_pipe(pipe_slow); 1702 %} 1703 1704 // vector mla 1705 1706 // dst_src1 = dst_src1 + src2 * src3 1707 instruct vmlaB(vReg dst_src1, vReg src2, vReg src3) 1708 %{ 1709 predicate(UseSVE > 0); 1710 match(Set dst_src1 (AddVB dst_src1 (MulVB src2 src3))); 1711 ins_cost(SVE_COST); 1712 format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (B)" %} 1713 ins_encode %{ 1714 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ B, 1715 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1716 %} 1717 ins_pipe(pipe_slow); 1718 %} 1719 1720 // dst_src1 = dst_src1 + src2 * src3 1721 instruct vmlaS(vReg dst_src1, vReg src2, vReg src3) 1722 %{ 1723 predicate(UseSVE > 0); 1724 match(Set dst_src1 (AddVS dst_src1 (MulVS src2 src3))); 1725 ins_cost(SVE_COST); 1726 format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (H)" %} 1727 ins_encode %{ 1728 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ H, 1729 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1730 %} 1731 ins_pipe(pipe_slow); 1732 %} 1733 1734 // dst_src1 = dst_src1 + src2 * src3 1735 instruct vmlaI(vReg dst_src1, vReg src2, vReg src3) 1736 %{ 1737 predicate(UseSVE > 0); 1738 match(Set dst_src1 (AddVI dst_src1 (MulVI src2 src3))); 1739 ins_cost(SVE_COST); 1740 format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (S)" %} 1741 ins_encode %{ 1742 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ S, 1743 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1744 %} 1745 ins_pipe(pipe_slow); 1746 %} 1747 1748 // dst_src1 = dst_src1 + src2 * src3 1749 instruct vmlaL(vReg dst_src1, vReg src2, vReg src3) 1750 %{ 1751 predicate(UseSVE > 0); 1752 match(Set dst_src1 (AddVL dst_src1 (MulVL src2 src3))); 1753 ins_cost(SVE_COST); 1754 format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (D)" %} 1755 ins_encode %{ 1756 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ D, 1757 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1758 %} 1759 ins_pipe(pipe_slow); 1760 %} 1761 1762 // vector mla - predicated 1763 1764 // dst_src1 = dst_src1 + src2 * src3 1765 instruct vmlaB_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) 1766 %{ 1767 predicate(UseSVE > 0); 1768 match(Set dst_src1 (AddVB (Binary dst_src1 (MulVB src2 src3)) pg)); 1769 ins_cost(SVE_COST); 1770 format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) (B)" %} 1771 ins_encode %{ 1772 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ B, as_PRegister($pg$$reg), 1773 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1774 %} 1775 ins_pipe(pipe_slow); 1776 %} 1777 1778 // dst_src1 = dst_src1 + src2 * src3 1779 instruct vmlaS_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) 1780 %{ 1781 predicate(UseSVE > 0); 1782 match(Set dst_src1 (AddVS (Binary dst_src1 (MulVS src2 src3)) pg)); 1783 ins_cost(SVE_COST); 1784 format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) (H)" %} 1785 ins_encode %{ 1786 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ H, as_PRegister($pg$$reg), 1787 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1788 %} 1789 ins_pipe(pipe_slow); 1790 %} 1791 1792 // dst_src1 = dst_src1 + src2 * src3 1793 instruct vmlaI_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) 1794 %{ 1795 predicate(UseSVE > 0); 1796 match(Set dst_src1 (AddVI (Binary dst_src1 (MulVI src2 src3)) pg)); 1797 ins_cost(SVE_COST); 1798 format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) (S)" %} 1799 ins_encode %{ 1800 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg), 1801 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1802 %} 1803 ins_pipe(pipe_slow); 1804 %} 1805 1806 // dst_src1 = dst_src1 + src2 * src3 1807 instruct vmlaL_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) 1808 %{ 1809 predicate(UseSVE > 0); 1810 match(Set dst_src1 (AddVL (Binary dst_src1 (MulVL src2 src3)) pg)); 1811 ins_cost(SVE_COST); 1812 format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) (D)" %} 1813 ins_encode %{ 1814 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg), 1815 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1816 %} 1817 ins_pipe(pipe_slow); 1818 %} 1819 1820 // vector mls 1821 1822 // dst_src1 = dst_src1 - src2 * src3 1823 instruct vmlsB(vReg dst_src1, vReg src2, vReg src3) 1824 %{ 1825 predicate(UseSVE > 0); 1826 match(Set dst_src1 (SubVB dst_src1 (MulVB src2 src3))); 1827 ins_cost(SVE_COST); 1828 format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (B)" %} 1829 ins_encode %{ 1830 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ B, 1831 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1832 %} 1833 ins_pipe(pipe_slow); 1834 %} 1835 1836 // dst_src1 = dst_src1 - src2 * src3 1837 instruct vmlsS(vReg dst_src1, vReg src2, vReg src3) 1838 %{ 1839 predicate(UseSVE > 0); 1840 match(Set dst_src1 (SubVS dst_src1 (MulVS src2 src3))); 1841 ins_cost(SVE_COST); 1842 format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (H)" %} 1843 ins_encode %{ 1844 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ H, 1845 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1846 %} 1847 ins_pipe(pipe_slow); 1848 %} 1849 1850 // dst_src1 = dst_src1 - src2 * src3 1851 instruct vmlsI(vReg dst_src1, vReg src2, vReg src3) 1852 %{ 1853 predicate(UseSVE > 0); 1854 match(Set dst_src1 (SubVI dst_src1 (MulVI src2 src3))); 1855 ins_cost(SVE_COST); 1856 format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (S)" %} 1857 ins_encode %{ 1858 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ S, 1859 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1860 %} 1861 ins_pipe(pipe_slow); 1862 %} 1863 1864 // dst_src1 = dst_src1 - src2 * src3 1865 instruct vmlsL(vReg dst_src1, vReg src2, vReg src3) 1866 %{ 1867 predicate(UseSVE > 0); 1868 match(Set dst_src1 (SubVL dst_src1 (MulVL src2 src3))); 1869 ins_cost(SVE_COST); 1870 format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (D)" %} 1871 ins_encode %{ 1872 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ D, 1873 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1874 %} 1875 ins_pipe(pipe_slow); 1876 %} 1877 1878 // vector mls - predicated 1879 1880 // dst_src1 = dst_src1 - src2 * src3 1881 instruct vmlsB_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) 1882 %{ 1883 predicate(UseSVE > 0); 1884 match(Set dst_src1 (SubVB (Binary dst_src1 (MulVB src2 src3)) pg)); 1885 ins_cost(SVE_COST); 1886 format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) (B)" %} 1887 ins_encode %{ 1888 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ B, as_PRegister($pg$$reg), 1889 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1890 %} 1891 ins_pipe(pipe_slow); 1892 %} 1893 1894 // dst_src1 = dst_src1 - src2 * src3 1895 instruct vmlsS_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) 1896 %{ 1897 predicate(UseSVE > 0); 1898 match(Set dst_src1 (SubVS (Binary dst_src1 (MulVS src2 src3)) pg)); 1899 ins_cost(SVE_COST); 1900 format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) (H)" %} 1901 ins_encode %{ 1902 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ H, as_PRegister($pg$$reg), 1903 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1904 %} 1905 ins_pipe(pipe_slow); 1906 %} 1907 1908 // dst_src1 = dst_src1 - src2 * src3 1909 instruct vmlsI_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) 1910 %{ 1911 predicate(UseSVE > 0); 1912 match(Set dst_src1 (SubVI (Binary dst_src1 (MulVI src2 src3)) pg)); 1913 ins_cost(SVE_COST); 1914 format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) (S)" %} 1915 ins_encode %{ 1916 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg), 1917 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1918 %} 1919 ins_pipe(pipe_slow); 1920 %} 1921 1922 // dst_src1 = dst_src1 - src2 * src3 1923 instruct vmlsL_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) 1924 %{ 1925 predicate(UseSVE > 0); 1926 match(Set dst_src1 (SubVL (Binary dst_src1 (MulVL src2 src3)) pg)); 1927 ins_cost(SVE_COST); 1928 format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) (D)" %} 1929 ins_encode %{ 1930 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg), 1931 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg)); 1932 %} 1933 ins_pipe(pipe_slow); 1934 %} 1935 1936 // vector mul 1937 1938 instruct vmulB(vReg dst_src1, vReg src2) %{ 1939 predicate(UseSVE > 0); 1940 match(Set dst_src1 (MulVB dst_src1 src2)); 1941 ins_cost(SVE_COST); 1942 format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (B)" %} 1943 ins_encode %{ 1944 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ B, 1945 ptrue, as_FloatRegister($src2$$reg)); 1946 %} 1947 ins_pipe(pipe_slow); 1948 %} 1949 1950 instruct vmulS(vReg dst_src1, vReg src2) %{ 1951 predicate(UseSVE > 0); 1952 match(Set dst_src1 (MulVS dst_src1 src2)); 1953 ins_cost(SVE_COST); 1954 format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (H)" %} 1955 ins_encode %{ 1956 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ H, 1957 ptrue, as_FloatRegister($src2$$reg)); 1958 %} 1959 ins_pipe(pipe_slow); 1960 %} 1961 1962 instruct vmulI(vReg dst_src1, vReg src2) %{ 1963 predicate(UseSVE > 0); 1964 match(Set dst_src1 (MulVI dst_src1 src2)); 1965 ins_cost(SVE_COST); 1966 format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (S)" %} 1967 ins_encode %{ 1968 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ S, 1969 ptrue, as_FloatRegister($src2$$reg)); 1970 %} 1971 ins_pipe(pipe_slow); 1972 %} 1973 1974 instruct vmulL(vReg dst_src1, vReg src2) %{ 1975 predicate(UseSVE > 0); 1976 match(Set dst_src1 (MulVL dst_src1 src2)); 1977 ins_cost(SVE_COST); 1978 format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (D)" %} 1979 ins_encode %{ 1980 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ D, 1981 ptrue, as_FloatRegister($src2$$reg)); 1982 %} 1983 ins_pipe(pipe_slow); 1984 %} 1985 1986 instruct vmulF(vReg dst, vReg src1, vReg src2) %{ 1987 predicate(UseSVE > 0); 1988 match(Set dst (MulVF src1 src2)); 1989 ins_cost(SVE_COST); 1990 format %{ "sve_fmul $dst, $src1, $src2\t # vector (sve) (S)" %} 1991 ins_encode %{ 1992 __ sve_fmul(as_FloatRegister($dst$$reg), __ S, 1993 as_FloatRegister($src1$$reg), 1994 as_FloatRegister($src2$$reg)); 1995 %} 1996 ins_pipe(pipe_slow); 1997 %} 1998 1999 instruct vmulD(vReg dst, vReg src1, vReg src2) %{ 2000 predicate(UseSVE > 0); 2001 match(Set dst (MulVD src1 src2)); 2002 ins_cost(SVE_COST); 2003 format %{ "sve_fmul $dst, $src1, $src2\t # vector (sve) (D)" %} 2004 ins_encode %{ 2005 __ sve_fmul(as_FloatRegister($dst$$reg), __ D, 2006 as_FloatRegister($src1$$reg), 2007 as_FloatRegister($src2$$reg)); 2008 %} 2009 ins_pipe(pipe_slow); 2010 %} 2011 2012 // vector mul - predicated 2013 2014 instruct vmulB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 2015 predicate(UseSVE > 0); 2016 match(Set dst_src1 (MulVB (Binary dst_src1 src2) pg)); 2017 ins_cost(SVE_COST); 2018 format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %} 2019 ins_encode %{ 2020 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ B, 2021 as_PRegister($pg$$reg), 2022 as_FloatRegister($src2$$reg)); 2023 %} 2024 ins_pipe(pipe_slow); 2025 %} 2026 2027 instruct vmulS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 2028 predicate(UseSVE > 0); 2029 match(Set dst_src1 (MulVS (Binary dst_src1 src2) pg)); 2030 ins_cost(SVE_COST); 2031 format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %} 2032 ins_encode %{ 2033 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ H, 2034 as_PRegister($pg$$reg), 2035 as_FloatRegister($src2$$reg)); 2036 %} 2037 ins_pipe(pipe_slow); 2038 %} 2039 2040 instruct vmulI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 2041 predicate(UseSVE > 0); 2042 match(Set dst_src1 (MulVI (Binary dst_src1 src2) pg)); 2043 ins_cost(SVE_COST); 2044 format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %} 2045 ins_encode %{ 2046 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ S, 2047 as_PRegister($pg$$reg), 2048 as_FloatRegister($src2$$reg)); 2049 %} 2050 ins_pipe(pipe_slow); 2051 %} 2052 2053 instruct vmulL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 2054 predicate(UseSVE > 0); 2055 match(Set dst_src1 (MulVL (Binary dst_src1 src2) pg)); 2056 ins_cost(SVE_COST); 2057 format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %} 2058 ins_encode %{ 2059 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ D, 2060 as_PRegister($pg$$reg), 2061 as_FloatRegister($src2$$reg)); 2062 %} 2063 ins_pipe(pipe_slow); 2064 %} 2065 2066 instruct vmulF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 2067 predicate(UseSVE > 0); 2068 match(Set dst_src1 (MulVF (Binary dst_src1 src2) pg)); 2069 ins_cost(SVE_COST); 2070 format %{ "sve_fmul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %} 2071 ins_encode %{ 2072 __ sve_fmul(as_FloatRegister($dst_src1$$reg), __ S, 2073 as_PRegister($pg$$reg), 2074 as_FloatRegister($src2$$reg)); 2075 %} 2076 ins_pipe(pipe_slow); 2077 %} 2078 2079 instruct vmulD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 2080 predicate(UseSVE > 0); 2081 match(Set dst_src1 (MulVD (Binary dst_src1 src2) pg)); 2082 ins_cost(SVE_COST); 2083 format %{ "sve_fmul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %} 2084 ins_encode %{ 2085 __ sve_fmul(as_FloatRegister($dst_src1$$reg), __ D, 2086 as_PRegister($pg$$reg), 2087 as_FloatRegister($src2$$reg)); 2088 %} 2089 ins_pipe(pipe_slow); 2090 %} 2091 2092 // vector neg 2093 2094 instruct vnegI(vReg dst, vReg src) %{ 2095 predicate(UseSVE > 0 && 2096 !n->as_Vector()->is_predicated_vector()); 2097 match(Set dst (NegVI src)); 2098 ins_cost(SVE_COST); 2099 format %{ "sve_neg $dst, $src\t# vector (sve) (B/H/S)" %} 2100 ins_encode %{ 2101 BasicType bt = Matcher::vector_element_basic_type(this); 2102 __ sve_neg(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt), 2103 ptrue, as_FloatRegister($src$$reg)); 2104 %} 2105 ins_pipe(pipe_slow); 2106 %} 2107 2108 instruct vnegL(vReg dst, vReg src) %{ 2109 predicate(UseSVE > 0 && 2110 !n->as_Vector()->is_predicated_vector()); 2111 match(Set dst (NegVL src)); 2112 ins_cost(SVE_COST); 2113 format %{ "sve_neg $dst, $src\t# vector (sve) (D)" %} 2114 ins_encode %{ 2115 __ sve_neg(as_FloatRegister($dst$$reg), __ D, 2116 ptrue, as_FloatRegister($src$$reg)); 2117 %} 2118 ins_pipe(pipe_slow); 2119 %} 2120 2121 instruct vnegF(vReg dst, vReg src) %{ 2122 predicate(UseSVE > 0 && 2123 !n->as_Vector()->is_predicated_vector()); 2124 match(Set dst (NegVF src)); 2125 ins_cost(SVE_COST); 2126 format %{ "sve_fneg $dst, $src\t# vector (sve) (S)" %} 2127 ins_encode %{ 2128 __ sve_fneg(as_FloatRegister($dst$$reg), __ S, 2129 ptrue, as_FloatRegister($src$$reg)); 2130 %} 2131 ins_pipe(pipe_slow); 2132 %} 2133 2134 instruct vnegD(vReg dst, vReg src) %{ 2135 predicate(UseSVE > 0 && 2136 !n->as_Vector()->is_predicated_vector()); 2137 match(Set dst (NegVD src)); 2138 ins_cost(SVE_COST); 2139 format %{ "sve_fneg $dst, $src\t# vector (sve) (D)" %} 2140 ins_encode %{ 2141 __ sve_fneg(as_FloatRegister($dst$$reg), __ D, 2142 ptrue, as_FloatRegister($src$$reg)); 2143 %} 2144 ins_pipe(pipe_slow); 2145 %} 2146 2147 // vector neg - predicated 2148 2149 instruct vnegI_masked(vReg dst_src, pRegGov pg) %{ 2150 predicate(UseSVE > 0); 2151 match(Set dst_src (NegVI dst_src pg)); 2152 ins_cost(SVE_COST); 2153 format %{ "sve_neg $dst_src, $pg, $dst_src\t# vector (sve) (B/H/S)" %} 2154 ins_encode %{ 2155 BasicType bt = Matcher::vector_element_basic_type(this); 2156 __ sve_neg(as_FloatRegister($dst_src$$reg), __ elemType_to_regVariant(bt), 2157 as_PRegister($pg$$reg), 2158 as_FloatRegister($dst_src$$reg)); 2159 %} 2160 ins_pipe(pipe_slow); 2161 %} 2162 2163 instruct vnegL_masked(vReg dst_src, pRegGov pg) %{ 2164 predicate(UseSVE > 0); 2165 match(Set dst_src (NegVL dst_src pg)); 2166 ins_cost(SVE_COST); 2167 format %{ "sve_neg $dst_src, $pg, $dst_src\t# vector (sve) (D)" %} 2168 ins_encode %{ 2169 __ sve_neg(as_FloatRegister($dst_src$$reg), __ D, 2170 as_PRegister($pg$$reg), 2171 as_FloatRegister($dst_src$$reg)); 2172 %} 2173 ins_pipe(pipe_slow); 2174 %} 2175 2176 instruct vnegF_masked(vReg dst_src, pRegGov pg) %{ 2177 predicate(UseSVE > 0); 2178 match(Set dst_src (NegVF dst_src pg)); 2179 ins_cost(SVE_COST); 2180 format %{ "sve_fneg $dst_src, $pg, $dst_src\t# vector (sve) (S)" %} 2181 ins_encode %{ 2182 __ sve_fneg(as_FloatRegister($dst_src$$reg), __ S, 2183 as_PRegister($pg$$reg), 2184 as_FloatRegister($dst_src$$reg)); 2185 %} 2186 ins_pipe(pipe_slow); 2187 %} 2188 2189 instruct vnegD_masked(vReg dst_src, pRegGov pg) %{ 2190 predicate(UseSVE > 0); 2191 match(Set dst_src (NegVD dst_src pg)); 2192 ins_cost(SVE_COST); 2193 format %{ "sve_fneg $dst_src, $pg, $dst_src\t# vector (sve) (D)" %} 2194 ins_encode %{ 2195 __ sve_fneg(as_FloatRegister($dst_src$$reg), __ D, 2196 as_PRegister($pg$$reg), 2197 as_FloatRegister($dst_src$$reg)); 2198 %} 2199 ins_pipe(pipe_slow); 2200 %} 2201 2202 // vector popcount 2203 2204 instruct vpopcountI(vReg dst, vReg src) %{ 2205 predicate(UseSVE > 0 && 2206 !n->as_Vector()->is_predicated_vector()); 2207 match(Set dst (PopCountVI src)); 2208 ins_cost(SVE_COST); 2209 format %{ "sve_cnt $dst, $src\t# vector (sve) (B/H/S)" %} 2210 ins_encode %{ 2211 assert(UsePopCountInstruction, "unsupported"); 2212 BasicType bt = Matcher::vector_element_basic_type(this); 2213 __ sve_cnt(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt), 2214 ptrue, as_FloatRegister($src$$reg)); 2215 %} 2216 ins_pipe(pipe_slow); 2217 %} 2218 2219 instruct vpopcountL(vReg dst, vReg src) %{ 2220 predicate(UseSVE > 0 && 2221 !n->as_Vector()->is_predicated_vector() && 2222 n->bottom_type()->is_vect()->element_basic_type() == T_LONG); 2223 match(Set dst (PopCountVL src)); 2224 ins_cost(SVE_COST); 2225 format %{ "sve_cnt $dst, $src\t# vector (sve) (D)" %} 2226 ins_encode %{ 2227 assert(UsePopCountInstruction, "unsupported"); 2228 __ sve_cnt(as_FloatRegister($dst$$reg), __ D, 2229 ptrue, as_FloatRegister($src$$reg)); 2230 %} 2231 ins_pipe(pipe_slow); 2232 %} 2233 2234 // If the PopCountVL is generated by auto-vectorization, the dst basic 2235 // type is T_INT. And once we have unified the type definition for 2236 // Vector API and auto-vectorization, this rule can be merged with 2237 // "vpopcountL" rule. 2238 instruct vpopcountLI(vReg dst, vReg src, vReg vtmp) %{ 2239 predicate(UseSVE > 0 && 2240 !n->as_Vector()->is_predicated_vector() && 2241 n->bottom_type()->is_vect()->element_basic_type() == T_INT); 2242 match(Set dst (PopCountVL src)); 2243 effect(TEMP_DEF dst, TEMP vtmp); 2244 ins_cost(3 * SVE_COST); 2245 format %{ "sve_cnt $dst, $src\n\t" 2246 "sve_dup $vtmp, #0\n\t" 2247 "sve_uzp1 $dst, $dst, $vtmp\t# vector (sve) (S)" %} 2248 ins_encode %{ 2249 assert(UsePopCountInstruction, "unsupported"); 2250 __ sve_cnt(as_FloatRegister($dst$$reg), __ D, 2251 ptrue, as_FloatRegister($src$$reg)); 2252 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S, 2253 as_FloatRegister($dst$$reg), __ D, as_FloatRegister($vtmp$$reg)); 2254 %} 2255 ins_pipe(pipe_slow); 2256 %} 2257 2258 // vector popcount - predicated 2259 2260 instruct vpopcountI_masked(vReg dst_src, pRegGov pg) %{ 2261 predicate(UseSVE > 0); 2262 match(Set dst_src (PopCountVI dst_src pg)); 2263 ins_cost(SVE_COST); 2264 format %{ "sve_cnt $dst_src, $pg, $dst_src\t# vector (sve) (B/H/S)" %} 2265 ins_encode %{ 2266 assert(UsePopCountInstruction, "unsupported"); 2267 BasicType bt = Matcher::vector_element_basic_type(this); 2268 __ sve_cnt(as_FloatRegister($dst_src$$reg), __ elemType_to_regVariant(bt), 2269 as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg)); 2270 %} 2271 ins_pipe(pipe_slow); 2272 %} 2273 2274 instruct vpopcountL_masked(vReg dst_src, pRegGov pg) %{ 2275 predicate(UseSVE > 0 && 2276 n->bottom_type()->is_vect()->element_basic_type() == T_LONG); 2277 match(Set dst_src (PopCountVL dst_src pg)); 2278 ins_cost(SVE_COST); 2279 format %{ "sve_cnt $dst_src, $pg, $dst_src\t# vector (sve) (D)" %} 2280 ins_encode %{ 2281 assert(UsePopCountInstruction, "unsupported"); 2282 __ sve_cnt(as_FloatRegister($dst_src$$reg), __ D, 2283 as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg)); 2284 %} 2285 ins_pipe(pipe_slow); 2286 %} 2287 2288 // vector blend 2289 2290 instruct vblend(vReg dst, vReg src1, vReg src2, pRegGov pg) %{ 2291 predicate(UseSVE > 0); 2292 match(Set dst (VectorBlend (Binary src1 src2) pg)); 2293 ins_cost(SVE_COST); 2294 format %{ "sve_sel $dst, $pg, $src2, $src1\t# vector blend (sve)" %} 2295 ins_encode %{ 2296 Assembler::SIMD_RegVariant size = 2297 __ elemType_to_regVariant(Matcher::vector_element_basic_type(this)); 2298 __ sve_sel(as_FloatRegister($dst$$reg), size, as_PRegister($pg$$reg), 2299 as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg)); 2300 %} 2301 ins_pipe(pipe_slow); 2302 %} 2303 2304 // vector store mask 2305 2306 instruct vstoremaskB(vReg dst, pRegGov src, immI_1 size) %{ 2307 predicate(UseSVE > 0); 2308 match(Set dst (VectorStoreMask src size)); 2309 ins_cost(SVE_COST); 2310 format %{ "vstoremask $dst, $src\t# vector store mask (sve) (B)" %} 2311 ins_encode %{ 2312 __ sve_cpy(as_FloatRegister($dst$$reg), __ B, as_PRegister($src$$reg), 1, false); 2313 %} 2314 ins_pipe(pipe_slow); 2315 %} 2316 2317 instruct vstoremask_narrow(vReg dst, pRegGov src, vReg tmp, immI_gt_1 size) %{ 2318 predicate(UseSVE > 0); 2319 match(Set dst (VectorStoreMask src size)); 2320 effect(TEMP_DEF dst, TEMP tmp); 2321 ins_cost(3 * SVE_COST); 2322 format %{ "vstoremask $dst, $src\t# vector store mask (sve) (H/S/D)" %} 2323 ins_encode %{ 2324 Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant((int)$size$$constant); 2325 __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($src$$reg), 1, false); 2326 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B, 2327 as_FloatRegister($dst$$reg), size, as_FloatRegister($tmp$$reg)); 2328 %} 2329 ins_pipe(pipe_slow); 2330 %} 2331 2332 // Combine LoadVector+VectorLoadMask when the vector element type is not T_BYTE 2333 2334 instruct vloadmask_loadV(pRegGov dst, indirect mem, vReg tmp, rFlagsReg cr) %{ 2335 predicate(UseSVE > 0 && 2336 n->as_Vector()->length_in_bytes() == MaxVectorSize && 2337 type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1); 2338 match(Set dst (VectorLoadMask (LoadVector mem))); 2339 effect(TEMP tmp, KILL cr); 2340 ins_cost(3 * SVE_COST); 2341 format %{ "sve_ld1b $tmp, $mem\n\t" 2342 "sve_cmpne $dst, $tmp, 0\t# load vector mask (sve) (H/S/D)" %} 2343 ins_encode %{ 2344 // Load mask values which are boolean type, and extend them to the 2345 // expected vector element type. Convert the vector to predicate. 2346 BasicType to_vect_bt = Matcher::vector_element_basic_type(this); 2347 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($tmp$$reg), 2348 ptrue, T_BOOLEAN, to_vect_bt, $mem->opcode(), 2349 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); 2350 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ elemType_to_regVariant(to_vect_bt), 2351 ptrue, as_FloatRegister($tmp$$reg), 0); 2352 %} 2353 ins_pipe(pipe_slow); 2354 %} 2355 2356 instruct vloadmask_loadV_partial(pRegGov dst, indirect mem, vReg vtmp, pRegGov ptmp, rFlagsReg cr) %{ 2357 predicate(UseSVE > 0 && 2358 n->as_Vector()->length_in_bytes() > 16 && 2359 n->as_Vector()->length_in_bytes() < MaxVectorSize && 2360 type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1); 2361 match(Set dst (VectorLoadMask (LoadVector mem))); 2362 effect(TEMP vtmp, TEMP ptmp, KILL cr); 2363 ins_cost(6 * SVE_COST); 2364 format %{ "vloadmask_loadV $dst, $mem\t# load vector mask partial (sve) (H/S/D)" %} 2365 ins_encode %{ 2366 // Load valid mask values which are boolean type, and extend them to the 2367 // expected vector element type. Convert the vector to predicate. 2368 BasicType to_vect_bt = Matcher::vector_element_basic_type(this); 2369 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(to_vect_bt); 2370 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this)); 2371 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($vtmp$$reg), 2372 as_PRegister($ptmp$$reg), T_BOOLEAN, to_vect_bt, $mem->opcode(), 2373 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); 2374 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($vtmp$$reg), 0); 2375 %} 2376 ins_pipe(pipe_slow); 2377 %} 2378 2379 // Combine VectorStoreMask+StoreVector when the vector element type is not T_BYTE 2380 2381 instruct storeV_vstoremask(indirect mem, pRegGov src, vReg tmp, immI_gt_1 esize) %{ 2382 predicate(UseSVE > 0 && 2383 Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize); 2384 match(Set mem (StoreVector mem (VectorStoreMask src esize))); 2385 effect(TEMP tmp); 2386 ins_cost(3 * SVE_COST); 2387 format %{ "sve_cpy $tmp, $src, 1\n\t" 2388 "sve_st1b $tmp, $mem\t# store vector mask (sve) (H/S/D)" %} 2389 ins_encode %{ 2390 BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src); 2391 assert(type2aelembytes(from_vect_bt) == (int)$esize$$constant, "unsupported type."); 2392 Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant); 2393 __ sve_cpy(as_FloatRegister($tmp$$reg), size, as_PRegister($src$$reg), 1, false); 2394 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($tmp$$reg), 2395 ptrue, T_BOOLEAN, from_vect_bt, $mem->opcode(), 2396 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); 2397 %} 2398 ins_pipe(pipe_slow); 2399 %} 2400 2401 instruct storeV_vstoremask_partial(indirect mem, pRegGov src, vReg vtmp, 2402 immI_gt_1 esize, pRegGov ptmp, rFlagsReg cr) %{ 2403 predicate(UseSVE > 0 && 2404 n->as_StoreVector()->memory_size() > 16 && 2405 type2aelembytes(n->as_StoreVector()->vect_type()->element_basic_type()) > 1 && 2406 Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize); 2407 match(Set mem (StoreVector mem (VectorStoreMask src esize))); 2408 effect(TEMP vtmp, TEMP ptmp, KILL cr); 2409 format %{ "storeV_vstoremask $src, $mem\t# store vector mask partial (sve) (H/S/D)" %} 2410 ins_cost(6 * SVE_COST); 2411 ins_encode %{ 2412 // Convert the valid src predicate to vector, and store the vector 2413 // elements as boolean values. 2414 BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src); 2415 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(from_vect_bt); 2416 __ sve_cpy(as_FloatRegister($vtmp$$reg), size, as_PRegister($src$$reg), 1, false); 2417 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src)); 2418 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($vtmp$$reg), 2419 as_PRegister($ptmp$$reg), T_BOOLEAN, from_vect_bt, $mem->opcode(), 2420 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); 2421 %} 2422 ins_pipe(pipe_slow); 2423 %} 2424 2425 // vector add reduction 2426 2427 instruct reduce_addI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{ 2428 predicate(UseSVE > 0 && 2429 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize); 2430 match(Set dst (AddReductionVI src1 src2)); 2431 effect(TEMP_DEF dst, TEMP tmp); 2432 ins_cost(SVE_COST); 2433 format %{ "sve_reduce_addI $dst, $src1, $src2\t# addI reduction (sve) (may extend)" %} 2434 ins_encode %{ 2435 BasicType bt = Matcher::vector_element_basic_type(this, $src2); 2436 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt, 2437 $src1$$Register, as_FloatRegister($src2$$reg), 2438 ptrue, as_FloatRegister($tmp$$reg)); 2439 %} 2440 ins_pipe(pipe_slow); 2441 %} 2442 2443 instruct reduce_addL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{ 2444 predicate(UseSVE > 0 && 2445 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize); 2446 match(Set dst (AddReductionVL src1 src2)); 2447 effect(TEMP_DEF dst, TEMP tmp); 2448 ins_cost(SVE_COST); 2449 format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction (sve)" %} 2450 ins_encode %{ 2451 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG, 2452 $src1$$Register, as_FloatRegister($src2$$reg), 2453 ptrue, as_FloatRegister($tmp$$reg)); 2454 %} 2455 ins_pipe(pipe_slow); 2456 %} 2457 2458 instruct reduce_addF(vRegF src1_dst, vReg src2) %{ 2459 predicate(UseSVE > 0 && 2460 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize); 2461 match(Set src1_dst (AddReductionVF src1_dst src2)); 2462 ins_cost(SVE_COST); 2463 format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (S)" %} 2464 ins_encode %{ 2465 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S, 2466 ptrue, as_FloatRegister($src2$$reg)); 2467 %} 2468 ins_pipe(pipe_slow); 2469 %} 2470 2471 instruct reduce_addD(vRegD src1_dst, vReg src2) %{ 2472 predicate(UseSVE > 0 && 2473 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize); 2474 match(Set src1_dst (AddReductionVD src1_dst src2)); 2475 ins_cost(SVE_COST); 2476 format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (D)" %} 2477 ins_encode %{ 2478 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D, 2479 ptrue, as_FloatRegister($src2$$reg)); 2480 %} 2481 ins_pipe(pipe_slow); 2482 %} 2483 2484 instruct reduce_addI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp, 2485 pRegGov ptmp, rFlagsReg cr) %{ 2486 predicate(UseSVE > 0 && 2487 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize); 2488 match(Set dst (AddReductionVI src1 src2)); 2489 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr); 2490 ins_cost(2 * SVE_COST); 2491 format %{ "sve_reduce_addI $dst, $src1, $src2\t# addI reduction partial (sve) (may extend)" %} 2492 ins_encode %{ 2493 BasicType bt = Matcher::vector_element_basic_type(this, $src2); 2494 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt); 2495 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2)); 2496 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt, 2497 $src1$$Register, as_FloatRegister($src2$$reg), 2498 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg)); 2499 %} 2500 ins_pipe(pipe_slow); 2501 %} 2502 2503 instruct reduce_addL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp, 2504 pRegGov ptmp, rFlagsReg cr) %{ 2505 predicate(UseSVE > 0 && 2506 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize); 2507 match(Set dst (AddReductionVL src1 src2)); 2508 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr); 2509 ins_cost(2 * SVE_COST); 2510 format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction partial (sve)" %} 2511 ins_encode %{ 2512 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2)); 2513 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG, 2514 $src1$$Register, as_FloatRegister($src2$$reg), 2515 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg)); 2516 %} 2517 ins_pipe(pipe_slow); 2518 %} 2519 2520 instruct reduce_addF_partial(vRegF src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{ 2521 predicate(UseSVE > 0 && 2522 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize); 2523 match(Set src1_dst (AddReductionVF src1_dst src2)); 2524 ins_cost(SVE_COST); 2525 effect(TEMP ptmp, KILL cr); 2526 format %{ "sve_reduce_addF $src1_dst, $src1_dst, $src2\t# addF reduction partial (sve) (S)" %} 2527 ins_encode %{ 2528 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src2)); 2529 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S, 2530 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg)); 2531 %} 2532 ins_pipe(pipe_slow); 2533 %} 2534 2535 instruct reduce_addD_partial(vRegD src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{ 2536 predicate(UseSVE > 0 && 2537 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize); 2538 match(Set src1_dst (AddReductionVD src1_dst src2)); 2539 ins_cost(SVE_COST); 2540 effect(TEMP ptmp, KILL cr); 2541 format %{ "sve_reduce_addD $src1_dst, $src1_dst, $src2\t# addD reduction partial (sve) (D)" %} 2542 ins_encode %{ 2543 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2)); 2544 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D, 2545 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg)); 2546 %} 2547 ins_pipe(pipe_slow); 2548 %} 2549 2550 // vector add reduction - predicated 2551 2552 instruct reduce_addI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{ 2553 predicate(UseSVE > 0); 2554 match(Set dst (AddReductionVI (Binary src1 src2) pg)); 2555 effect(TEMP_DEF dst, TEMP tmp); 2556 ins_cost(SVE_COST); 2557 format %{ "sve_reduce_addI $dst, $src1, $pg, $src2\t# addI reduction predicated (sve) (may extend)" %} 2558 ins_encode %{ 2559 BasicType bt = Matcher::vector_element_basic_type(this, $src2); 2560 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt, 2561 $src1$$Register, as_FloatRegister($src2$$reg), 2562 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg)); 2563 %} 2564 ins_pipe(pipe_slow); 2565 %} 2566 2567 instruct reduce_addL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{ 2568 predicate(UseSVE > 0); 2569 match(Set dst (AddReductionVL (Binary src1 src2) pg)); 2570 effect(TEMP_DEF dst, TEMP tmp); 2571 ins_cost(SVE_COST); 2572 format %{ "sve_reduce_addL $dst, $src1, $pg, $src2\t# addL reduction predicated (sve)" %} 2573 ins_encode %{ 2574 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG, 2575 $src1$$Register, as_FloatRegister($src2$$reg), 2576 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg)); 2577 %} 2578 ins_pipe(pipe_slow); 2579 %} 2580 2581 instruct reduce_addF_masked(vRegF src1_dst, vReg src2, pRegGov pg) %{ 2582 predicate(UseSVE > 0); 2583 match(Set src1_dst (AddReductionVF (Binary src1_dst src2) pg)); 2584 ins_cost(SVE_COST); 2585 format %{ "sve_reduce_addF $src1_dst, $pg, $src2\t# addF reduction predicated (sve)" %} 2586 ins_encode %{ 2587 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S, 2588 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg)); 2589 %} 2590 ins_pipe(pipe_slow); 2591 %} 2592 2593 instruct reduce_addD_masked(vRegD src1_dst, vReg src2, pRegGov pg) %{ 2594 predicate(UseSVE > 0); 2595 match(Set src1_dst (AddReductionVD (Binary src1_dst src2) pg)); 2596 ins_cost(SVE_COST); 2597 format %{ "sve_reduce_addD $src1_dst, $pg, $src2\t# addD reduction predicated (sve)" %} 2598 ins_encode %{ 2599 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D, 2600 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg)); 2601 %} 2602 ins_pipe(pipe_slow); 2603 %} 2604 2605 // vector and reduction 2606 2607 instruct reduce_andI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{ 2608 predicate(UseSVE > 0 && 2609 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG && 2610 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize); 2611 match(Set dst (AndReductionV src1 src2)); 2612 effect(TEMP_DEF dst, TEMP tmp); 2613 ins_cost(SVE_COST); 2614 format %{ "sve_reduce_andI $dst, $src1, $src2\t# andI reduction (sve) (may extend)" %} 2615 ins_encode %{ 2616 BasicType bt = Matcher::vector_element_basic_type(this, $src2); 2617 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt, 2618 $src1$$Register, as_FloatRegister($src2$$reg), 2619 ptrue, as_FloatRegister($tmp$$reg)); 2620 %} 2621 ins_pipe(pipe_slow); 2622 %} 2623 2624 instruct reduce_andL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{ 2625 predicate(UseSVE > 0 && 2626 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG && 2627 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize); 2628 match(Set dst (AndReductionV src1 src2)); 2629 effect(TEMP_DEF dst, TEMP tmp); 2630 ins_cost(SVE_COST); 2631 format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction (sve)" %} 2632 ins_encode %{ 2633 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG, 2634 $src1$$Register, as_FloatRegister($src2$$reg), 2635 ptrue, as_FloatRegister($tmp$$reg)); 2636 %} 2637 ins_pipe(pipe_slow); 2638 %} 2639 2640 instruct reduce_andI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp, 2641 pRegGov ptmp, rFlagsReg cr) %{ 2642 predicate(UseSVE > 0 && 2643 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG && 2644 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize); 2645 match(Set dst (AndReductionV src1 src2)); 2646 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr); 2647 ins_cost(2 * SVE_COST); 2648 format %{ "sve_reduce_andI $dst, $src1, $src2\t# andI reduction partial (sve) (may extend)" %} 2649 ins_encode %{ 2650 BasicType bt = Matcher::vector_element_basic_type(this, $src2); 2651 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt); 2652 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2)); 2653 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt, 2654 $src1$$Register, as_FloatRegister($src2$$reg), 2655 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg)); 2656 %} 2657 ins_pipe(pipe_slow); 2658 %} 2659 2660 instruct reduce_andL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp, 2661 pRegGov ptmp, rFlagsReg cr) %{ 2662 predicate(UseSVE > 0 && 2663 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG && 2664 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize); 2665 match(Set dst (AndReductionV src1 src2)); 2666 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr); 2667 ins_cost(2 * SVE_COST); 2668 format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction partial (sve)" %} 2669 ins_encode %{ 2670 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2)); 2671 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG, 2672 $src1$$Register, as_FloatRegister($src2$$reg), 2673 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg)); 2674 %} 2675 ins_pipe(pipe_slow); 2676 %} 2677 2678 // vector and reduction - predicated 2679 2680 instruct reduce_andI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{ 2681 predicate(UseSVE > 0 && 2682 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG); 2683 match(Set dst (AndReductionV (Binary src1 src2) pg)); 2684 effect(TEMP_DEF dst, TEMP tmp); 2685 ins_cost(SVE_COST); 2686 format %{ "sve_reduce_andI $dst, $src1, $pg, $src2\t# andI reduction predicated (sve) (may extend)" %} 2687 ins_encode %{ 2688 BasicType bt = Matcher::vector_element_basic_type(this, $src2); 2689 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt, 2690 $src1$$Register, as_FloatRegister($src2$$reg), 2691 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg)); 2692 %} 2693 ins_pipe(pipe_slow); 2694 %} 2695 2696 instruct reduce_andL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{ 2697 predicate(UseSVE > 0 && 2698 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG); 2699 match(Set dst (AndReductionV (Binary src1 src2) pg)); 2700 effect(TEMP_DEF dst, TEMP tmp); 2701 ins_cost(SVE_COST); 2702 format %{ "sve_reduce_andL $dst, $src1, $pg, $src2\t# andL reduction predicated (sve)" %} 2703 ins_encode %{ 2704 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG, 2705 $src1$$Register, as_FloatRegister($src2$$reg), 2706 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg)); 2707 %} 2708 ins_pipe(pipe_slow); 2709 %} 2710 2711 // vector or reduction 2712 2713 instruct reduce_orI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{ 2714 predicate(UseSVE > 0 && 2715 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG && 2716 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize); 2717 match(Set dst (OrReductionV src1 src2)); 2718 effect(TEMP_DEF dst, TEMP tmp); 2719 ins_cost(SVE_COST); 2720 format %{ "sve_reduce_orI $dst, $src1, $src2\t# orI reduction (sve) (may extend)" %} 2721 ins_encode %{ 2722 BasicType bt = Matcher::vector_element_basic_type(this, $src2); 2723 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt, 2724 $src1$$Register, as_FloatRegister($src2$$reg), 2725 ptrue, as_FloatRegister($tmp$$reg)); 2726 %} 2727 ins_pipe(pipe_slow); 2728 %} 2729 2730 instruct reduce_orL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{ 2731 predicate(UseSVE > 0 && 2732 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG && 2733 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize); 2734 match(Set dst (OrReductionV src1 src2)); 2735 effect(TEMP_DEF dst, TEMP tmp); 2736 ins_cost(SVE_COST); 2737 format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction (sve)" %} 2738 ins_encode %{ 2739 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG, 2740 $src1$$Register, as_FloatRegister($src2$$reg), 2741 ptrue, as_FloatRegister($tmp$$reg)); 2742 %} 2743 ins_pipe(pipe_slow); 2744 %} 2745 2746 instruct reduce_orI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp, 2747 pRegGov ptmp, rFlagsReg cr) %{ 2748 predicate(UseSVE > 0 && 2749 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG && 2750 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize); 2751 match(Set dst (OrReductionV src1 src2)); 2752 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr); 2753 ins_cost(2 * SVE_COST); 2754 format %{ "sve_reduce_orI $dst, $src1, $src2\t# orI reduction partial (sve) (may extend)" %} 2755 ins_encode %{ 2756 BasicType bt = Matcher::vector_element_basic_type(this, $src2); 2757 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt); 2758 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2)); 2759 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt, 2760 $src1$$Register, as_FloatRegister($src2$$reg), 2761 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg)); 2762 %} 2763 ins_pipe(pipe_slow); 2764 %} 2765 2766 instruct reduce_orL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp, 2767 pRegGov ptmp, rFlagsReg cr) %{ 2768 predicate(UseSVE > 0 && 2769 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG && 2770 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize); 2771 match(Set dst (OrReductionV src1 src2)); 2772 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr); 2773 ins_cost(2 * SVE_COST); 2774 format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction partial (sve)" %} 2775 ins_encode %{ 2776 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2)); 2777 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG, 2778 $src1$$Register, as_FloatRegister($src2$$reg), 2779 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg)); 2780 %} 2781 ins_pipe(pipe_slow); 2782 %} 2783 2784 // vector or reduction - predicated 2785 2786 instruct reduce_orI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{ 2787 predicate(UseSVE > 0 && 2788 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG); 2789 match(Set dst (OrReductionV (Binary src1 src2) pg)); 2790 effect(TEMP_DEF dst, TEMP tmp); 2791 ins_cost(SVE_COST); 2792 format %{ "sve_reduce_orI $dst, $src1, $pg, $src2\t# orI reduction predicated (sve) (may extend)" %} 2793 ins_encode %{ 2794 BasicType bt = Matcher::vector_element_basic_type(this, $src2); 2795 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt, 2796 $src1$$Register, as_FloatRegister($src2$$reg), 2797 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg)); 2798 %} 2799 ins_pipe(pipe_slow); 2800 %} 2801 2802 instruct reduce_orL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{ 2803 predicate(UseSVE > 0 && 2804 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG); 2805 match(Set dst (OrReductionV (Binary src1 src2) pg)); 2806 effect(TEMP_DEF dst, TEMP tmp); 2807 ins_cost(SVE_COST); 2808 format %{ "sve_reduce_orL $dst, $src1, $pg, $src2\t# orL reduction predicated (sve)" %} 2809 ins_encode %{ 2810 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG, 2811 $src1$$Register, as_FloatRegister($src2$$reg), 2812 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg)); 2813 %} 2814 ins_pipe(pipe_slow); 2815 %} 2816 2817 // vector xor reduction 2818 2819 instruct reduce_eorI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{ 2820 predicate(UseSVE > 0 && 2821 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG && 2822 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize); 2823 match(Set dst (XorReductionV src1 src2)); 2824 effect(TEMP_DEF dst, TEMP tmp); 2825 ins_cost(SVE_COST); 2826 format %{ "sve_reduce_eorI $dst, $src1, $src2\t# eorI reduction (sve) (may extend)" %} 2827 ins_encode %{ 2828 BasicType bt = Matcher::vector_element_basic_type(this, $src2); 2829 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt, 2830 $src1$$Register, as_FloatRegister($src2$$reg), 2831 ptrue, as_FloatRegister($tmp$$reg)); 2832 %} 2833 ins_pipe(pipe_slow); 2834 %} 2835 2836 instruct reduce_eorL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{ 2837 predicate(UseSVE > 0 && 2838 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG && 2839 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize); 2840 match(Set dst (XorReductionV src1 src2)); 2841 effect(TEMP_DEF dst, TEMP tmp); 2842 ins_cost(SVE_COST); 2843 format %{ "sve_reduce_eorL $dst, $src1, $src2\t# eorL reduction (sve)" %} 2844 ins_encode %{ 2845 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG, 2846 $src1$$Register, as_FloatRegister($src2$$reg), 2847 ptrue, as_FloatRegister($tmp$$reg)); 2848 %} 2849 ins_pipe(pipe_slow); 2850 %} 2851 2852 instruct reduce_eorI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp, 2853 pRegGov ptmp, rFlagsReg cr) %{ 2854 predicate(UseSVE > 0 && 2855 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG && 2856 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize); 2857 match(Set dst (XorReductionV src1 src2)); 2858 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr); 2859 ins_cost(2 * SVE_COST); 2860 format %{ "sve_reduce_eorI $dst, $src1, $src2\t# eorI reduction partial (sve) (may extend)" %} 2861 ins_encode %{ 2862 BasicType bt = Matcher::vector_element_basic_type(this, $src2); 2863 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt); 2864 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2)); 2865 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt, 2866 $src1$$Register, as_FloatRegister($src2$$reg), 2867 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg)); 2868 %} 2869 ins_pipe(pipe_slow); 2870 %} 2871 2872 instruct reduce_eorL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp, 2873 pRegGov ptmp, rFlagsReg cr) %{ 2874 predicate(UseSVE > 0 && 2875 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG && 2876 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize); 2877 match(Set dst (XorReductionV src1 src2)); 2878 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr); 2879 ins_cost(2 * SVE_COST); 2880 format %{ "sve_reduce_eorL $dst, $src1, $src2\t# eorL reduction partial (sve)" %} 2881 ins_encode %{ 2882 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2)); 2883 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG, 2884 $src1$$Register, as_FloatRegister($src2$$reg), 2885 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg)); 2886 %} 2887 ins_pipe(pipe_slow); 2888 %} 2889 2890 // vector xor reduction - predicated 2891 2892 instruct reduce_eorI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{ 2893 predicate(UseSVE > 0 && 2894 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG); 2895 match(Set dst (XorReductionV (Binary src1 src2) pg)); 2896 effect(TEMP_DEF dst, TEMP tmp); 2897 ins_cost(SVE_COST); 2898 format %{ "sve_reduce_eorI $dst, $src1, $pg, $src2\t# eorI reduction predicated (sve) (may extend)" %} 2899 ins_encode %{ 2900 BasicType bt = Matcher::vector_element_basic_type(this, $src2); 2901 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt, 2902 $src1$$Register, as_FloatRegister($src2$$reg), 2903 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg)); 2904 %} 2905 ins_pipe(pipe_slow); 2906 %} 2907 2908 instruct reduce_eorL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{ 2909 predicate(UseSVE > 0 && 2910 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG); 2911 match(Set dst (XorReductionV (Binary src1 src2) pg)); 2912 effect(TEMP_DEF dst, TEMP tmp); 2913 ins_cost(SVE_COST); 2914 format %{ "sve_reduce_eorL $dst, $src1, $pg, $src2\t# eorL reduction predicated (sve)" %} 2915 ins_encode %{ 2916 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG, 2917 $src1$$Register, as_FloatRegister($src2$$reg), 2918 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg)); 2919 %} 2920 ins_pipe(pipe_slow); 2921 %} 2922 2923 // vector max reduction 2924 2925 instruct reduce_maxI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{ 2926 predicate(UseSVE > 0 && 2927 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize && 2928 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG && 2929 is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type())); 2930 match(Set dst (MaxReductionV src1 src2)); 2931 effect(TEMP_DEF dst, TEMP tmp, KILL cr); 2932 ins_cost(SVE_COST); 2933 format %{ "sve_reduce_maxI $dst, $src1, $src2\t# maxI reduction (sve)" %} 2934 ins_encode %{ 2935 BasicType bt = Matcher::vector_element_basic_type(this, $src2); 2936 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt, 2937 $src1$$Register, as_FloatRegister($src2$$reg), 2938 ptrue, as_FloatRegister($tmp$$reg)); 2939 %} 2940 ins_pipe(pipe_slow); 2941 %} 2942 2943 instruct reduce_maxL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{ 2944 predicate(UseSVE > 0 && 2945 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize && 2946 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG); 2947 match(Set dst (MaxReductionV src1 src2)); 2948 effect(TEMP_DEF dst, TEMP tmp, KILL cr); 2949 ins_cost(SVE_COST); 2950 format %{ "sve_reduce_maxL $dst, $src1, $src2\t# maxL reduction (sve)" %} 2951 ins_encode %{ 2952 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG, 2953 $src1$$Register, as_FloatRegister($src2$$reg), 2954 ptrue, as_FloatRegister($tmp$$reg)); 2955 %} 2956 ins_pipe(pipe_slow); 2957 %} 2958 2959 instruct reduce_maxI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp, 2960 pRegGov ptmp, rFlagsReg cr) %{ 2961 predicate(UseSVE > 0 && 2962 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize && 2963 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG && 2964 is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type())); 2965 match(Set dst (MaxReductionV src1 src2)); 2966 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr); 2967 ins_cost(2 * SVE_COST); 2968 format %{ "sve_reduce_maxI $dst, $src1, $src2\t# maxI reduction partial (sve)" %} 2969 ins_encode %{ 2970 BasicType bt = Matcher::vector_element_basic_type(this, $src2); 2971 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt); 2972 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2)); 2973 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt, 2974 $src1$$Register, as_FloatRegister($src2$$reg), 2975 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg)); 2976 %} 2977 ins_pipe(pipe_slow); 2978 %} 2979 2980 instruct reduce_maxL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp, 2981 pRegGov ptmp, rFlagsReg cr) %{ 2982 predicate(UseSVE > 0 && 2983 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize && 2984 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG); 2985 match(Set dst (MaxReductionV src1 src2)); 2986 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr); 2987 ins_cost(2 * SVE_COST); 2988 format %{ "sve_reduce_maxL $dst, $src1, $src2\t# maxL reduction partial (sve)" %} 2989 ins_encode %{ 2990 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2)); 2991 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG, 2992 $src1$$Register, as_FloatRegister($src2$$reg), 2993 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg)); 2994 %} 2995 ins_pipe(pipe_slow); 2996 %} 2997 2998 instruct reduce_maxF(vRegF dst, vRegF src1, vReg src2) %{ 2999 predicate(UseSVE > 0 && 3000 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT && 3001 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize); 3002 match(Set dst (MaxReductionV src1 src2)); 3003 ins_cost(INSN_COST); 3004 effect(TEMP_DEF dst); 3005 format %{ "sve_reduce_maxF $dst, $src1, $src2\t# maxF reduction (sve)" %} 3006 ins_encode %{ 3007 __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src2$$reg)); 3008 __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg)); 3009 %} 3010 ins_pipe(pipe_slow); 3011 %} 3012 3013 instruct reduce_maxF_partial(vRegF dst, vRegF src1, vReg src2, 3014 pRegGov ptmp, rFlagsReg cr) %{ 3015 predicate(UseSVE > 0 && 3016 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT && 3017 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize); 3018 match(Set dst (MaxReductionV src1 src2)); 3019 ins_cost(INSN_COST); 3020 effect(TEMP_DEF dst, TEMP ptmp, KILL cr); 3021 format %{ "sve_reduce_maxF $dst, $src1, $src2\t# maxF reduction partial (sve)" %} 3022 ins_encode %{ 3023 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src2)); 3024 __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg)); 3025 __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg)); 3026 %} 3027 ins_pipe(pipe_slow); 3028 %} 3029 3030 instruct reduce_maxD(vRegD dst, vRegD src1, vReg src2) %{ 3031 predicate(UseSVE > 0 && 3032 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE && 3033 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize); 3034 match(Set dst (MaxReductionV src1 src2)); 3035 ins_cost(INSN_COST); 3036 effect(TEMP_DEF dst); 3037 format %{ "sve_reduce_maxD $dst, $src1, $src2\t# maxD reduction (sve)" %} 3038 ins_encode %{ 3039 __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src2$$reg)); 3040 __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg)); 3041 %} 3042 ins_pipe(pipe_slow); 3043 %} 3044 3045 instruct reduce_maxD_partial(vRegD dst, vRegD src1, vReg src2, 3046 pRegGov ptmp, rFlagsReg cr) %{ 3047 predicate(UseSVE > 0 && 3048 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE && 3049 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize); 3050 match(Set dst (MaxReductionV src1 src2)); 3051 ins_cost(INSN_COST); 3052 effect(TEMP_DEF dst, TEMP ptmp, KILL cr); 3053 format %{ "sve_reduce_maxD $dst, $src1, $src2\t# maxD reduction partial (sve)" %} 3054 ins_encode %{ 3055 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2)); 3056 __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg)); 3057 __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg)); 3058 %} 3059 ins_pipe(pipe_slow); 3060 %} 3061 3062 // vector max reduction - predicated 3063 3064 instruct reduce_maxI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, 3065 pRegGov pg, rFlagsReg cr) %{ 3066 predicate(UseSVE > 0 && 3067 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG && 3068 is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type())); 3069 match(Set dst (MaxReductionV (Binary src1 src2) pg)); 3070 effect(TEMP_DEF dst, TEMP tmp, KILL cr); 3071 ins_cost(SVE_COST); 3072 format %{ "sve_reduce_maxI $dst, $src1, $pg, $src2\t# maxI reduction predicated (sve)" %} 3073 ins_encode %{ 3074 BasicType bt = Matcher::vector_element_basic_type(this, $src2); 3075 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt, 3076 $src1$$Register, as_FloatRegister($src2$$reg), 3077 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg)); 3078 %} 3079 ins_pipe(pipe_slow); 3080 %} 3081 3082 instruct reduce_maxL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, 3083 pRegGov pg, rFlagsReg cr) %{ 3084 predicate(UseSVE > 0 && 3085 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG); 3086 match(Set dst (MaxReductionV (Binary src1 src2) pg)); 3087 effect(TEMP_DEF dst, TEMP tmp, KILL cr); 3088 ins_cost(SVE_COST); 3089 format %{ "sve_reduce_maxL $dst, $src1, $pg, $src2\t# maxL reduction predicated (sve)" %} 3090 ins_encode %{ 3091 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG, 3092 $src1$$Register, as_FloatRegister($src2$$reg), 3093 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg)); 3094 %} 3095 ins_pipe(pipe_slow); 3096 %} 3097 3098 instruct reduce_maxF_masked(vRegF dst, vRegF src1, vReg src2, pRegGov pg) %{ 3099 predicate(UseSVE > 0 && 3100 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT); 3101 match(Set dst (MaxReductionV (Binary src1 src2) pg)); 3102 ins_cost(SVE_COST); 3103 effect(TEMP_DEF dst); 3104 format %{ "sve_reduce_maxF $dst, $src1, $pg, $src2\t# maxF reduction predicated (sve)" %} 3105 ins_encode %{ 3106 __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg)); 3107 __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg)); 3108 %} 3109 ins_pipe(pipe_slow); 3110 %} 3111 3112 instruct reduce_maxD_masked(vRegD dst, vRegD src1, vReg src2, pRegGov pg) %{ 3113 predicate(UseSVE > 0 && 3114 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE); 3115 match(Set dst (MaxReductionV (Binary src1 src2) pg)); 3116 ins_cost(SVE_COST); 3117 effect(TEMP_DEF dst); 3118 format %{ "sve_reduce_maxD $dst, $src1, $pg, $src2\t# maxD reduction predicated (sve)" %} 3119 ins_encode %{ 3120 __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg)); 3121 __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg)); 3122 %} 3123 ins_pipe(pipe_slow); 3124 %} 3125 3126 // vector min reduction 3127 3128 instruct reduce_minI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{ 3129 predicate(UseSVE > 0 && 3130 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize && 3131 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG && 3132 is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type())); 3133 match(Set dst (MinReductionV src1 src2)); 3134 effect(TEMP_DEF dst, TEMP tmp, KILL cr); 3135 ins_cost(SVE_COST); 3136 format %{ "sve_reduce_minI $dst, $src1, $src2\t# minI reduction (sve)" %} 3137 ins_encode %{ 3138 BasicType bt = Matcher::vector_element_basic_type(this, $src2); 3139 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt, 3140 $src1$$Register, as_FloatRegister($src2$$reg), 3141 ptrue, as_FloatRegister($tmp$$reg)); 3142 %} 3143 ins_pipe(pipe_slow); 3144 %} 3145 3146 instruct reduce_minL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{ 3147 predicate(UseSVE > 0 && 3148 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize && 3149 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG); 3150 match(Set dst (MinReductionV src1 src2)); 3151 effect(TEMP_DEF dst, TEMP tmp, KILL cr); 3152 ins_cost(SVE_COST); 3153 format %{ "sve_reduce_minL $dst, $src1, $src2\t# minL reduction (sve)" %} 3154 ins_encode %{ 3155 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG, 3156 $src1$$Register, as_FloatRegister($src2$$reg), 3157 ptrue, as_FloatRegister($tmp$$reg)); 3158 %} 3159 ins_pipe(pipe_slow); 3160 %} 3161 3162 instruct reduce_minI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp, 3163 pRegGov ptmp, rFlagsReg cr) %{ 3164 predicate(UseSVE > 0 && 3165 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize && 3166 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG && 3167 is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type())); 3168 match(Set dst (MinReductionV src1 src2)); 3169 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr); 3170 ins_cost(2 * SVE_COST); 3171 format %{ "sve_reduce_minI $dst, $src1, $src2\t# minI reduction partial (sve)" %} 3172 ins_encode %{ 3173 BasicType bt = Matcher::vector_element_basic_type(this, $src2); 3174 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt); 3175 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2)); 3176 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt, 3177 $src1$$Register, as_FloatRegister($src2$$reg), 3178 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg)); 3179 %} 3180 ins_pipe(pipe_slow); 3181 %} 3182 3183 instruct reduce_minL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp, 3184 pRegGov ptmp, rFlagsReg cr) %{ 3185 predicate(UseSVE > 0 && 3186 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize && 3187 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG); 3188 match(Set dst (MinReductionV src1 src2)); 3189 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr); 3190 ins_cost(2 * SVE_COST); 3191 format %{ "sve_reduce_minL $dst, $src1, $src2\t# minL reduction partial (sve)" %} 3192 ins_encode %{ 3193 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2)); 3194 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG, 3195 $src1$$Register, as_FloatRegister($src2$$reg), 3196 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg)); 3197 %} 3198 ins_pipe(pipe_slow); 3199 %} 3200 3201 instruct reduce_minF(vRegF dst, vRegF src1, vReg src2) %{ 3202 predicate(UseSVE > 0 && 3203 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT && 3204 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize); 3205 match(Set dst (MinReductionV src1 src2)); 3206 ins_cost(INSN_COST); 3207 effect(TEMP_DEF dst); 3208 format %{ "sve_reduce_minF $dst, $src1, $src2\t# minF reduction (sve)" %} 3209 ins_encode %{ 3210 __ sve_fminv(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src2$$reg)); 3211 __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg)); 3212 %} 3213 ins_pipe(pipe_slow); 3214 %} 3215 3216 instruct reduce_minF_partial(vRegF dst, vRegF src1, vReg src2, 3217 pRegGov ptmp, rFlagsReg cr) %{ 3218 predicate(UseSVE > 0 && 3219 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT && 3220 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize); 3221 match(Set dst (MinReductionV src1 src2)); 3222 ins_cost(INSN_COST); 3223 effect(TEMP_DEF dst, TEMP ptmp, KILL cr); 3224 format %{ "sve_reduce_minF $dst, $src1, $src2\t# minF reduction partial (sve)" %} 3225 ins_encode %{ 3226 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src2)); 3227 __ sve_fminv(as_FloatRegister($dst$$reg), __ S, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg)); 3228 __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg)); 3229 %} 3230 ins_pipe(pipe_slow); 3231 %} 3232 3233 instruct reduce_minD(vRegD dst, vRegD src1, vReg src2) %{ 3234 predicate(UseSVE > 0 && 3235 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE && 3236 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize); 3237 match(Set dst (MinReductionV src1 src2)); 3238 ins_cost(INSN_COST); 3239 effect(TEMP_DEF dst); 3240 format %{ "sve_reduce_minD $dst, $src1, $src2\t# minD reduction (sve)" %} 3241 ins_encode %{ 3242 __ sve_fminv(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src2$$reg)); 3243 __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg)); 3244 %} 3245 ins_pipe(pipe_slow); 3246 %} 3247 3248 instruct reduce_minD_partial(vRegD dst, vRegD src1, vReg src2, 3249 pRegGov ptmp, rFlagsReg cr) %{ 3250 predicate(UseSVE > 0 && 3251 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE && 3252 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize); 3253 match(Set dst (MinReductionV src1 src2)); 3254 ins_cost(INSN_COST); 3255 effect(TEMP_DEF dst, TEMP ptmp, KILL cr); 3256 format %{ "sve_reduce_minD $dst, $src1, $src2\t# minD reduction partial (sve)" %} 3257 ins_encode %{ 3258 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2)); 3259 __ sve_fminv(as_FloatRegister($dst$$reg), __ D, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg)); 3260 __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg)); 3261 %} 3262 ins_pipe(pipe_slow); 3263 %} 3264 3265 // vector min reduction - predicated 3266 3267 instruct reduce_minI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, 3268 pRegGov pg, rFlagsReg cr) %{ 3269 predicate(UseSVE > 0 && 3270 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG && 3271 is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type())); 3272 match(Set dst (MinReductionV (Binary src1 src2) pg)); 3273 effect(TEMP_DEF dst, TEMP tmp, KILL cr); 3274 ins_cost(SVE_COST); 3275 format %{ "sve_reduce_minI $dst, $src1, $pg, $src2\t# minI reduction predicated (sve)" %} 3276 ins_encode %{ 3277 BasicType bt = Matcher::vector_element_basic_type(this, $src2); 3278 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt, 3279 $src1$$Register, as_FloatRegister($src2$$reg), 3280 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg)); 3281 %} 3282 ins_pipe(pipe_slow); 3283 %} 3284 3285 instruct reduce_minL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, 3286 pRegGov pg, rFlagsReg cr) %{ 3287 predicate(UseSVE > 0 && 3288 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG); 3289 match(Set dst (MinReductionV (Binary src1 src2) pg)); 3290 effect(TEMP_DEF dst, TEMP tmp, KILL cr); 3291 ins_cost(SVE_COST); 3292 format %{ "sve_reduce_minL $dst, $src1, $pg, $src2\t# minL reduction predicated (sve)" %} 3293 ins_encode %{ 3294 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG, 3295 $src1$$Register, as_FloatRegister($src2$$reg), 3296 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg)); 3297 %} 3298 ins_pipe(pipe_slow); 3299 %} 3300 3301 instruct reduce_minF_masked(vRegF dst, vRegF src1, vReg src2, pRegGov pg) %{ 3302 predicate(UseSVE > 0 && 3303 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT); 3304 match(Set dst (MinReductionV (Binary src1 src2) pg)); 3305 ins_cost(SVE_COST); 3306 effect(TEMP_DEF dst); 3307 format %{ "sve_reduce_minF $dst, $src1, $pg, $src2\t# minF reduction predicated (sve)" %} 3308 ins_encode %{ 3309 __ sve_fminv(as_FloatRegister($dst$$reg), __ S, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg)); 3310 __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg)); 3311 %} 3312 ins_pipe(pipe_slow); 3313 %} 3314 3315 instruct reduce_minD_masked(vRegD dst, vRegD src1, vReg src2, pRegGov pg) %{ 3316 predicate(UseSVE > 0 && 3317 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE); 3318 match(Set dst (MinReductionV (Binary src1 src2) pg)); 3319 ins_cost(SVE_COST); 3320 effect(TEMP_DEF dst); 3321 format %{ "sve_reduce_minD $dst, $src1, $pg, $src2\t# minD reduction predicated (sve)" %} 3322 ins_encode %{ 3323 __ sve_fminv(as_FloatRegister($dst$$reg), __ D, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg)); 3324 __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg)); 3325 %} 3326 ins_pipe(pipe_slow); 3327 %} 3328 3329 // vector Math.rint, floor, ceil 3330 3331 instruct vroundD(vReg dst, vReg src, immI rmode) %{ 3332 predicate(UseSVE > 0 && 3333 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE); 3334 match(Set dst (RoundDoubleModeV src rmode)); 3335 format %{ "sve_frint $dst, $src, $rmode\t# vector (sve) (D)" %} 3336 ins_encode %{ 3337 switch ($rmode$$constant) { 3338 case RoundDoubleModeNode::rmode_rint: 3339 __ sve_frintn(as_FloatRegister($dst$$reg), __ D, 3340 ptrue, as_FloatRegister($src$$reg)); 3341 break; 3342 case RoundDoubleModeNode::rmode_floor: 3343 __ sve_frintm(as_FloatRegister($dst$$reg), __ D, 3344 ptrue, as_FloatRegister($src$$reg)); 3345 break; 3346 case RoundDoubleModeNode::rmode_ceil: 3347 __ sve_frintp(as_FloatRegister($dst$$reg), __ D, 3348 ptrue, as_FloatRegister($src$$reg)); 3349 break; 3350 } 3351 %} 3352 ins_pipe(pipe_slow); 3353 %} 3354 3355 instruct vroundFtoI(vReg dst, vReg src, vReg tmp1, vReg tmp2, vReg tmp3, pRegGov ptmp) 3356 %{ 3357 predicate(UseSVE > 0); 3358 match(Set dst (RoundVF src)); 3359 effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP ptmp); 3360 format %{ "sve_vround $dst, S, $src\t# round F to I vector" %} 3361 ins_encode %{ 3362 BasicType bt = Matcher::vector_element_basic_type(this); 3363 int vlen = Matcher::vector_length_in_bytes(this); 3364 if (vlen > 16) { 3365 __ vector_round_sve(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), 3366 as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg), 3367 as_PRegister($ptmp$$reg), __ S); 3368 } else { 3369 __ vector_round_neon(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), 3370 as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg), 3371 as_FloatRegister($tmp3$$reg), 3372 __ esize2arrangement(type2aelembytes(bt), 3373 /*isQ*/ vlen == 16)); 3374 } 3375 %} 3376 ins_pipe(pipe_class_default); 3377 %} 3378 3379 instruct vroundDtoL(vReg dst, vReg src, vReg tmp1, vReg tmp2, vReg tmp3, pRegGov ptmp) 3380 %{ 3381 predicate(UseSVE > 0); 3382 match(Set dst (RoundVD src)); 3383 effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP ptmp); 3384 format %{ "sve_vround $dst, D, $src\t# round D to L vector" %} 3385 ins_encode %{ 3386 BasicType bt = Matcher::vector_element_basic_type(this); 3387 int vlen = Matcher::vector_length_in_bytes(this); 3388 if (vlen > 16) { 3389 __ vector_round_sve(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), 3390 as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg), 3391 as_PRegister($ptmp$$reg), __ D); 3392 } else { 3393 __ vector_round_neon(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), 3394 as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg), 3395 as_FloatRegister($tmp3$$reg), 3396 __ esize2arrangement(type2aelembytes(bt), 3397 /*isQ*/ vlen == 16)); 3398 } 3399 %} 3400 ins_pipe(pipe_class_default); 3401 %} 3402 3403 // vector replicate 3404 3405 instruct replicateB(vReg dst, iRegIorL2I src) %{ 3406 predicate(UseSVE > 0); 3407 match(Set dst (ReplicateB src)); 3408 ins_cost(SVE_COST); 3409 format %{ "sve_dup $dst, $src\t# vector (sve) (B)" %} 3410 ins_encode %{ 3411 __ sve_dup(as_FloatRegister($dst$$reg), __ B, as_Register($src$$reg)); 3412 %} 3413 ins_pipe(pipe_slow); 3414 %} 3415 3416 instruct replicateS(vReg dst, iRegIorL2I src) %{ 3417 predicate(UseSVE > 0); 3418 match(Set dst (ReplicateS src)); 3419 ins_cost(SVE_COST); 3420 format %{ "sve_dup $dst, $src\t# vector (sve) (H)" %} 3421 ins_encode %{ 3422 __ sve_dup(as_FloatRegister($dst$$reg), __ H, as_Register($src$$reg)); 3423 %} 3424 ins_pipe(pipe_slow); 3425 %} 3426 3427 instruct replicateI(vReg dst, iRegIorL2I src) %{ 3428 predicate(UseSVE > 0); 3429 match(Set dst (ReplicateI src)); 3430 ins_cost(SVE_COST); 3431 format %{ "sve_dup $dst, $src\t# vector (sve) (S)" %} 3432 ins_encode %{ 3433 __ sve_dup(as_FloatRegister($dst$$reg), __ S, as_Register($src$$reg)); 3434 %} 3435 ins_pipe(pipe_slow); 3436 %} 3437 3438 instruct replicateL(vReg dst, iRegL src) %{ 3439 predicate(UseSVE > 0); 3440 match(Set dst (ReplicateL src)); 3441 ins_cost(SVE_COST); 3442 format %{ "sve_dup $dst, $src\t# vector (sve) (D)" %} 3443 ins_encode %{ 3444 __ sve_dup(as_FloatRegister($dst$$reg), __ D, as_Register($src$$reg)); 3445 %} 3446 ins_pipe(pipe_slow); 3447 %} 3448 3449 instruct replicateB_imm8(vReg dst, immI8 con) %{ 3450 predicate(UseSVE > 0); 3451 match(Set dst (ReplicateB con)); 3452 ins_cost(SVE_COST); 3453 format %{ "sve_dup $dst, $con\t# vector (sve) (B)" %} 3454 ins_encode %{ 3455 __ sve_dup(as_FloatRegister($dst$$reg), __ B, $con$$constant); 3456 %} 3457 ins_pipe(pipe_slow); 3458 %} 3459 3460 instruct replicateS_imm8(vReg dst, immI8_shift8 con) %{ 3461 predicate(UseSVE > 0); 3462 match(Set dst (ReplicateS con)); 3463 ins_cost(SVE_COST); 3464 format %{ "sve_dup $dst, $con\t# vector (sve) (H)" %} 3465 ins_encode %{ 3466 __ sve_dup(as_FloatRegister($dst$$reg), __ H, $con$$constant); 3467 %} 3468 ins_pipe(pipe_slow); 3469 %} 3470 3471 instruct replicateI_imm8(vReg dst, immI8_shift8 con) %{ 3472 predicate(UseSVE > 0); 3473 match(Set dst (ReplicateI con)); 3474 ins_cost(SVE_COST); 3475 format %{ "sve_dup $dst, $con\t# vector (sve) (S)" %} 3476 ins_encode %{ 3477 __ sve_dup(as_FloatRegister($dst$$reg), __ S, $con$$constant); 3478 %} 3479 ins_pipe(pipe_slow); 3480 %} 3481 3482 instruct replicateL_imm8(vReg dst, immL8_shift8 con) %{ 3483 predicate(UseSVE > 0); 3484 match(Set dst (ReplicateL con)); 3485 ins_cost(SVE_COST); 3486 format %{ "sve_dup $dst, $con\t# vector (sve) (D)" %} 3487 ins_encode %{ 3488 __ sve_dup(as_FloatRegister($dst$$reg), __ D, $con$$constant); 3489 %} 3490 ins_pipe(pipe_slow); 3491 %} 3492 3493 instruct replicateF(vReg dst, vRegF src) %{ 3494 predicate(UseSVE > 0); 3495 match(Set dst (ReplicateF src)); 3496 ins_cost(SVE_COST); 3497 format %{ "sve_cpy $dst, $src\t# vector (sve) (S)" %} 3498 ins_encode %{ 3499 __ sve_cpy(as_FloatRegister($dst$$reg), __ S, 3500 ptrue, as_FloatRegister($src$$reg)); 3501 %} 3502 ins_pipe(pipe_slow); 3503 %} 3504 3505 instruct replicateD(vReg dst, vRegD src) %{ 3506 predicate(UseSVE > 0); 3507 match(Set dst (ReplicateD src)); 3508 ins_cost(SVE_COST); 3509 format %{ "sve_cpy $dst, $src\t# vector (sve) (D)" %} 3510 ins_encode %{ 3511 __ sve_cpy(as_FloatRegister($dst$$reg), __ D, 3512 ptrue, as_FloatRegister($src$$reg)); 3513 %} 3514 ins_pipe(pipe_slow); 3515 %} 3516 3517 // vector shift 3518 3519 instruct vasrB(vReg dst, vReg shift) %{ 3520 predicate(UseSVE > 0); 3521 match(Set dst (RShiftVB dst shift)); 3522 ins_cost(SVE_COST); 3523 format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (B)" %} 3524 ins_encode %{ 3525 __ sve_asr(as_FloatRegister($dst$$reg), __ B, 3526 ptrue, as_FloatRegister($shift$$reg)); 3527 %} 3528 ins_pipe(pipe_slow); 3529 %} 3530 3531 instruct vasrS(vReg dst, vReg shift) %{ 3532 predicate(UseSVE > 0); 3533 match(Set dst (RShiftVS dst shift)); 3534 ins_cost(SVE_COST); 3535 format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (H)" %} 3536 ins_encode %{ 3537 __ sve_asr(as_FloatRegister($dst$$reg), __ H, 3538 ptrue, as_FloatRegister($shift$$reg)); 3539 %} 3540 ins_pipe(pipe_slow); 3541 %} 3542 3543 instruct vasrI(vReg dst, vReg shift) %{ 3544 predicate(UseSVE > 0); 3545 match(Set dst (RShiftVI dst shift)); 3546 ins_cost(SVE_COST); 3547 format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (S)" %} 3548 ins_encode %{ 3549 __ sve_asr(as_FloatRegister($dst$$reg), __ S, 3550 ptrue, as_FloatRegister($shift$$reg)); 3551 %} 3552 ins_pipe(pipe_slow); 3553 %} 3554 3555 instruct vasrL(vReg dst, vReg shift) %{ 3556 predicate(UseSVE > 0); 3557 match(Set dst (RShiftVL dst shift)); 3558 ins_cost(SVE_COST); 3559 format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (D)" %} 3560 ins_encode %{ 3561 __ sve_asr(as_FloatRegister($dst$$reg), __ D, 3562 ptrue, as_FloatRegister($shift$$reg)); 3563 %} 3564 ins_pipe(pipe_slow); 3565 %} 3566 3567 instruct vlslB(vReg dst, vReg shift) %{ 3568 predicate(UseSVE > 0); 3569 match(Set dst (LShiftVB dst shift)); 3570 ins_cost(SVE_COST); 3571 format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (B)" %} 3572 ins_encode %{ 3573 __ sve_lsl(as_FloatRegister($dst$$reg), __ B, 3574 ptrue, as_FloatRegister($shift$$reg)); 3575 %} 3576 ins_pipe(pipe_slow); 3577 %} 3578 3579 instruct vlslS(vReg dst, vReg shift) %{ 3580 predicate(UseSVE > 0); 3581 match(Set dst (LShiftVS dst shift)); 3582 ins_cost(SVE_COST); 3583 format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (H)" %} 3584 ins_encode %{ 3585 __ sve_lsl(as_FloatRegister($dst$$reg), __ H, 3586 ptrue, as_FloatRegister($shift$$reg)); 3587 %} 3588 ins_pipe(pipe_slow); 3589 %} 3590 3591 instruct vlslI(vReg dst, vReg shift) %{ 3592 predicate(UseSVE > 0); 3593 match(Set dst (LShiftVI dst shift)); 3594 ins_cost(SVE_COST); 3595 format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (S)" %} 3596 ins_encode %{ 3597 __ sve_lsl(as_FloatRegister($dst$$reg), __ S, 3598 ptrue, as_FloatRegister($shift$$reg)); 3599 %} 3600 ins_pipe(pipe_slow); 3601 %} 3602 3603 instruct vlslL(vReg dst, vReg shift) %{ 3604 predicate(UseSVE > 0); 3605 match(Set dst (LShiftVL dst shift)); 3606 ins_cost(SVE_COST); 3607 format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (D)" %} 3608 ins_encode %{ 3609 __ sve_lsl(as_FloatRegister($dst$$reg), __ D, 3610 ptrue, as_FloatRegister($shift$$reg)); 3611 %} 3612 ins_pipe(pipe_slow); 3613 %} 3614 3615 instruct vlsrB(vReg dst, vReg shift) %{ 3616 predicate(UseSVE > 0); 3617 match(Set dst (URShiftVB dst shift)); 3618 ins_cost(SVE_COST); 3619 format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (B)" %} 3620 ins_encode %{ 3621 __ sve_lsr(as_FloatRegister($dst$$reg), __ B, 3622 ptrue, as_FloatRegister($shift$$reg)); 3623 %} 3624 ins_pipe(pipe_slow); 3625 %} 3626 3627 instruct vlsrS(vReg dst, vReg shift) %{ 3628 predicate(UseSVE > 0); 3629 match(Set dst (URShiftVS dst shift)); 3630 ins_cost(SVE_COST); 3631 format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (H)" %} 3632 ins_encode %{ 3633 __ sve_lsr(as_FloatRegister($dst$$reg), __ H, 3634 ptrue, as_FloatRegister($shift$$reg)); 3635 %} 3636 ins_pipe(pipe_slow); 3637 %} 3638 3639 instruct vlsrI(vReg dst, vReg shift) %{ 3640 predicate(UseSVE > 0); 3641 match(Set dst (URShiftVI dst shift)); 3642 ins_cost(SVE_COST); 3643 format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (S)" %} 3644 ins_encode %{ 3645 __ sve_lsr(as_FloatRegister($dst$$reg), __ S, 3646 ptrue, as_FloatRegister($shift$$reg)); 3647 %} 3648 ins_pipe(pipe_slow); 3649 %} 3650 3651 instruct vlsrL(vReg dst, vReg shift) %{ 3652 predicate(UseSVE > 0); 3653 match(Set dst (URShiftVL dst shift)); 3654 ins_cost(SVE_COST); 3655 format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (D)" %} 3656 ins_encode %{ 3657 __ sve_lsr(as_FloatRegister($dst$$reg), __ D, 3658 ptrue, as_FloatRegister($shift$$reg)); 3659 %} 3660 ins_pipe(pipe_slow); 3661 %} 3662 3663 instruct vasrB_imm(vReg dst, vReg src, immI shift) %{ 3664 predicate(UseSVE > 0); 3665 match(Set dst (RShiftVB src (RShiftCntV shift))); 3666 ins_cost(SVE_COST); 3667 format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (B)" %} 3668 ins_encode %{ 3669 int con = (int)$shift$$constant; 3670 if (con == 0) { 3671 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), 3672 as_FloatRegister($src$$reg)); 3673 return; 3674 } 3675 if (con >= 8) con = 7; 3676 __ sve_asr(as_FloatRegister($dst$$reg), __ B, 3677 as_FloatRegister($src$$reg), con); 3678 %} 3679 ins_pipe(pipe_slow); 3680 %} 3681 3682 instruct vasrS_imm(vReg dst, vReg src, immI shift) %{ 3683 predicate(UseSVE > 0); 3684 match(Set dst (RShiftVS src (RShiftCntV shift))); 3685 ins_cost(SVE_COST); 3686 format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (H)" %} 3687 ins_encode %{ 3688 int con = (int)$shift$$constant; 3689 if (con == 0) { 3690 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), 3691 as_FloatRegister($src$$reg)); 3692 return; 3693 } 3694 if (con >= 16) con = 15; 3695 __ sve_asr(as_FloatRegister($dst$$reg), __ H, 3696 as_FloatRegister($src$$reg), con); 3697 %} 3698 ins_pipe(pipe_slow); 3699 %} 3700 3701 instruct vasrI_imm(vReg dst, vReg src, immI shift) %{ 3702 predicate(UseSVE > 0); 3703 match(Set dst (RShiftVI src (RShiftCntV shift))); 3704 ins_cost(SVE_COST); 3705 format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (S)" %} 3706 ins_encode %{ 3707 int con = (int)$shift$$constant; 3708 if (con == 0) { 3709 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), 3710 as_FloatRegister($src$$reg)); 3711 return; 3712 } 3713 __ sve_asr(as_FloatRegister($dst$$reg), __ S, 3714 as_FloatRegister($src$$reg), con); 3715 %} 3716 ins_pipe(pipe_slow); 3717 %} 3718 3719 instruct vasrL_imm(vReg dst, vReg src, immI shift) %{ 3720 predicate(UseSVE > 0); 3721 match(Set dst (RShiftVL src (RShiftCntV shift))); 3722 ins_cost(SVE_COST); 3723 format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (D)" %} 3724 ins_encode %{ 3725 int con = (int)$shift$$constant; 3726 if (con == 0) { 3727 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), 3728 as_FloatRegister($src$$reg)); 3729 return; 3730 } 3731 __ sve_asr(as_FloatRegister($dst$$reg), __ D, 3732 as_FloatRegister($src$$reg), con); 3733 %} 3734 ins_pipe(pipe_slow); 3735 %} 3736 3737 instruct vlsrB_imm(vReg dst, vReg src, immI shift) %{ 3738 predicate(UseSVE > 0); 3739 match(Set dst (URShiftVB src (RShiftCntV shift))); 3740 ins_cost(SVE_COST); 3741 format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (B)" %} 3742 ins_encode %{ 3743 int con = (int)$shift$$constant; 3744 if (con == 0) { 3745 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), 3746 as_FloatRegister($src$$reg)); 3747 return; 3748 } 3749 if (con >= 8) { 3750 __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), 3751 as_FloatRegister($src$$reg)); 3752 return; 3753 } 3754 __ sve_lsr(as_FloatRegister($dst$$reg), __ B, 3755 as_FloatRegister($src$$reg), con); 3756 %} 3757 ins_pipe(pipe_slow); 3758 %} 3759 3760 instruct vlsrS_imm(vReg dst, vReg src, immI shift) %{ 3761 predicate(UseSVE > 0); 3762 match(Set dst (URShiftVS src (RShiftCntV shift))); 3763 ins_cost(SVE_COST); 3764 format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (H)" %} 3765 ins_encode %{ 3766 int con = (int)$shift$$constant; 3767 if (con == 0) { 3768 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), 3769 as_FloatRegister($src$$reg)); 3770 return; 3771 } 3772 if (con >= 16) { 3773 __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), 3774 as_FloatRegister($src$$reg)); 3775 return; 3776 } 3777 __ sve_lsr(as_FloatRegister($dst$$reg), __ H, 3778 as_FloatRegister($src$$reg), con); 3779 %} 3780 ins_pipe(pipe_slow); 3781 %} 3782 3783 instruct vlsrI_imm(vReg dst, vReg src, immI shift) %{ 3784 predicate(UseSVE > 0); 3785 match(Set dst (URShiftVI src (RShiftCntV shift))); 3786 ins_cost(SVE_COST); 3787 format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (S)" %} 3788 ins_encode %{ 3789 int con = (int)$shift$$constant; 3790 if (con == 0) { 3791 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), 3792 as_FloatRegister($src$$reg)); 3793 return; 3794 } 3795 __ sve_lsr(as_FloatRegister($dst$$reg), __ S, 3796 as_FloatRegister($src$$reg), con); 3797 %} 3798 ins_pipe(pipe_slow); 3799 %} 3800 3801 instruct vlsrL_imm(vReg dst, vReg src, immI shift) %{ 3802 predicate(UseSVE > 0); 3803 match(Set dst (URShiftVL src (RShiftCntV shift))); 3804 ins_cost(SVE_COST); 3805 format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (D)" %} 3806 ins_encode %{ 3807 int con = (int)$shift$$constant; 3808 if (con == 0) { 3809 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), 3810 as_FloatRegister($src$$reg)); 3811 return; 3812 } 3813 __ sve_lsr(as_FloatRegister($dst$$reg), __ D, 3814 as_FloatRegister($src$$reg), con); 3815 %} 3816 ins_pipe(pipe_slow); 3817 %} 3818 3819 instruct vlslB_imm(vReg dst, vReg src, immI shift) %{ 3820 predicate(UseSVE > 0); 3821 match(Set dst (LShiftVB src (LShiftCntV shift))); 3822 ins_cost(SVE_COST); 3823 format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (B)" %} 3824 ins_encode %{ 3825 int con = (int)$shift$$constant; 3826 if (con >= 8) { 3827 __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), 3828 as_FloatRegister($src$$reg)); 3829 return; 3830 } 3831 __ sve_lsl(as_FloatRegister($dst$$reg), __ B, 3832 as_FloatRegister($src$$reg), con); 3833 %} 3834 ins_pipe(pipe_slow); 3835 %} 3836 3837 instruct vlslS_imm(vReg dst, vReg src, immI shift) %{ 3838 predicate(UseSVE > 0); 3839 match(Set dst (LShiftVS src (LShiftCntV shift))); 3840 ins_cost(SVE_COST); 3841 format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (H)" %} 3842 ins_encode %{ 3843 int con = (int)$shift$$constant; 3844 if (con >= 16) { 3845 __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), 3846 as_FloatRegister($src$$reg)); 3847 return; 3848 } 3849 __ sve_lsl(as_FloatRegister($dst$$reg), __ H, 3850 as_FloatRegister($src$$reg), con); 3851 %} 3852 ins_pipe(pipe_slow); 3853 %} 3854 3855 instruct vlslI_imm(vReg dst, vReg src, immI shift) %{ 3856 predicate(UseSVE > 0); 3857 match(Set dst (LShiftVI src (LShiftCntV shift))); 3858 ins_cost(SVE_COST); 3859 format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (S)" %} 3860 ins_encode %{ 3861 int con = (int)$shift$$constant; 3862 __ sve_lsl(as_FloatRegister($dst$$reg), __ S, 3863 as_FloatRegister($src$$reg), con); 3864 %} 3865 ins_pipe(pipe_slow); 3866 %} 3867 3868 instruct vlslL_imm(vReg dst, vReg src, immI shift) %{ 3869 predicate(UseSVE > 0); 3870 match(Set dst (LShiftVL src (LShiftCntV shift))); 3871 ins_cost(SVE_COST); 3872 format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (D)" %} 3873 ins_encode %{ 3874 int con = (int)$shift$$constant; 3875 __ sve_lsl(as_FloatRegister($dst$$reg), __ D, 3876 as_FloatRegister($src$$reg), con); 3877 %} 3878 ins_pipe(pipe_slow); 3879 %} 3880 3881 instruct vshiftcntB(vReg dst, iRegIorL2I cnt) %{ 3882 predicate(UseSVE > 0 && 3883 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE)); 3884 match(Set dst (LShiftCntV cnt)); 3885 match(Set dst (RShiftCntV cnt)); 3886 format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (B)" %} 3887 ins_encode %{ 3888 __ sve_dup(as_FloatRegister($dst$$reg), __ B, as_Register($cnt$$reg)); 3889 %} 3890 ins_pipe(pipe_slow); 3891 %} 3892 3893 instruct vshiftcntS(vReg dst, iRegIorL2I cnt) %{ 3894 predicate(UseSVE > 0 && 3895 (n->bottom_type()->is_vect()->element_basic_type() == T_SHORT || 3896 (n->bottom_type()->is_vect()->element_basic_type() == T_CHAR))); 3897 match(Set dst (LShiftCntV cnt)); 3898 match(Set dst (RShiftCntV cnt)); 3899 format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (H)" %} 3900 ins_encode %{ 3901 __ sve_dup(as_FloatRegister($dst$$reg), __ H, as_Register($cnt$$reg)); 3902 %} 3903 ins_pipe(pipe_slow); 3904 %} 3905 3906 instruct vshiftcntI(vReg dst, iRegIorL2I cnt) %{ 3907 predicate(UseSVE > 0 && 3908 (n->bottom_type()->is_vect()->element_basic_type() == T_INT)); 3909 match(Set dst (LShiftCntV cnt)); 3910 match(Set dst (RShiftCntV cnt)); 3911 format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (S)" %} 3912 ins_encode %{ 3913 __ sve_dup(as_FloatRegister($dst$$reg), __ S, as_Register($cnt$$reg)); 3914 %} 3915 ins_pipe(pipe_slow); 3916 %} 3917 3918 instruct vshiftcntL(vReg dst, iRegIorL2I cnt) %{ 3919 predicate(UseSVE > 0 && 3920 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG)); 3921 match(Set dst (LShiftCntV cnt)); 3922 match(Set dst (RShiftCntV cnt)); 3923 format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (D)" %} 3924 ins_encode %{ 3925 __ sve_dup(as_FloatRegister($dst$$reg), __ D, as_Register($cnt$$reg)); 3926 %} 3927 ins_pipe(pipe_slow); 3928 %} 3929 3930 // vector shift - predicated 3931 3932 instruct vasrB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 3933 predicate(UseSVE > 0); 3934 match(Set dst_src1 (RShiftVB (Binary dst_src1 src2) pg)); 3935 ins_cost(SVE_COST); 3936 format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %} 3937 ins_encode %{ 3938 __ sve_asr(as_FloatRegister($dst_src1$$reg), __ B, 3939 as_PRegister($pg$$reg), 3940 as_FloatRegister($src2$$reg)); 3941 %} 3942 ins_pipe(pipe_slow); 3943 %} 3944 3945 instruct vasrS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 3946 predicate(UseSVE > 0); 3947 match(Set dst_src1 (RShiftVS (Binary dst_src1 src2) pg)); 3948 ins_cost(SVE_COST); 3949 format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %} 3950 ins_encode %{ 3951 __ sve_asr(as_FloatRegister($dst_src1$$reg), __ H, 3952 as_PRegister($pg$$reg), 3953 as_FloatRegister($src2$$reg)); 3954 %} 3955 ins_pipe(pipe_slow); 3956 %} 3957 3958 instruct vasrI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 3959 predicate(UseSVE > 0); 3960 match(Set dst_src1 (RShiftVI (Binary dst_src1 src2) pg)); 3961 ins_cost(SVE_COST); 3962 format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %} 3963 ins_encode %{ 3964 __ sve_asr(as_FloatRegister($dst_src1$$reg), __ S, 3965 as_PRegister($pg$$reg), 3966 as_FloatRegister($src2$$reg)); 3967 %} 3968 ins_pipe(pipe_slow); 3969 %} 3970 3971 instruct vasrL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 3972 predicate(UseSVE > 0); 3973 match(Set dst_src1 (RShiftVL (Binary dst_src1 src2) pg)); 3974 ins_cost(SVE_COST); 3975 format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %} 3976 ins_encode %{ 3977 __ sve_asr(as_FloatRegister($dst_src1$$reg), __ D, 3978 as_PRegister($pg$$reg), 3979 as_FloatRegister($src2$$reg)); 3980 %} 3981 ins_pipe(pipe_slow); 3982 %} 3983 3984 instruct vlslB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 3985 predicate(UseSVE > 0); 3986 match(Set dst_src1 (LShiftVB (Binary dst_src1 src2) pg)); 3987 ins_cost(SVE_COST); 3988 format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %} 3989 ins_encode %{ 3990 __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ B, 3991 as_PRegister($pg$$reg), 3992 as_FloatRegister($src2$$reg)); 3993 %} 3994 ins_pipe(pipe_slow); 3995 %} 3996 3997 instruct vlslS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 3998 predicate(UseSVE > 0); 3999 match(Set dst_src1 (LShiftVS (Binary dst_src1 src2) pg)); 4000 ins_cost(SVE_COST); 4001 format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %} 4002 ins_encode %{ 4003 __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ H, 4004 as_PRegister($pg$$reg), 4005 as_FloatRegister($src2$$reg)); 4006 %} 4007 ins_pipe(pipe_slow); 4008 %} 4009 4010 instruct vlslI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 4011 predicate(UseSVE > 0); 4012 match(Set dst_src1 (LShiftVI (Binary dst_src1 src2) pg)); 4013 ins_cost(SVE_COST); 4014 format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %} 4015 ins_encode %{ 4016 __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ S, 4017 as_PRegister($pg$$reg), 4018 as_FloatRegister($src2$$reg)); 4019 %} 4020 ins_pipe(pipe_slow); 4021 %} 4022 4023 instruct vlslL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 4024 predicate(UseSVE > 0); 4025 match(Set dst_src1 (LShiftVL (Binary dst_src1 src2) pg)); 4026 ins_cost(SVE_COST); 4027 format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %} 4028 ins_encode %{ 4029 __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ D, 4030 as_PRegister($pg$$reg), 4031 as_FloatRegister($src2$$reg)); 4032 %} 4033 ins_pipe(pipe_slow); 4034 %} 4035 4036 instruct vlsrB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 4037 predicate(UseSVE > 0); 4038 match(Set dst_src1 (URShiftVB (Binary dst_src1 src2) pg)); 4039 ins_cost(SVE_COST); 4040 format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %} 4041 ins_encode %{ 4042 __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ B, 4043 as_PRegister($pg$$reg), 4044 as_FloatRegister($src2$$reg)); 4045 %} 4046 ins_pipe(pipe_slow); 4047 %} 4048 4049 instruct vlsrS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 4050 predicate(UseSVE > 0); 4051 match(Set dst_src1 (URShiftVS (Binary dst_src1 src2) pg)); 4052 ins_cost(SVE_COST); 4053 format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %} 4054 ins_encode %{ 4055 __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ H, 4056 as_PRegister($pg$$reg), 4057 as_FloatRegister($src2$$reg)); 4058 %} 4059 ins_pipe(pipe_slow); 4060 %} 4061 4062 instruct vlsrI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 4063 predicate(UseSVE > 0); 4064 match(Set dst_src1 (URShiftVI (Binary dst_src1 src2) pg)); 4065 ins_cost(SVE_COST); 4066 format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %} 4067 ins_encode %{ 4068 __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ S, 4069 as_PRegister($pg$$reg), 4070 as_FloatRegister($src2$$reg)); 4071 %} 4072 ins_pipe(pipe_slow); 4073 %} 4074 4075 instruct vlsrL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 4076 predicate(UseSVE > 0); 4077 match(Set dst_src1 (URShiftVL (Binary dst_src1 src2) pg)); 4078 ins_cost(SVE_COST); 4079 format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %} 4080 ins_encode %{ 4081 __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ D, 4082 as_PRegister($pg$$reg), 4083 as_FloatRegister($src2$$reg)); 4084 %} 4085 ins_pipe(pipe_slow); 4086 %} 4087 4088 instruct vasrB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{ 4089 predicate(UseSVE > 0); 4090 match(Set dst_src (RShiftVB (Binary dst_src (RShiftCntV shift)) pg)); 4091 ins_cost(SVE_COST); 4092 format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %} 4093 ins_encode %{ 4094 int con = (int)$shift$$constant; 4095 assert(con > 0 && con < 8, "invalid shift immediate"); 4096 __ sve_asr(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con); 4097 %} 4098 ins_pipe(pipe_slow); 4099 %} 4100 4101 instruct vasrS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{ 4102 predicate(UseSVE > 0); 4103 match(Set dst_src (RShiftVS (Binary dst_src (RShiftCntV shift)) pg)); 4104 ins_cost(SVE_COST); 4105 format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %} 4106 ins_encode %{ 4107 int con = (int)$shift$$constant; 4108 assert(con > 0 && con < 16, "invalid shift immediate"); 4109 __ sve_asr(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con); 4110 %} 4111 ins_pipe(pipe_slow); 4112 %} 4113 4114 instruct vasrI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{ 4115 predicate(UseSVE > 0); 4116 match(Set dst_src (RShiftVI (Binary dst_src (RShiftCntV shift)) pg)); 4117 ins_cost(SVE_COST); 4118 format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %} 4119 ins_encode %{ 4120 int con = (int)$shift$$constant; 4121 assert(con > 0 && con < 32, "invalid shift immediate"); 4122 __ sve_asr(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con); 4123 %} 4124 ins_pipe(pipe_slow); 4125 %} 4126 4127 instruct vasrL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{ 4128 predicate(UseSVE > 0); 4129 match(Set dst_src (RShiftVL (Binary dst_src (RShiftCntV shift)) pg)); 4130 ins_cost(SVE_COST); 4131 format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %} 4132 ins_encode %{ 4133 int con = (int)$shift$$constant; 4134 assert(con > 0 && con < 64, "invalid shift immediate"); 4135 __ sve_asr(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con); 4136 %} 4137 ins_pipe(pipe_slow); 4138 %} 4139 4140 instruct vlsrB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{ 4141 predicate(UseSVE > 0); 4142 match(Set dst_src (URShiftVB (Binary dst_src (RShiftCntV shift)) pg)); 4143 ins_cost(SVE_COST); 4144 format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %} 4145 ins_encode %{ 4146 int con = (int)$shift$$constant; 4147 assert(con > 0 && con < 8, "invalid shift immediate"); 4148 __ sve_lsr(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con); 4149 %} 4150 ins_pipe(pipe_slow); 4151 %} 4152 4153 instruct vlsrS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{ 4154 predicate(UseSVE > 0); 4155 match(Set dst_src (URShiftVS (Binary dst_src (RShiftCntV shift)) pg)); 4156 ins_cost(SVE_COST); 4157 format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %} 4158 ins_encode %{ 4159 int con = (int)$shift$$constant; 4160 assert(con > 0 && con < 16, "invalid shift immediate"); 4161 __ sve_lsr(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con); 4162 %} 4163 ins_pipe(pipe_slow); 4164 %} 4165 4166 instruct vlsrI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{ 4167 predicate(UseSVE > 0); 4168 match(Set dst_src (URShiftVI (Binary dst_src (RShiftCntV shift)) pg)); 4169 ins_cost(SVE_COST); 4170 format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %} 4171 ins_encode %{ 4172 int con = (int)$shift$$constant; 4173 assert(con > 0 && con < 32, "invalid shift immediate"); 4174 __ sve_lsr(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con); 4175 %} 4176 ins_pipe(pipe_slow); 4177 %} 4178 4179 instruct vlsrL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{ 4180 predicate(UseSVE > 0); 4181 match(Set dst_src (URShiftVL (Binary dst_src (RShiftCntV shift)) pg)); 4182 ins_cost(SVE_COST); 4183 format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %} 4184 ins_encode %{ 4185 int con = (int)$shift$$constant; 4186 assert(con > 0 && con < 64, "invalid shift immediate"); 4187 __ sve_lsr(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con); 4188 %} 4189 ins_pipe(pipe_slow); 4190 %} 4191 4192 instruct vlslB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{ 4193 predicate(UseSVE > 0); 4194 match(Set dst_src (LShiftVB (Binary dst_src (LShiftCntV shift)) pg)); 4195 ins_cost(SVE_COST); 4196 format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %} 4197 ins_encode %{ 4198 int con = (int)$shift$$constant; 4199 assert(con >= 0 && con < 8, "invalid shift immediate"); 4200 __ sve_lsl(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con); 4201 %} 4202 ins_pipe(pipe_slow); 4203 %} 4204 4205 instruct vlslS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{ 4206 predicate(UseSVE > 0); 4207 match(Set dst_src (LShiftVS (Binary dst_src (LShiftCntV shift)) pg)); 4208 ins_cost(SVE_COST); 4209 format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %} 4210 ins_encode %{ 4211 int con = (int)$shift$$constant; 4212 assert(con >= 0 && con < 16, "invalid shift immediate"); 4213 __ sve_lsl(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con); 4214 %} 4215 ins_pipe(pipe_slow); 4216 %} 4217 4218 instruct vlslI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{ 4219 predicate(UseSVE > 0); 4220 match(Set dst_src (LShiftVI (Binary dst_src (LShiftCntV shift)) pg)); 4221 ins_cost(SVE_COST); 4222 format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %} 4223 ins_encode %{ 4224 int con = (int)$shift$$constant; 4225 assert(con >= 0 && con < 32, "invalid shift immediate"); 4226 __ sve_lsl(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con); 4227 %} 4228 ins_pipe(pipe_slow); 4229 %} 4230 4231 instruct vlslL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{ 4232 predicate(UseSVE > 0); 4233 match(Set dst_src (LShiftVL (Binary dst_src (LShiftCntV shift)) pg)); 4234 ins_cost(SVE_COST); 4235 format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %} 4236 ins_encode %{ 4237 int con = (int)$shift$$constant; 4238 assert(con >= 0 && con < 64, "invalid shift immediate"); 4239 __ sve_lsl(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con); 4240 %} 4241 ins_pipe(pipe_slow); 4242 %} 4243 4244 // vector sqrt 4245 4246 instruct vsqrtF(vReg dst, vReg src) %{ 4247 predicate(UseSVE > 0 && 4248 !n->as_Vector()->is_predicated_vector()); 4249 match(Set dst (SqrtVF src)); 4250 ins_cost(SVE_COST); 4251 format %{ "sve_fsqrt $dst, $src\t# vector (sve) (S)" %} 4252 ins_encode %{ 4253 __ sve_fsqrt(as_FloatRegister($dst$$reg), __ S, 4254 ptrue, as_FloatRegister($src$$reg)); 4255 %} 4256 ins_pipe(pipe_slow); 4257 %} 4258 4259 instruct vsqrtD(vReg dst, vReg src) %{ 4260 predicate(UseSVE > 0 && 4261 !n->as_Vector()->is_predicated_vector()); 4262 match(Set dst (SqrtVD src)); 4263 ins_cost(SVE_COST); 4264 format %{ "sve_fsqrt $dst, $src\t# vector (sve) (D)" %} 4265 ins_encode %{ 4266 __ sve_fsqrt(as_FloatRegister($dst$$reg), __ D, 4267 ptrue, as_FloatRegister($src$$reg)); 4268 %} 4269 ins_pipe(pipe_slow); 4270 %} 4271 4272 // vector sqrt - predicated 4273 4274 instruct vsqrtF_masked(vReg dst_src, pRegGov pg) %{ 4275 predicate(UseSVE > 0); 4276 match(Set dst_src (SqrtVF dst_src pg)); 4277 ins_cost(SVE_COST); 4278 format %{ "sve_fsqrt $dst_src, $pg, $dst_src\t# vector (sve) (S)" %} 4279 ins_encode %{ 4280 __ sve_fsqrt(as_FloatRegister($dst_src$$reg), __ S, 4281 as_PRegister($pg$$reg), 4282 as_FloatRegister($dst_src$$reg)); 4283 %} 4284 ins_pipe(pipe_slow); 4285 %} 4286 4287 instruct vsqrtD_masked(vReg dst_src, pRegGov pg) %{ 4288 predicate(UseSVE > 0); 4289 match(Set dst_src (SqrtVD dst_src pg)); 4290 ins_cost(SVE_COST); 4291 format %{ "sve_fsqrt $dst_src, $pg, $dst_src\t# vector (sve) (D)" %} 4292 ins_encode %{ 4293 __ sve_fsqrt(as_FloatRegister($dst_src$$reg), __ D, 4294 as_PRegister($pg$$reg), 4295 as_FloatRegister($dst_src$$reg)); 4296 %} 4297 ins_pipe(pipe_slow); 4298 %} 4299 4300 // vector sub 4301 4302 instruct vsubB(vReg dst, vReg src1, vReg src2) %{ 4303 predicate(UseSVE > 0); 4304 match(Set dst (SubVB src1 src2)); 4305 ins_cost(SVE_COST); 4306 format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (B)" %} 4307 ins_encode %{ 4308 __ sve_sub(as_FloatRegister($dst$$reg), __ B, 4309 as_FloatRegister($src1$$reg), 4310 as_FloatRegister($src2$$reg)); 4311 %} 4312 ins_pipe(pipe_slow); 4313 %} 4314 4315 instruct vsubS(vReg dst, vReg src1, vReg src2) %{ 4316 predicate(UseSVE > 0); 4317 match(Set dst (SubVS src1 src2)); 4318 ins_cost(SVE_COST); 4319 format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (H)" %} 4320 ins_encode %{ 4321 __ sve_sub(as_FloatRegister($dst$$reg), __ H, 4322 as_FloatRegister($src1$$reg), 4323 as_FloatRegister($src2$$reg)); 4324 %} 4325 ins_pipe(pipe_slow); 4326 %} 4327 4328 instruct vsubI(vReg dst, vReg src1, vReg src2) %{ 4329 predicate(UseSVE > 0); 4330 match(Set dst (SubVI src1 src2)); 4331 ins_cost(SVE_COST); 4332 format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (S)" %} 4333 ins_encode %{ 4334 __ sve_sub(as_FloatRegister($dst$$reg), __ S, 4335 as_FloatRegister($src1$$reg), 4336 as_FloatRegister($src2$$reg)); 4337 %} 4338 ins_pipe(pipe_slow); 4339 %} 4340 4341 instruct vsubL(vReg dst, vReg src1, vReg src2) %{ 4342 predicate(UseSVE > 0); 4343 match(Set dst (SubVL src1 src2)); 4344 ins_cost(SVE_COST); 4345 format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (D)" %} 4346 ins_encode %{ 4347 __ sve_sub(as_FloatRegister($dst$$reg), __ D, 4348 as_FloatRegister($src1$$reg), 4349 as_FloatRegister($src2$$reg)); 4350 %} 4351 ins_pipe(pipe_slow); 4352 %} 4353 4354 instruct vsubF(vReg dst, vReg src1, vReg src2) %{ 4355 predicate(UseSVE > 0); 4356 match(Set dst (SubVF src1 src2)); 4357 ins_cost(SVE_COST); 4358 format %{ "sve_fsub $dst, $src1, $src2\t # vector (sve) (S)" %} 4359 ins_encode %{ 4360 __ sve_fsub(as_FloatRegister($dst$$reg), __ S, 4361 as_FloatRegister($src1$$reg), 4362 as_FloatRegister($src2$$reg)); 4363 %} 4364 ins_pipe(pipe_slow); 4365 %} 4366 4367 instruct vsubD(vReg dst, vReg src1, vReg src2) %{ 4368 predicate(UseSVE > 0); 4369 match(Set dst (SubVD src1 src2)); 4370 ins_cost(SVE_COST); 4371 format %{ "sve_fsub $dst, $src1, $src2\t # vector (sve) (D)" %} 4372 ins_encode %{ 4373 __ sve_fsub(as_FloatRegister($dst$$reg), __ D, 4374 as_FloatRegister($src1$$reg), 4375 as_FloatRegister($src2$$reg)); 4376 %} 4377 ins_pipe(pipe_slow); 4378 %} 4379 4380 // vector sub - predicated 4381 4382 instruct vsubB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 4383 predicate(UseSVE > 0); 4384 match(Set dst_src1 (SubVB (Binary dst_src1 src2) pg)); 4385 ins_cost(SVE_COST); 4386 format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %} 4387 ins_encode %{ 4388 __ sve_sub(as_FloatRegister($dst_src1$$reg), __ B, 4389 as_PRegister($pg$$reg), 4390 as_FloatRegister($src2$$reg)); 4391 %} 4392 ins_pipe(pipe_slow); 4393 %} 4394 4395 instruct vsubS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 4396 predicate(UseSVE > 0); 4397 match(Set dst_src1 (SubVS (Binary dst_src1 src2) pg)); 4398 ins_cost(SVE_COST); 4399 format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %} 4400 ins_encode %{ 4401 __ sve_sub(as_FloatRegister($dst_src1$$reg), __ H, 4402 as_PRegister($pg$$reg), 4403 as_FloatRegister($src2$$reg)); 4404 %} 4405 ins_pipe(pipe_slow); 4406 %} 4407 4408 instruct vsubI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 4409 predicate(UseSVE > 0); 4410 match(Set dst_src1 (SubVI (Binary dst_src1 src2) pg)); 4411 ins_cost(SVE_COST); 4412 format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %} 4413 ins_encode %{ 4414 __ sve_sub(as_FloatRegister($dst_src1$$reg), __ S, 4415 as_PRegister($pg$$reg), 4416 as_FloatRegister($src2$$reg)); 4417 %} 4418 ins_pipe(pipe_slow); 4419 %} 4420 4421 instruct vsubL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 4422 predicate(UseSVE > 0); 4423 match(Set dst_src1 (SubVL (Binary dst_src1 src2) pg)); 4424 ins_cost(SVE_COST); 4425 format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %} 4426 ins_encode %{ 4427 __ sve_sub(as_FloatRegister($dst_src1$$reg), __ D, 4428 as_PRegister($pg$$reg), 4429 as_FloatRegister($src2$$reg)); 4430 %} 4431 ins_pipe(pipe_slow); 4432 %} 4433 4434 instruct vsubF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 4435 predicate(UseSVE > 0); 4436 match(Set dst_src1 (SubVF (Binary dst_src1 src2) pg)); 4437 ins_cost(SVE_COST); 4438 format %{ "sve_fsub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %} 4439 ins_encode %{ 4440 __ sve_fsub(as_FloatRegister($dst_src1$$reg), __ S, 4441 as_PRegister($pg$$reg), 4442 as_FloatRegister($src2$$reg)); 4443 %} 4444 ins_pipe(pipe_slow); 4445 %} 4446 4447 instruct vsubD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{ 4448 predicate(UseSVE > 0); 4449 match(Set dst_src1 (SubVD (Binary dst_src1 src2) pg)); 4450 ins_cost(SVE_COST); 4451 format %{ "sve_fsub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %} 4452 ins_encode %{ 4453 __ sve_fsub(as_FloatRegister($dst_src1$$reg), __ D, 4454 as_PRegister($pg$$reg), 4455 as_FloatRegister($src2$$reg)); 4456 %} 4457 ins_pipe(pipe_slow); 4458 %} 4459 4460 // ------------------------------ Vector mask cast -------------------------- 4461 4462 instruct vmaskcast(pRegGov dst_src) %{ 4463 predicate(UseSVE > 0 && 4464 n->bottom_type()->is_vect()->length() == n->in(1)->bottom_type()->is_vect()->length() && 4465 n->bottom_type()->is_vect()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes()); 4466 match(Set dst_src (VectorMaskCast dst_src)); 4467 ins_cost(0); 4468 format %{ "vmaskcast $dst_src\t# empty (sve)" %} 4469 ins_encode %{ 4470 // empty 4471 %} 4472 ins_pipe(pipe_class_empty); 4473 %} 4474 4475 instruct vmaskcast_extend(pRegGov dst, pReg src) 4476 %{ 4477 predicate(UseSVE > 0 && 4478 (Matcher::vector_length_in_bytes(n) == 2 * Matcher::vector_length_in_bytes(n->in(1)) || 4479 Matcher::vector_length_in_bytes(n) == 4 * Matcher::vector_length_in_bytes(n->in(1)) || 4480 Matcher::vector_length_in_bytes(n) == 8 * Matcher::vector_length_in_bytes(n->in(1)))); 4481 match(Set dst (VectorMaskCast src)); 4482 ins_cost(SVE_COST * 3); 4483 format %{ "sve_vmaskcast_extend $dst, $src\t# extend predicate $src" %} 4484 ins_encode %{ 4485 __ sve_vmaskcast_extend(as_PRegister($dst$$reg), as_PRegister($src$$reg), 4486 Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src)); 4487 %} 4488 ins_pipe(pipe_slow); 4489 %} 4490 4491 instruct vmaskcast_narrow(pRegGov dst, pReg src) 4492 %{ 4493 predicate(UseSVE > 0 && 4494 (Matcher::vector_length_in_bytes(n) * 2 == Matcher::vector_length_in_bytes(n->in(1)) || 4495 Matcher::vector_length_in_bytes(n) * 4 == Matcher::vector_length_in_bytes(n->in(1)) || 4496 Matcher::vector_length_in_bytes(n) * 8 == Matcher::vector_length_in_bytes(n->in(1)))); 4497 match(Set dst (VectorMaskCast src)); 4498 ins_cost(SVE_COST * 3); 4499 format %{ "sve_vmaskcast_narrow $dst, $src\t# narrow predicate $src" %} 4500 ins_encode %{ 4501 __ sve_vmaskcast_narrow(as_PRegister($dst$$reg), as_PRegister($src$$reg), 4502 Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src)); 4503 %} 4504 ins_pipe(pipe_slow); 4505 %} 4506 4507 // ------------------------------ Vector cast ------------------------------- 4508 4509 instruct vcvtBtoX_extend(vReg dst, vReg src) 4510 %{ 4511 predicate(UseSVE > 0); 4512 match(Set dst (VectorCastB2X src)); 4513 ins_cost(2 * SVE_COST); 4514 format %{ "sve_vectorcast_b2x $dst, $src\t# convert B to X vector (extend)" %} 4515 ins_encode %{ 4516 BasicType to_bt = Matcher::vector_element_basic_type(this); 4517 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt); 4518 __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ B); 4519 if (to_bt == T_FLOAT || to_bt == T_DOUBLE) { 4520 __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size); 4521 } 4522 %} 4523 ins_pipe(pipe_slow); 4524 %} 4525 4526 instruct vcvtStoB(vReg dst, vReg src, vReg tmp) 4527 %{ 4528 predicate(UseSVE > 0 && 4529 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE); 4530 match(Set dst (VectorCastS2X src)); 4531 effect(TEMP tmp); 4532 ins_cost(2 * SVE_COST); 4533 format %{ "sve_vectorcast_s2b $dst, $src\t# convert H to B vector" %} 4534 ins_encode %{ 4535 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B, 4536 as_FloatRegister($src$$reg), __ H, as_FloatRegister($tmp$$reg)); 4537 %} 4538 ins_pipe(pipe_slow); 4539 %} 4540 4541 instruct vcvtStoX_extend(vReg dst, vReg src) 4542 %{ 4543 predicate(UseSVE > 0 && 4544 type2aelembytes(Matcher::vector_element_basic_type(n)) > 2); 4545 match(Set dst (VectorCastS2X src)); 4546 ins_cost(2 * SVE_COST); 4547 format %{ "sve_vectorcast_s2x $dst, $src\t# convert H to X vector (extend)" %} 4548 ins_encode %{ 4549 BasicType to_bt = Matcher::vector_element_basic_type(this); 4550 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt); 4551 __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ H); 4552 if (to_bt == T_FLOAT || to_bt == T_DOUBLE) { 4553 __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size); 4554 } 4555 %} 4556 ins_pipe(pipe_slow); 4557 %} 4558 4559 instruct vcvtItoB(vReg dst, vReg src, vReg tmp) 4560 %{ 4561 predicate(UseSVE > 0 && 4562 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE); 4563 match(Set dst (VectorCastI2X src)); 4564 effect(TEMP_DEF dst, TEMP tmp); 4565 ins_cost(3 * SVE_COST); 4566 format %{ "sve_vectorcast_i2b $dst, $src\t# convert I to B vector" %} 4567 ins_encode %{ 4568 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B, 4569 as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg)); 4570 %} 4571 ins_pipe(pipe_slow); 4572 %} 4573 4574 instruct vcvtItoS(vReg dst, vReg src, vReg tmp) 4575 %{ 4576 predicate(UseSVE > 0 && 4577 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT); 4578 match(Set dst (VectorCastI2X src)); 4579 effect(TEMP tmp); 4580 ins_cost(2 * SVE_COST); 4581 format %{ "sve_vectorcast_i2s $dst, $src\t# convert I to H vector" %} 4582 ins_encode %{ 4583 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ H, 4584 as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg)); 4585 %} 4586 ins_pipe(pipe_slow); 4587 %} 4588 4589 instruct vcvtItoL(vReg dst, vReg src) 4590 %{ 4591 predicate(UseSVE > 0 && 4592 n->bottom_type()->is_vect()->element_basic_type() == T_LONG); 4593 match(Set dst (VectorCastI2X src)); 4594 ins_cost(SVE_COST); 4595 format %{ "sve_vectorcast_i2l $dst, $src\t# convert I to L vector" %} 4596 ins_encode %{ 4597 __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S); 4598 %} 4599 ins_pipe(pipe_slow); 4600 %} 4601 4602 instruct vcvtItoF(vReg dst, vReg src) 4603 %{ 4604 predicate(UseSVE > 0 && 4605 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT); 4606 match(Set dst (VectorCastI2X src)); 4607 ins_cost(SVE_COST); 4608 format %{ "sve_vectorcast_i2f $dst, $src\t# convert I to F vector" %} 4609 ins_encode %{ 4610 __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S); 4611 %} 4612 ins_pipe(pipe_slow); 4613 %} 4614 4615 instruct vcvtItoD(vReg dst, vReg src) 4616 %{ 4617 predicate(UseSVE > 0 && 4618 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE); 4619 match(Set dst (VectorCastI2X src)); 4620 ins_cost(2 * SVE_COST); 4621 format %{ "sve_vectorcast_i2d $dst, $src\t# convert I to D vector" %} 4622 ins_encode %{ 4623 __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg)); 4624 __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ D); 4625 %} 4626 ins_pipe(pipe_slow); 4627 %} 4628 4629 instruct vcvtLtoX_narrow(vReg dst, vReg src, vReg tmp) 4630 %{ 4631 predicate(UseSVE > 0 && is_integral_type(Matcher::vector_element_basic_type(n))); 4632 match(Set dst (VectorCastL2X src)); 4633 effect(TEMP_DEF dst, TEMP tmp); 4634 ins_cost(2 * SVE_COST); 4635 format %{ "sve_vectorcast_l2x $dst, $src\t# convert L to B/H/S vector (narrow)" %} 4636 ins_encode %{ 4637 BasicType to_bt = Matcher::vector_element_basic_type(this); 4638 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt); 4639 __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size, 4640 as_FloatRegister($src$$reg), __ D, as_FloatRegister($tmp$$reg)); 4641 %} 4642 ins_pipe(pipe_slow); 4643 %} 4644 4645 instruct vcvtLtoF(vReg dst, vReg src, vReg tmp) 4646 %{ 4647 predicate(UseSVE > 0 && 4648 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT); 4649 match(Set dst (VectorCastL2X src)); 4650 effect(TEMP_DEF dst, TEMP tmp); 4651 ins_cost(3 * SVE_COST); 4652 format %{ "sve_vectorcast_l2f $dst, $src\t# convert L to F vector" %} 4653 ins_encode %{ 4654 __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D); 4655 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S, 4656 as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg)); 4657 4658 %} 4659 ins_pipe(pipe_slow); 4660 %} 4661 4662 instruct vcvtLtoD(vReg dst, vReg src) 4663 %{ 4664 predicate(UseSVE > 0 && 4665 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE); 4666 match(Set dst (VectorCastL2X src)); 4667 ins_cost(SVE_COST); 4668 format %{ "sve_vectorcast_l2d $dst, $src\t# convert L to D vector" %} 4669 ins_encode %{ 4670 __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D); 4671 %} 4672 ins_pipe(pipe_slow); 4673 %} 4674 4675 instruct vcvtFtoX_narrow(vReg dst, vReg src, vReg tmp) 4676 %{ 4677 predicate(UseSVE > 0 && 4678 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE || 4679 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT)); 4680 match(Set dst (VectorCastF2X src)); 4681 effect(TEMP_DEF dst, TEMP tmp); 4682 ins_cost(3 * SVE_COST); 4683 format %{ "sve_vectorcast_f2x $dst, $src\t# convert F to B/H vector" %} 4684 ins_encode %{ 4685 BasicType to_bt = Matcher::vector_element_basic_type(this); 4686 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt); 4687 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S); 4688 __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size, 4689 as_FloatRegister($dst$$reg), __ S, as_FloatRegister($tmp$$reg)); 4690 %} 4691 ins_pipe(pipe_slow); 4692 %} 4693 4694 instruct vcvtFtoI(vReg dst, vReg src) 4695 %{ 4696 predicate(UseSVE > 0 && 4697 (n->bottom_type()->is_vect()->element_basic_type() == T_INT)); 4698 match(Set dst (VectorCastF2X src)); 4699 ins_cost(SVE_COST); 4700 format %{ "sve_vectorcast_f2x $dst, $src\t# convert F to I vector" %} 4701 ins_encode %{ 4702 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S); 4703 %} 4704 ins_pipe(pipe_slow); 4705 %} 4706 4707 instruct vcvtFtoL(vReg dst, vReg src) 4708 %{ 4709 predicate(UseSVE > 0 && 4710 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG)); 4711 match(Set dst (VectorCastF2X src)); 4712 ins_cost(SVE_COST * 2); 4713 format %{ "sve_vectorcast_f2x $dst, $src\t# convert F to L vector" %} 4714 ins_encode %{ 4715 __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg)); 4716 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S); 4717 %} 4718 ins_pipe(pipe_slow); 4719 %} 4720 4721 instruct vcvtFtoD(vReg dst, vReg src) 4722 %{ 4723 predicate(UseSVE > 0 && 4724 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE); 4725 match(Set dst (VectorCastF2X src)); 4726 ins_cost(2 * SVE_COST); 4727 format %{ "sve_vectorcast_f2d $dst, $dst\t# convert F to D vector" %} 4728 ins_encode %{ 4729 __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S); 4730 __ sve_fcvt(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S); 4731 %} 4732 ins_pipe(pipe_slow); 4733 %} 4734 4735 instruct vcvtDtoX_narrow(vReg dst, vReg src, vReg tmp) 4736 %{ 4737 predicate(UseSVE > 0 && 4738 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE || 4739 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT || 4740 n->bottom_type()->is_vect()->element_basic_type() == T_INT)); 4741 match(Set dst (VectorCastD2X src)); 4742 effect(TEMP_DEF dst, TEMP tmp); 4743 ins_cost(3 * SVE_COST); 4744 format %{ "sve_vectorcast_d2x $dst, $src\t# convert D to X vector (narrow)" %} 4745 ins_encode %{ 4746 BasicType to_bt = Matcher::vector_element_basic_type(this); 4747 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt); 4748 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D); 4749 __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size, 4750 as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg)); 4751 %} 4752 ins_pipe(pipe_slow); 4753 %} 4754 4755 instruct vcvtDtoL(vReg dst, vReg src) 4756 %{ 4757 predicate(UseSVE > 0 && 4758 n->bottom_type()->is_vect()->element_basic_type() == T_LONG); 4759 match(Set dst (VectorCastD2X src)); 4760 ins_cost(SVE_COST); 4761 format %{ "sve_vectorcast_d2l $dst, $src\t# convert D to L vector" %} 4762 ins_encode %{ 4763 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D); 4764 %} 4765 ins_pipe(pipe_slow); 4766 %} 4767 4768 instruct vcvtDtoF(vReg dst, vReg src, vReg tmp) 4769 %{ 4770 predicate(UseSVE > 0 && 4771 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT); 4772 match(Set dst (VectorCastD2X src)); 4773 effect(TEMP_DEF dst, TEMP tmp); 4774 ins_cost(3 * SVE_COST); 4775 format %{ "sve_vectorcast_d2f $dst, S, $dst\t# convert D to F vector" %} 4776 ins_encode %{ 4777 __ sve_fcvt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D); 4778 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S, 4779 as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg)); 4780 %} 4781 ins_pipe(pipe_slow); 4782 %} 4783 4784 // ------------------------------ Vector extract --------------------------------- 4785 4786 instruct extractB(iRegINoSp dst, vReg src, immI idx, vReg vtmp) 4787 %{ 4788 predicate(UseSVE > 0 && n->in(2)->get_int() >= 16); 4789 match(Set dst (ExtractB src idx)); 4790 effect(TEMP vtmp); 4791 ins_cost(2 * SVE_COST); 4792 format %{ "sve_extract_integral $dst, B, $src, $idx\t# extract from vector(B)" %} 4793 ins_encode %{ 4794 __ sve_extract_integral(as_Register($dst$$reg), __ B, as_FloatRegister($src$$reg), 4795 (int)($idx$$constant), /* is_signed */ true, as_FloatRegister($vtmp$$reg)); 4796 %} 4797 ins_pipe(pipe_slow); 4798 %} 4799 4800 instruct extractS(iRegINoSp dst, vReg src, immI idx, vReg vtmp) 4801 %{ 4802 predicate(UseSVE > 0 && n->in(2)->get_int() >= 8); 4803 match(Set dst (ExtractS src idx)); 4804 effect(TEMP vtmp); 4805 ins_cost(2 * SVE_COST); 4806 format %{ "sve_extract_integral $dst, H, $src, $idx\t# extract from vector(S)" %} 4807 ins_encode %{ 4808 __ sve_extract_integral(as_Register($dst$$reg), __ H, as_FloatRegister($src$$reg), 4809 (int)($idx$$constant), /* is_signed */ true, as_FloatRegister($vtmp$$reg)); 4810 %} 4811 ins_pipe(pipe_slow); 4812 %} 4813 4814 instruct extractI(iRegINoSp dst, vReg src, immI idx, vReg vtmp) 4815 %{ 4816 predicate(UseSVE > 0 && n->in(2)->get_int() >= 4); 4817 match(Set dst (ExtractI src idx)); 4818 effect(TEMP vtmp); 4819 ins_cost(2 * SVE_COST); 4820 format %{ "sve_extract_integral $dst, S, $src, $idx\t# extract from vector(I)" %} 4821 ins_encode %{ 4822 __ sve_extract_integral(as_Register($dst$$reg), __ S, as_FloatRegister($src$$reg), 4823 (int)($idx$$constant), /* is_signed */ true, as_FloatRegister($vtmp$$reg)); 4824 %} 4825 ins_pipe(pipe_slow); 4826 %} 4827 4828 instruct extractL(iRegLNoSp dst, vReg src, immI idx, vReg vtmp) 4829 %{ 4830 predicate(UseSVE > 0 && n->in(2)->get_int() >= 2); 4831 match(Set dst (ExtractL src idx)); 4832 effect(TEMP vtmp); 4833 ins_cost(2 * SVE_COST); 4834 format %{ "sve_extract_integral $dst, D, $src, $idx\t# extract from vector(L)" %} 4835 ins_encode %{ 4836 __ sve_extract_integral(as_Register($dst$$reg), __ D, as_FloatRegister($src$$reg), 4837 (int)($idx$$constant), /* is_signed */ false, as_FloatRegister($vtmp$$reg)); 4838 %} 4839 ins_pipe(pipe_slow); 4840 %} 4841 4842 instruct extractB_LT16(iRegINoSp dst, vReg src, immI idx) 4843 %{ 4844 predicate(UseSVE > 0 && n->in(2)->get_int() < 16); 4845 match(Set dst (ExtractB src idx)); 4846 ins_cost(INSN_COST); 4847 format %{ "smov $dst, B, $src, $idx\t# extract from vector(B)" %} 4848 ins_encode %{ 4849 __ smov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ B, $idx$$constant); 4850 %} 4851 ins_pipe(pipe_class_default); 4852 %} 4853 4854 instruct extractS_LT8(iRegINoSp dst, vReg src, immI idx) 4855 %{ 4856 predicate(UseSVE > 0 && n->in(2)->get_int() < 8); 4857 match(Set dst (ExtractS src idx)); 4858 ins_cost(INSN_COST); 4859 format %{ "smov $dst, H, $src, $idx\t# extract from vector(S)" %} 4860 ins_encode %{ 4861 __ smov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ H, $idx$$constant); 4862 %} 4863 ins_pipe(pipe_class_default); 4864 %} 4865 4866 instruct extractI_LT4(iRegINoSp dst, vReg src, immI idx) 4867 %{ 4868 predicate(UseSVE > 0 && n->in(2)->get_int() < 4); 4869 match(Set dst (ExtractI src idx)); 4870 ins_cost(INSN_COST); 4871 format %{ "smov $dst, S, $src, $idx\t# extract from vector(I)" %} 4872 ins_encode %{ 4873 __ smov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ S, $idx$$constant); 4874 %} 4875 ins_pipe(pipe_class_default); 4876 %} 4877 4878 instruct extractL_LT2(iRegLNoSp dst, vReg src, immI idx) 4879 %{ 4880 predicate(UseSVE > 0 && n->in(2)->get_int() < 2); 4881 match(Set dst (ExtractL src idx)); 4882 ins_cost(INSN_COST); 4883 format %{ "umov $dst, D, $src, $idx\t# extract from vector(L)" %} 4884 ins_encode %{ 4885 __ umov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ D, $idx$$constant); 4886 %} 4887 ins_pipe(pipe_class_default); 4888 %} 4889 4890 instruct extractF(vRegF dst, vReg src, immI idx) 4891 %{ 4892 predicate(UseSVE > 0); 4893 match(Set dst (ExtractF src idx)); 4894 ins_cost(2 * SVE_COST); 4895 format %{ "sve_extract_f $dst, S, $src, $idx\t# extract from vector(F)" %} 4896 ins_encode %{ 4897 if ((as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg)) && ($idx$$constant == 0)) { 4898 /* empty */ 4899 } else if ($idx$$constant == 0) { 4900 __ fmovs(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg)); 4901 } else if ($idx$$constant < 4) { 4902 __ ins(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($src$$reg), 0, (int)($idx$$constant)); 4903 } else { 4904 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg)); 4905 __ sve_ext(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), $idx$$constant << 2); 4906 } 4907 %} 4908 ins_pipe(pipe_slow); 4909 %} 4910 4911 instruct extractD(vRegD dst, vReg src, immI idx) 4912 %{ 4913 predicate(UseSVE > 0); 4914 match(Set dst (ExtractD src idx)); 4915 ins_cost(2 * SVE_COST); 4916 format %{ "sve_extract_d $dst, D, $src, $idx\t# extract from vector(D)" %} 4917 ins_encode %{ 4918 if ((as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg)) && ($idx$$constant == 0)) { 4919 /* empty */ 4920 } else if ($idx$$constant == 0) { 4921 __ fmovd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg)); 4922 } else if ($idx$$constant == 1) { 4923 __ ins(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), 0, 1); 4924 } else { 4925 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg)); 4926 __ sve_ext(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), $idx$$constant << 3); 4927 } 4928 %} 4929 ins_pipe(pipe_slow); 4930 %} 4931 4932 // ------------------------------- VectorTest ---------------------------------- 4933 4934 instruct vtest_alltrue(iRegINoSp dst, pRegGov src1, pRegGov src2, pReg ptmp, rFlagsReg cr) 4935 %{ 4936 predicate(UseSVE > 0 && 4937 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow); 4938 match(Set dst (VectorTest src1 src2)); 4939 effect(TEMP ptmp, KILL cr); 4940 ins_cost(SVE_COST); 4941 format %{ "sve_eors $ptmp, $src1, $src2\t# $src2 is all true mask\n" 4942 "csetw $dst, EQ\t# VectorTest (sve) - alltrue" %} 4943 ins_encode %{ 4944 __ sve_eors(as_PRegister($ptmp$$reg), ptrue, 4945 as_PRegister($src1$$reg), as_PRegister($src2$$reg)); 4946 __ csetw(as_Register($dst$$reg), Assembler::EQ); 4947 %} 4948 ins_pipe(pipe_slow); 4949 %} 4950 4951 instruct vtest_anytrue(iRegINoSp dst, pRegGov src1, pRegGov src2, rFlagsReg cr) 4952 %{ 4953 predicate(UseSVE > 0 && 4954 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne); 4955 match(Set dst (VectorTest src1 src2)); 4956 effect(KILL cr); 4957 ins_cost(SVE_COST); 4958 format %{ "sve_ptest $src1\n\t" 4959 "csetw $dst, NE\t# VectorTest (sve) - anytrue" %} 4960 ins_encode %{ 4961 // "src2" is not used for sve. 4962 __ sve_ptest(ptrue, as_PRegister($src1$$reg)); 4963 __ csetw(as_Register($dst$$reg), Assembler::NE); 4964 %} 4965 ins_pipe(pipe_slow); 4966 %} 4967 // ------------------------------ Vector insert --------------------------------- 4968 4969 instruct insertI_le128bits(vReg dst, vReg src, iRegIorL2I val, immI idx) %{ 4970 predicate(UseSVE > 0 && 4971 (Matcher::vector_length_in_bytes(n) == 8 || Matcher::vector_length_in_bytes(n) == 16) && 4972 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE || 4973 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT || 4974 n->bottom_type()->is_vect()->element_basic_type() == T_INT)); 4975 match(Set dst (VectorInsert (Binary src val) idx)); 4976 ins_cost(2 * INSN_COST); 4977 format %{ "orr $dst, T8/16B, $src, $src\n\t" 4978 "mov $dst, B/H/S, $idx, $val\t# insertI into vector(64/128bits)" %} 4979 ins_encode %{ 4980 BasicType bt = Matcher::vector_element_basic_type(this); 4981 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) { 4982 __ orr(as_FloatRegister($dst$$reg), Matcher::vector_length_in_bytes(this) == 8 ? __ T8B : __ T16B, 4983 as_FloatRegister($src$$reg), as_FloatRegister($src$$reg)); 4984 } 4985 __ mov(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(Matcher::vector_element_basic_type(this)), 4986 $idx$$constant, $val$$Register); 4987 %} 4988 ins_pipe(pipe_slow); 4989 %} 4990 4991 instruct insertI_small_index(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg vtmp, pRegGov pgtmp, rFlagsReg cr) %{ 4992 predicate(UseSVE > 0 && n->in(2)->get_int() < 32 && 4993 Matcher::vector_length_in_bytes(n) > 16 && 4994 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE || 4995 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT || 4996 n->bottom_type()->is_vect()->element_basic_type() == T_INT)); 4997 match(Set dst (VectorInsert (Binary src val) idx)); 4998 effect(TEMP vtmp, TEMP pgtmp, KILL cr); 4999 ins_cost(4 * SVE_COST); 5000 format %{ "sve_index $vtmp, -16, 1\t# (B/H/S)\n\t" 5001 "sve_cmpeq $pgtmp, $vtmp, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t" 5002 "sve_orr $dst, $src, $src\n\t" 5003 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %} 5004 ins_encode %{ 5005 BasicType bt = Matcher::vector_element_basic_type(this, $src); 5006 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 5007 __ block_comment("insert into vector (B/H/S) {"); 5008 __ sve_index(as_FloatRegister($vtmp$$reg), size, -16, 1); 5009 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue, 5010 as_FloatRegister($vtmp$$reg), (int)($idx$$constant) - 16); 5011 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) { 5012 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg)); 5013 } 5014 __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), $val$$Register); 5015 __ block_comment("} insert into vector (B/H/S)"); 5016 %} 5017 ins_pipe(pipe_slow); 5018 %} 5019 5020 instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg vtmp1, vReg vtmp2, pRegGov pgtmp, rFlagsReg cr) %{ 5021 predicate(UseSVE > 0 && n->in(2)->get_int() >= 32 && 5022 Matcher::vector_length_in_bytes(n) > 16 && 5023 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE || 5024 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT || 5025 n->bottom_type()->is_vect()->element_basic_type() == T_INT)); 5026 match(Set dst (VectorInsert (Binary src val) idx)); 5027 effect(TEMP vtmp1, TEMP vtmp2, TEMP pgtmp, KILL cr); 5028 ins_cost(5 * SVE_COST); 5029 format %{ "sve_index $vtmp1, 0, 1\t# (B/H/S)\n\t" 5030 "sve_dup $vtmp2, $idx\t# (B/H/S)\n\t" 5031 "sve_cmpeq $pgtmp, $vtmp1, $vtmp2\n\t" 5032 "sve_orr $dst, $src, $src\n\t" 5033 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %} 5034 ins_encode %{ 5035 BasicType bt = Matcher::vector_element_basic_type(this, $src); 5036 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 5037 __ block_comment("insert into vector (B/H/S) {"); 5038 __ sve_index(as_FloatRegister($vtmp1$$reg), size, 0, 1); 5039 __ sve_dup(as_FloatRegister($vtmp2$$reg), size, (int)($idx$$constant)); 5040 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue, 5041 as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg)); 5042 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) { 5043 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg)); 5044 } 5045 __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), $val$$Register); 5046 __ block_comment("} insert into vector (B/H/S)"); 5047 %} 5048 ins_pipe(pipe_slow); 5049 %} 5050 5051 instruct insertL_128bits(vReg dst, vReg src, iRegL val, immI idx) %{ 5052 predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) == 16 && 5053 n->bottom_type()->is_vect()->element_basic_type() == T_LONG); 5054 match(Set dst (VectorInsert (Binary src val) idx)); 5055 ins_cost(2 * INSN_COST); 5056 format %{ "orr $dst, T16B, $src, $src\n\t" 5057 "mov $dst, D, $idx, $val\t# insertL into vector(128bits)" %} 5058 ins_encode %{ 5059 BasicType bt = Matcher::vector_element_basic_type(this); 5060 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) { 5061 __ orr(as_FloatRegister($dst$$reg), __ T16B, 5062 as_FloatRegister($src$$reg), as_FloatRegister($src$$reg)); 5063 } 5064 __ mov(as_FloatRegister($dst$$reg), __ D, $idx$$constant, $val$$Register); 5065 %} 5066 ins_pipe(pipe_slow); 5067 %} 5068 5069 instruct insertL(vReg dst, vReg src, iRegL val, immI idx, vReg vtmp, pRegGov pgtmp, rFlagsReg cr) %{ 5070 predicate(UseSVE > 0 && 5071 Matcher::vector_length_in_bytes(n) > 16 && 5072 n->bottom_type()->is_vect()->element_basic_type() == T_LONG); 5073 match(Set dst (VectorInsert (Binary src val) idx)); 5074 effect(TEMP vtmp, TEMP pgtmp, KILL cr); 5075 ins_cost(4 * SVE_COST); 5076 format %{ "sve_index $vtmp, D, -16, 1\n\t" 5077 "sve_cmpeq $pgtmp, $vtmp, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t" 5078 "sve_orr $dst, $src, $src\n\t" 5079 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (L)" %} 5080 ins_encode %{ 5081 __ block_comment("insert into vector (L) {"); 5082 __ sve_index(as_FloatRegister($vtmp$$reg), __ D, -16, 1); 5083 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue, 5084 as_FloatRegister($vtmp$$reg), (int)($idx$$constant) - 16); 5085 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) { 5086 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg)); 5087 } 5088 __ sve_cpy(as_FloatRegister($dst$$reg), __ D, 5089 as_PRegister($pgtmp$$reg), $val$$Register); 5090 __ block_comment("} insert into vector (L)"); 5091 %} 5092 ins_pipe(pipe_slow); 5093 %} 5094 5095 instruct insertF_le128bits(vReg dst, vReg src, vRegF val, immI idx) %{ 5096 predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT && 5097 (Matcher::vector_length_in_bytes(n) == 8 || Matcher::vector_length_in_bytes(n) == 16)); 5098 match(Set dst (VectorInsert (Binary src val) idx)); 5099 ins_cost(2 * INSN_COST); 5100 effect(TEMP_DEF dst); 5101 format %{ "orr $dst, T8/16B, $src, $src\n\t" 5102 "ins $dst, S, $val, $idx, 0\t# insertF into vector(64/128bits)" %} 5103 ins_encode %{ 5104 __ orr(as_FloatRegister($dst$$reg), Matcher::vector_length_in_bytes(this) == 8 ? __ T8B : __ T16B, 5105 as_FloatRegister($src$$reg), as_FloatRegister($src$$reg)); 5106 __ ins(as_FloatRegister($dst$$reg), __ S, 5107 as_FloatRegister($val$$reg), $idx$$constant, 0); 5108 %} 5109 ins_pipe(pipe_slow); 5110 %} 5111 5112 instruct insertF_small_index(vReg dst, vReg src, vRegF val, immI idx, pRegGov pgtmp, rFlagsReg cr) %{ 5113 predicate(UseSVE > 0 && n->in(2)->get_int() < 32 && 5114 Matcher::vector_length_in_bytes(n) > 16 && 5115 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT); 5116 match(Set dst (VectorInsert (Binary src val) idx)); 5117 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr); 5118 ins_cost(4 * SVE_COST); 5119 format %{ "sve_index $dst, S, -16, 1\n\t" 5120 "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t" 5121 "sve_orr $dst, $src, $src\n\t" 5122 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %} 5123 ins_encode %{ 5124 __ block_comment("insert into vector (F) {"); 5125 __ sve_index(as_FloatRegister($dst$$reg), __ S, -16, 1); 5126 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue, 5127 as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16); 5128 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg)); 5129 __ sve_cpy(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg)); 5130 __ block_comment("} insert into vector (F)"); 5131 %} 5132 ins_pipe(pipe_slow); 5133 %} 5134 5135 instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr) %{ 5136 predicate(UseSVE > 0 && n->in(2)->get_int() >= 32 && 5137 Matcher::vector_length_in_bytes(n) > 16 && 5138 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT); 5139 match(Set dst (VectorInsert (Binary src val) idx)); 5140 effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr); 5141 ins_cost(5 * SVE_COST); 5142 format %{ "sve_index $tmp1, S, 0, 1\n\t" 5143 "sve_dup $dst, S, $idx\n\t" 5144 "sve_cmpeq $pgtmp, $tmp1, $dst\n\t" 5145 "sve_orr $dst, $src, $src\n\t" 5146 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %} 5147 ins_encode %{ 5148 __ block_comment("insert into vector (F) {"); 5149 __ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1); 5150 __ sve_dup(as_FloatRegister($dst$$reg), __ S, (int)($idx$$constant)); 5151 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue, 5152 as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg)); 5153 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), 5154 as_FloatRegister($src$$reg)); 5155 __ sve_cpy(as_FloatRegister($dst$$reg), __ S, 5156 as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg)); 5157 __ block_comment("} insert into vector (F)"); 5158 %} 5159 ins_pipe(pipe_slow); 5160 %} 5161 5162 instruct insertD_128bits(vReg dst, vReg src, vRegD val, immI idx) %{ 5163 predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) == 16 && 5164 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE); 5165 match(Set dst (VectorInsert (Binary src val) idx)); 5166 ins_cost(2 * INSN_COST); 5167 effect(TEMP_DEF dst); 5168 format %{ "orr $dst, T16B, $src, $src\n\t" 5169 "ins $dst, D, $val, $idx, 0\t# insertD into vector(128bits)" %} 5170 ins_encode %{ 5171 __ orr(as_FloatRegister($dst$$reg), __ T16B, 5172 as_FloatRegister($src$$reg), as_FloatRegister($src$$reg)); 5173 __ ins(as_FloatRegister($dst$$reg), __ D, 5174 as_FloatRegister($val$$reg), $idx$$constant, 0); 5175 %} 5176 ins_pipe(pipe_slow); 5177 %} 5178 5179 instruct insertD(vReg dst, vReg src, vRegD val, immI idx, pRegGov pgtmp, rFlagsReg cr) %{ 5180 predicate(UseSVE > 0 && 5181 Matcher::vector_length_in_bytes(n) > 16 && 5182 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE); 5183 match(Set dst (VectorInsert (Binary src val) idx)); 5184 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr); 5185 ins_cost(4 * SVE_COST); 5186 format %{ "sve_index $dst, D, -16, 1\n\t" 5187 "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t" 5188 "sve_orr $dst, $src, $src\n\t" 5189 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (D)" %} 5190 ins_encode %{ 5191 __ block_comment("insert into vector (D) {"); 5192 __ sve_index(as_FloatRegister($dst$$reg), __ D, -16, 1); 5193 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue, 5194 as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16); 5195 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), 5196 as_FloatRegister($src$$reg)); 5197 __ sve_cpy(as_FloatRegister($dst$$reg), __ D, 5198 as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg)); 5199 __ block_comment("} insert into vector (D)"); 5200 %} 5201 ins_pipe(pipe_slow); 5202 %} 5203 5204 // ------------------------------ Vector shuffle ------------------------------- 5205 5206 instruct loadshuffle(vReg dst, vReg src) %{ 5207 predicate(UseSVE > 0); 5208 match(Set dst (VectorLoadShuffle src)); 5209 ins_cost(SVE_COST); 5210 format %{ "sve_loadshuffle $dst, $src\t# vector load shuffle (B/H/S/D)" %} 5211 ins_encode %{ 5212 BasicType bt = Matcher::vector_element_basic_type(this); 5213 if (bt == T_BYTE) { 5214 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) { 5215 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), 5216 as_FloatRegister($src$$reg)); 5217 } 5218 } else { 5219 __ sve_vector_extend(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt), 5220 as_FloatRegister($src$$reg), __ B); 5221 } 5222 %} 5223 ins_pipe(pipe_slow); 5224 %} 5225 5226 // ------------------------------ Vector rearrange ------------------------------- 5227 5228 instruct rearrange(vReg dst, vReg src, vReg shuffle) 5229 %{ 5230 predicate(UseSVE > 0); 5231 match(Set dst (VectorRearrange src shuffle)); 5232 ins_cost(SVE_COST); 5233 format %{ "sve_tbl $dst, $src, $shuffle\t# vector rearrange" %} 5234 ins_encode %{ 5235 BasicType bt = Matcher::vector_element_basic_type(this, $src); 5236 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 5237 __ sve_tbl(as_FloatRegister($dst$$reg), size, 5238 as_FloatRegister($src$$reg), as_FloatRegister($shuffle$$reg)); 5239 %} 5240 ins_pipe(pipe_slow); 5241 %} 5242 5243 // ------------------------------ Vector Load Gather --------------------------------- 5244 5245 instruct gatherI(vReg dst, indirect mem, vReg idx) %{ 5246 predicate(UseSVE > 0 && 5247 n->as_LoadVectorGather()->memory_size() == MaxVectorSize && 5248 (n->bottom_type()->is_vect()->element_basic_type() == T_INT || 5249 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT)); 5250 match(Set dst (LoadVectorGather mem idx)); 5251 ins_cost(SVE_COST); 5252 format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (S)" %} 5253 ins_encode %{ 5254 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), ptrue, 5255 as_Register($mem$$base), as_FloatRegister($idx$$reg)); 5256 %} 5257 ins_pipe(pipe_slow); 5258 %} 5259 5260 instruct gatherL(vReg dst, indirect mem, vReg idx) %{ 5261 predicate(UseSVE > 0 && 5262 n->as_LoadVectorGather()->memory_size() == MaxVectorSize && 5263 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG || 5264 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE)); 5265 match(Set dst (LoadVectorGather mem idx)); 5266 ins_cost(2 * SVE_COST); 5267 format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (D)" %} 5268 ins_encode %{ 5269 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg)); 5270 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), ptrue, as_Register($mem$$base), 5271 as_FloatRegister($idx$$reg)); 5272 %} 5273 ins_pipe(pipe_slow); 5274 %} 5275 5276 // ------------------------------ Vector Load Gather Partial------------------------------- 5277 5278 instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{ 5279 predicate(UseSVE > 0 && 5280 n->as_LoadVectorGather()->memory_size() < MaxVectorSize && 5281 (n->bottom_type()->is_vect()->element_basic_type() == T_INT || 5282 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT)); 5283 match(Set dst (LoadVectorGather mem idx)); 5284 effect(TEMP ptmp, KILL cr); 5285 ins_cost(2 * SVE_COST + INSN_COST); 5286 format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (S)" %} 5287 ins_encode %{ 5288 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this)); 5289 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg), 5290 as_Register($mem$$base), as_FloatRegister($idx$$reg)); 5291 %} 5292 ins_pipe(pipe_slow); 5293 %} 5294 5295 instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{ 5296 predicate(UseSVE > 0 && 5297 n->as_LoadVectorGather()->memory_size() < MaxVectorSize && 5298 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG || 5299 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE)); 5300 match(Set dst (LoadVectorGather mem idx)); 5301 effect(TEMP ptmp, KILL cr); 5302 ins_cost(3 * SVE_COST + INSN_COST); 5303 format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (D)" %} 5304 ins_encode %{ 5305 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this)); 5306 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg)); 5307 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg), 5308 as_Register($mem$$base), as_FloatRegister($idx$$reg)); 5309 %} 5310 ins_pipe(pipe_slow); 5311 %} 5312 5313 // ------------------------------ Vector Load Gather Predicated ------------------------------- 5314 5315 instruct gatherI_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{ 5316 predicate(UseSVE > 0 && 5317 (n->bottom_type()->is_vect()->element_basic_type() == T_INT || 5318 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT)); 5319 match(Set dst (LoadVectorGatherMasked mem (Binary idx pg))); 5320 ins_cost(SVE_COST); 5321 format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (S)" %} 5322 ins_encode %{ 5323 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg), 5324 as_Register($mem$$base), as_FloatRegister($idx$$reg)); 5325 %} 5326 ins_pipe(pipe_slow); 5327 %} 5328 5329 instruct gatherL_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{ 5330 predicate(UseSVE > 0 && 5331 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG || 5332 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE)); 5333 match(Set dst (LoadVectorGatherMasked mem (Binary idx pg))); 5334 ins_cost(2 * SVE_COST); 5335 format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (D)" %} 5336 ins_encode %{ 5337 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg)); 5338 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg), 5339 as_Register($mem$$base), as_FloatRegister($idx$$reg)); 5340 %} 5341 ins_pipe(pipe_slow); 5342 %} 5343 5344 // ------------------------------ Vector Store Scatter ------------------------------- 5345 5346 instruct scatterI(indirect mem, vReg src, vReg idx) %{ 5347 predicate(UseSVE > 0 && 5348 n->as_StoreVectorScatter()->memory_size() == MaxVectorSize && 5349 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT || 5350 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT)); 5351 match(Set mem (StoreVectorScatter mem (Binary src idx))); 5352 ins_cost(SVE_COST); 5353 format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (S)" %} 5354 ins_encode %{ 5355 __ sve_st1w_scatter(as_FloatRegister($src$$reg), ptrue, 5356 as_Register($mem$$base), as_FloatRegister($idx$$reg)); 5357 %} 5358 ins_pipe(pipe_slow); 5359 %} 5360 5361 instruct scatterL(indirect mem, vReg src, vReg idx) %{ 5362 predicate(UseSVE > 0 && 5363 n->as_StoreVectorScatter()->memory_size() == MaxVectorSize && 5364 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG || 5365 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE)); 5366 match(Set mem (StoreVectorScatter mem (Binary src idx))); 5367 ins_cost(2 * SVE_COST); 5368 format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (D)" %} 5369 ins_encode %{ 5370 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg)); 5371 __ sve_st1d_scatter(as_FloatRegister($src$$reg), ptrue, 5372 as_Register($mem$$base), as_FloatRegister($idx$$reg)); 5373 %} 5374 ins_pipe(pipe_slow); 5375 %} 5376 5377 // ------------------------------ Vector Store Scatter Partial ------------------------------- 5378 5379 instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{ 5380 predicate(UseSVE > 0 && 5381 n->as_StoreVectorScatter()->memory_size() < MaxVectorSize && 5382 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT || 5383 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT)); 5384 match(Set mem (StoreVectorScatter mem (Binary src idx))); 5385 effect(TEMP ptmp, KILL cr); 5386 ins_cost(2 * SVE_COST + INSN_COST); 5387 format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (S)" %} 5388 ins_encode %{ 5389 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src)); 5390 __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg), 5391 as_Register($mem$$base), as_FloatRegister($idx$$reg)); 5392 %} 5393 ins_pipe(pipe_slow); 5394 %} 5395 5396 instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{ 5397 predicate(UseSVE > 0 && 5398 n->as_StoreVectorScatter()->memory_size() < MaxVectorSize && 5399 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG || 5400 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE)); 5401 match(Set mem (StoreVectorScatter mem (Binary src idx))); 5402 effect(TEMP ptmp, KILL cr); 5403 ins_cost(3 * SVE_COST + INSN_COST); 5404 format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (D)" %} 5405 ins_encode %{ 5406 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src)); 5407 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg)); 5408 __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg), 5409 as_Register($mem$$base), as_FloatRegister($idx$$reg)); 5410 %} 5411 ins_pipe(pipe_slow); 5412 %} 5413 5414 // ------------------------------ Vector Store Scatter Predicated ------------------------------- 5415 5416 instruct scatterI_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{ 5417 predicate(UseSVE > 0 && 5418 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT || 5419 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT)); 5420 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg)))); 5421 ins_cost(SVE_COST); 5422 format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicate (S)" %} 5423 ins_encode %{ 5424 __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg), 5425 as_Register($mem$$base), as_FloatRegister($idx$$reg)); 5426 %} 5427 ins_pipe(pipe_slow); 5428 %} 5429 5430 instruct scatterL_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{ 5431 predicate(UseSVE > 0 && 5432 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG || 5433 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE)); 5434 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg)))); 5435 ins_cost(2 * SVE_COST); 5436 format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated (D)" %} 5437 ins_encode %{ 5438 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg)); 5439 __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg), 5440 as_Register($mem$$base), as_FloatRegister($idx$$reg)); 5441 %} 5442 ins_pipe(pipe_slow); 5443 %} 5444 5445 // ------------------------------ Vector Load Const ------------------------------- 5446 5447 instruct loadconB(vReg dst, immI0 src) %{ 5448 predicate(UseSVE > 0 && 5449 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE); 5450 match(Set dst (VectorLoadConst src)); 5451 ins_cost(SVE_COST); 5452 format %{ "sve_index $dst, 0, 1\t# generate iota indices" %} 5453 ins_encode %{ 5454 __ sve_index(as_FloatRegister($dst$$reg), __ B, 0, 1); 5455 %} 5456 ins_pipe(pipe_slow); 5457 %} 5458 5459 // Intrisics for String.indexOf(char) 5460 5461 5462 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch, 5463 iRegI_R0 result, vReg ztmp1, vReg ztmp2, 5464 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) 5465 %{ 5466 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch)); 5467 predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L)); 5468 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr); 5469 5470 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %} 5471 5472 ins_encode %{ 5473 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register, 5474 as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg), 5475 as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), true /* isL */); 5476 %} 5477 ins_pipe(pipe_class_memory); 5478 %} 5479 5480 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch, 5481 iRegI_R0 result, vReg ztmp1, vReg ztmp2, 5482 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) 5483 %{ 5484 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch)); 5485 predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U)); 5486 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr); 5487 5488 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %} 5489 5490 ins_encode %{ 5491 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register, 5492 as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg), 5493 as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), false /* isL */); 5494 %} 5495 ins_pipe(pipe_class_memory); 5496 %} 5497 5498 // ---------------------------- Vector mask reductions --------------------------- 5499 instruct vmask_truecount(iRegINoSp dst, pReg src) %{ 5500 predicate(UseSVE > 0); 5501 match(Set dst (VectorMaskTrueCount src)); 5502 ins_cost(SVE_COST); 5503 format %{ "vmask_truecount $dst, $src\t# vector mask truecount (sve)" %} 5504 ins_encode %{ 5505 BasicType bt = Matcher::vector_element_basic_type(this, $src); 5506 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 5507 __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($src$$reg)); 5508 %} 5509 ins_pipe(pipe_slow); 5510 %} 5511 5512 // Return the index of the first mask lane that is set, or vector length if none of 5513 // them are set. 5514 instruct vmask_firsttrue(iRegINoSp dst, pReg src, pReg ptmp) %{ 5515 predicate(UseSVE > 0); 5516 match(Set dst (VectorMaskFirstTrue src)); 5517 effect(TEMP ptmp); 5518 ins_cost(3 * SVE_COST); 5519 format %{ "vmask_firsttrue $dst, $src\t# vector mask firsttrue (sve)" %} 5520 ins_encode %{ 5521 uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src); 5522 BasicType bt = Matcher::vector_element_basic_type(this, $src); 5523 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 5524 // When the input predicate is all-false, the result should be the vector length 5525 // instead of max vector register size. 5526 if (length_in_bytes == MaxVectorSize) { 5527 __ sve_brkb(as_PRegister($ptmp$$reg), ptrue, as_PRegister($src$$reg), false); 5528 } else { 5529 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src)); 5530 __ sve_brkb(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg), as_PRegister($src$$reg), false); 5531 } 5532 __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($ptmp$$reg)); 5533 %} 5534 ins_pipe(pipe_slow); 5535 %} 5536 5537 instruct vmask_lasttrue(iRegINoSp dst, pReg src, pReg ptmp) %{ 5538 predicate(UseSVE > 0); 5539 match(Set dst (VectorMaskLastTrue src)); 5540 effect(TEMP ptmp); 5541 ins_cost(3 * SVE_COST); 5542 format %{ "vmask_lasttrue $dst, $src\t# vector mask lasttrue (sve)" %} 5543 ins_encode %{ 5544 BasicType bt = Matcher::vector_element_basic_type(this, $src); 5545 __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($src$$reg), as_PRegister($ptmp$$reg)); 5546 %} 5547 ins_pipe(pipe_slow); 5548 %} 5549 5550 instruct vmask_tolong(iRegLNoSp dst, pReg src, vReg vtmp1, vReg vtmp2) %{ 5551 predicate(UseSVE > 0 && 5552 n->in(1)->bottom_type()->is_vect()->length() <= 64); 5553 match(Set dst (VectorMaskToLong src)); 5554 effect(TEMP vtmp1, TEMP vtmp2); 5555 ins_cost(13 * SVE_COST); 5556 format %{ "vmask_tolong $dst, $src\t# vector mask tolong (sve)" %} 5557 ins_encode %{ 5558 __ sve_vmask_tolong(as_Register($dst$$reg), as_PRegister($src$$reg), 5559 Matcher::vector_element_basic_type(this, $src), 5560 Matcher::vector_length(this, $src), 5561 as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg)); 5562 %} 5563 ins_pipe(pipe_slow); 5564 %} 5565 5566 // ---------------------------- Vector mask generation --------------------------- 5567 // The rules below set predicate registers. They can guarantee the high bits of dst 5568 // are cleared with zero when the vector length is less than the full size of 5569 // hardware vector register width. 5570 5571 5572 // maskAll (full or partial predicate size) 5573 5574 instruct vmaskAll_immI(pRegGov dst, immI src) %{ 5575 predicate(UseSVE > 0); 5576 match(Set dst (MaskAll src)); 5577 ins_cost(SVE_COST); 5578 format %{ "sve_ptrue_lanecnt/sve_pfalse $dst\t# mask all (sve) (B/H/S)" %} 5579 ins_encode %{ 5580 int con = (int)$src$$constant; 5581 if (con == 0) { 5582 __ sve_pfalse(as_PRegister($dst$$reg)); 5583 } else { 5584 assert(con == -1, "invalid constant value for mask"); 5585 BasicType bt = Matcher::vector_element_basic_type(this); 5586 __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt), 5587 Matcher::vector_length(this)); 5588 } 5589 %} 5590 ins_pipe(pipe_slow); 5591 %} 5592 5593 instruct vmaskAllI(pRegGov dst, iRegIorL2I src, vReg tmp, rFlagsReg cr) %{ 5594 predicate(UseSVE > 0); 5595 match(Set dst (MaskAll src)); 5596 effect(TEMP tmp, KILL cr); 5597 ins_cost(3 * SVE_COST); 5598 format %{ "sve_dup $tmp, $src\n\t" 5599 "sve_ptrue $dst, vector_length\n\t" 5600 "sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) (B/H/S)" %} 5601 ins_encode %{ 5602 BasicType bt = Matcher::vector_element_basic_type(this); 5603 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 5604 uint length_in_bytes = Matcher::vector_length_in_bytes(this); 5605 __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg)); 5606 if (length_in_bytes < MaxVectorSize) { 5607 __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), size, Matcher::vector_length(this)); 5608 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, 5609 as_PRegister($dst$$reg), as_FloatRegister($tmp$$reg), 0); 5610 } else { 5611 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0); 5612 } 5613 %} 5614 ins_pipe(pipe_slow); 5615 %} 5616 5617 instruct vmaskAll_immL(pRegGov dst, immL src) %{ 5618 predicate(UseSVE > 0); 5619 match(Set dst (MaskAll src)); 5620 ins_cost(SVE_COST); 5621 format %{ "sve_ptrue_lanecnt/sve_pfalse $dst\t# mask all (sve) (D)" %} 5622 ins_encode %{ 5623 long con = (long)$src$$constant; 5624 if (con == 0) { 5625 __ sve_pfalse(as_PRegister($dst$$reg)); 5626 } else { 5627 assert(con == -1, "invalid constant value for mask"); 5628 BasicType bt = Matcher::vector_element_basic_type(this); 5629 __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt), 5630 Matcher::vector_length(this)); 5631 } 5632 %} 5633 ins_pipe(pipe_slow); 5634 %} 5635 5636 instruct vmaskAllL(pRegGov dst, iRegL src, vReg tmp, rFlagsReg cr) %{ 5637 predicate(UseSVE > 0); 5638 match(Set dst (MaskAll src)); 5639 effect(TEMP tmp, KILL cr); 5640 ins_cost(3 * SVE_COST); 5641 format %{ "sve_dup $tmp, $src\n\t" 5642 "sve_ptrue $dst, vector_length\n\t" 5643 "sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) (D)" %} 5644 ins_encode %{ 5645 BasicType bt = Matcher::vector_element_basic_type(this); 5646 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 5647 uint length_in_bytes = Matcher::vector_length_in_bytes(this); 5648 __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg)); 5649 if (length_in_bytes < MaxVectorSize) { 5650 __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), size, Matcher::vector_length(this)); 5651 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, 5652 as_PRegister($dst$$reg), as_FloatRegister($tmp$$reg), 0); 5653 } else { 5654 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0); 5655 } 5656 %} 5657 ins_pipe(pipe_slow); 5658 %} 5659 5660 // vector mask compare 5661 5662 instruct vmaskcmp(pRegGov dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %{ 5663 predicate(UseSVE > 0); 5664 match(Set dst (VectorMaskCmp (Binary src1 src2) cond)); 5665 effect(KILL cr); 5666 ins_cost(2 * SVE_COST); 5667 format %{ "sve_cmp $dst, $src1, $src2\t# vector mask cmp (sve)" %} 5668 ins_encode %{ 5669 uint length_in_bytes = Matcher::vector_length_in_bytes(this); 5670 BasicType bt = Matcher::vector_element_basic_type(this); 5671 if (length_in_bytes == MaxVectorSize) { 5672 __ sve_compare(as_PRegister($dst$$reg), bt, ptrue, as_FloatRegister($src1$$reg), 5673 as_FloatRegister($src2$$reg), (int)$cond$$constant); 5674 } else { 5675 __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt), 5676 Matcher::vector_length(this)); 5677 __ sve_compare(as_PRegister($dst$$reg), bt, as_PRegister($dst$$reg), as_FloatRegister($src1$$reg), 5678 as_FloatRegister($src2$$reg), (int)$cond$$constant); 5679 } 5680 %} 5681 ins_pipe(pipe_slow); 5682 %} 5683 5684 instruct vmaskcmp_masked(pRegGov dst, vReg src1, vReg src2, immI cond, pRegGov pg, rFlagsReg cr) %{ 5685 predicate(UseSVE > 0); 5686 match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond pg))); 5687 effect(KILL cr); 5688 ins_cost(SVE_COST); 5689 format %{ "sve_cmp $dst, $pg, $src1, $src2\t# vector mask cmp (sve)" %} 5690 ins_encode %{ 5691 BasicType bt = Matcher::vector_element_basic_type(this); 5692 __ sve_compare(as_PRegister($dst$$reg), bt, as_PRegister($pg$$reg), as_FloatRegister($src1$$reg), 5693 as_FloatRegister($src2$$reg), (int)$cond$$constant); 5694 %} 5695 ins_pipe(pipe_slow); 5696 %} 5697 5698 // vector load mask 5699 5700 instruct vloadmaskB(pRegGov dst, vReg src, rFlagsReg cr) %{ 5701 predicate(UseSVE > 0 && 5702 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE); 5703 match(Set dst (VectorLoadMask src)); 5704 effect(KILL cr); 5705 ins_cost(SVE_COST); 5706 format %{ "vloadmaskB $dst, $src\t# vector load mask (sve) (B)" %} 5707 ins_encode %{ 5708 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ B, 5709 ptrue, as_FloatRegister($src$$reg), 0); 5710 %} 5711 ins_pipe(pipe_slow); 5712 %} 5713 5714 instruct vloadmask_extend(pRegGov dst, vReg src, vReg tmp, rFlagsReg cr) %{ 5715 predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() != T_BYTE); 5716 match(Set dst (VectorLoadMask src)); 5717 effect(TEMP tmp, KILL cr); 5718 ins_cost(3 * SVE_COST); 5719 format %{ "vloadmask $dst, $src\t# vector load mask (sve) (H/S/D)" %} 5720 ins_encode %{ 5721 BasicType bt = Matcher::vector_element_basic_type(this); 5722 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 5723 __ sve_vector_extend(as_FloatRegister($tmp$$reg), size, as_FloatRegister($src$$reg), __ B); 5724 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0); 5725 %} 5726 ins_pipe(pipe_slow); 5727 %} 5728 5729 // ---------------------------- Compress/Expand Operations --------------------------- 5730 5731 instruct mcompress(pReg dst, pReg pg, rFlagsReg cr) %{ 5732 predicate(UseSVE > 0); 5733 match(Set dst (CompressM pg)); 5734 effect(KILL cr); 5735 ins_cost(2 * SVE_COST); 5736 format %{ "sve_cntp rscratch1, $pg\n\t" 5737 "sve_whilelo $dst, zr, rscratch1\t# mask compress (B/H/S/D)" %} 5738 ins_encode %{ 5739 BasicType bt = Matcher::vector_element_basic_type(this); 5740 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 5741 __ sve_cntp(rscratch1, size, ptrue, as_PRegister($pg$$reg)); 5742 __ sve_whilelo(as_PRegister($dst$$reg), size, zr, rscratch1); 5743 %} 5744 ins_pipe(pipe_slow); 5745 %} 5746 5747 instruct vcompress(vReg dst, vReg src, pRegGov pg) %{ 5748 predicate(UseSVE > 0 && 5749 (n->bottom_type()->is_vect()->element_basic_type() == T_INT || 5750 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT || 5751 n->bottom_type()->is_vect()->element_basic_type() == T_LONG || 5752 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE)); 5753 match(Set dst (CompressV src pg)); 5754 ins_cost(SVE_COST); 5755 format %{ "sve_compact $dst, $src, $pg\t# vector compress (S/D)" %} 5756 ins_encode %{ 5757 BasicType bt = Matcher::vector_element_basic_type(this); 5758 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 5759 __ sve_compact(as_FloatRegister($dst$$reg), size, as_FloatRegister($src$$reg), as_PRegister($pg$$reg)); 5760 %} 5761 ins_pipe(pipe_slow); 5762 %} 5763 5764 instruct vcompressB(vReg dst, vReg src, pReg pg, vReg vtmp1, vReg vtmp2, vReg vtmp3, vReg vtmp4, 5765 pReg ptmp, pRegGov pgtmp) %{ 5766 predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE); 5767 effect(TEMP_DEF dst, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP ptmp, TEMP pgtmp); 5768 match(Set dst (CompressV src pg)); 5769 ins_cost(13 * SVE_COST); 5770 format %{ "sve_compact $dst, $src, $pg\t# vector compress (B)" %} 5771 ins_encode %{ 5772 __ sve_compress_byte(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_PRegister($pg$$reg), 5773 as_FloatRegister($vtmp1$$reg),as_FloatRegister($vtmp2$$reg), 5774 as_FloatRegister($vtmp3$$reg),as_FloatRegister($vtmp4$$reg), 5775 as_PRegister($ptmp$$reg), as_PRegister($pgtmp$$reg)); 5776 %} 5777 ins_pipe(pipe_slow); 5778 %} 5779 5780 instruct vcompressS(vReg dst, vReg src, pReg pg, vReg vtmp1, vReg vtmp2, pRegGov pgtmp) %{ 5781 predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT); 5782 effect(TEMP_DEF dst, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp); 5783 match(Set dst (CompressV src pg)); 5784 ins_cost(38 * SVE_COST); 5785 format %{ "sve_compact $dst, $src, $pg\t# vector compress (H)" %} 5786 ins_encode %{ 5787 __ sve_compress_short(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_PRegister($pg$$reg), 5788 as_FloatRegister($vtmp1$$reg),as_FloatRegister($vtmp2$$reg), as_PRegister($pgtmp$$reg)); 5789 %} 5790 ins_pipe(pipe_slow); 5791 %} 5792 5793 instruct vexpand(vReg dst, vReg src, pRegGov pg) %{ 5794 match(Set dst (ExpandV src pg)); 5795 effect(TEMP_DEF dst); 5796 ins_cost(4 * SVE_COST); 5797 format %{ "sve_dup $dst, S/D, 0\n\t" 5798 "sve_histcnt $dst, S/D, $pg, $dst, $dst\n\t" 5799 "sve_sub $dst, S/D, 1\n\t" 5800 "sve_tbl $dst, S/D, $src, $dst\t# vector expand (S/D)" %} 5801 ins_encode %{ 5802 // Example input: src = 1 2 3 4 5 6 7 8 5803 // pg = 1 0 0 1 1 0 1 1 5804 // Expected result: dst = 4 0 0 5 6 0 7 8 5805 5806 // The basic idea is to use TBL which can shuffle the elements in the given 5807 // vector flexibly. HISTCNT + SUB is used to generate the second source input 5808 // for TBL whose value is used to select the indexed element from src vector. 5809 5810 BasicType bt = Matcher::vector_element_basic_type(this); 5811 assert(UseSVE == 2 && !is_subword_type(bt), "unsupported"); 5812 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 5813 // dst = 0 0 0 0 0 0 0 0 5814 __ sve_dup(as_FloatRegister($dst$$reg), size, 0); 5815 // dst = 5 0 0 4 3 0 2 1 5816 __ sve_histcnt(as_FloatRegister($dst$$reg), size, as_PRegister($pg$$reg), 5817 as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg)); 5818 // dst = 4 -1 -1 3 2 -1 1 0 5819 __ sve_sub(as_FloatRegister($dst$$reg), size, 1); 5820 // dst = 4 0 0 5 6 0 7 8 5821 __ sve_tbl(as_FloatRegister($dst$$reg), size, as_FloatRegister($src$$reg), 5822 as_FloatRegister($dst$$reg)); 5823 %} 5824 ins_pipe(pipe_slow); 5825 %} 5826 5827 instruct vmask_gen(pRegGov pg, iRegL len, rFlagsReg cr) %{ 5828 predicate(UseSVE > 0); 5829 match(Set pg (VectorMaskGen len)); 5830 effect(KILL cr); 5831 ins_cost(SVE_COST); 5832 format %{ "sve_whilelo $pg, zr, $len\t # sve" %} 5833 ins_encode %{ 5834 BasicType bt = Matcher::vector_element_basic_type(this); 5835 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 5836 __ sve_whilelo(as_PRegister($pg$$reg), size, zr, as_Register($len$$reg)); 5837 %} 5838 ins_pipe(pipe_slow); 5839 %} 5840 5841 // ------------------------------ CountLeadingZerosV ------------------------------ 5842 5843 instruct vcountLeadingZeros(vReg dst, vReg src) %{ 5844 predicate(UseSVE > 0 && 5845 !n->as_Vector()->is_predicated_vector()); 5846 match(Set dst (CountLeadingZerosV src)); 5847 ins_cost(SVE_COST); 5848 format %{ "sve_clz $dst, $src\t# vector (sve)" %} 5849 ins_encode %{ 5850 BasicType bt = Matcher::vector_element_basic_type(this); 5851 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 5852 __ sve_clz(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($src$$reg)); 5853 %} 5854 ins_pipe(pipe_slow); 5855 %} 5856 5857 // The dst and src should use the same register to make sure the 5858 // inactive lanes in dst save the same elements as src. 5859 instruct vcountLeadingZeros_masked(vReg dst_src, pRegGov pg) %{ 5860 predicate(UseSVE > 0); 5861 match(Set dst_src (CountLeadingZerosV dst_src pg)); 5862 ins_cost(SVE_COST); 5863 format %{ "sve_clz $dst_src, $pg, $dst_src\t# vector (sve)" %} 5864 ins_encode %{ 5865 BasicType bt = Matcher::vector_element_basic_type(this); 5866 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 5867 __ sve_clz(as_FloatRegister($dst_src$$reg), size, 5868 as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg)); 5869 %} 5870 ins_pipe(pipe_slow); 5871 %} 5872 5873 // ------------------------------ CountTrailingZerosV ----------------------------- 5874 5875 instruct vcountTrailingZeros(vReg dst, vReg src) %{ 5876 predicate(UseSVE > 0 && 5877 !n->as_Vector()->is_predicated_vector()); 5878 match(Set dst (CountTrailingZerosV src)); 5879 ins_cost(2 * SVE_COST); 5880 format %{ "sve_rbit $dst, $src\n\t" 5881 "sve_clz $dst, $dst\t# vector (sve)" %} 5882 ins_encode %{ 5883 BasicType bt = Matcher::vector_element_basic_type(this); 5884 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 5885 __ sve_rbit(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($src$$reg)); 5886 __ sve_clz(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($dst$$reg)); 5887 %} 5888 ins_pipe(pipe_slow); 5889 %} 5890 5891 // The dst and src should use the same register to make sure the 5892 // inactive lanes in dst save the same elements as src. 5893 instruct vcountTrailingZeros_masked(vReg dst_src, pRegGov pg) %{ 5894 predicate(UseSVE > 0); 5895 match(Set dst_src (CountTrailingZerosV dst_src pg)); 5896 ins_cost(2 * SVE_COST); 5897 format %{ "sve_rbit $dst_src, $pg, $dst_src\n\t" 5898 "sve_clz $dst_src, $pg, $dst_src\t# vector (sve)" %} 5899 ins_encode %{ 5900 BasicType bt = Matcher::vector_element_basic_type(this); 5901 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 5902 __ sve_rbit(as_FloatRegister($dst_src$$reg), size, 5903 as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg)); 5904 __ sve_clz(as_FloatRegister($dst_src$$reg), size, 5905 as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg)); 5906 %} 5907 ins_pipe(pipe_slow); 5908 %} 5909 5910 // ---------------------------------- ReverseV ------------------------------------ 5911 5912 instruct vreverse(vReg dst, vReg src) %{ 5913 predicate(UseSVE > 0 && 5914 !n->as_Vector()->is_predicated_vector()); 5915 match(Set dst (ReverseV src)); 5916 ins_cost(SVE_COST); 5917 format %{ "sve_rbit $dst, $src\t# vector (sve)" %} 5918 ins_encode %{ 5919 BasicType bt = Matcher::vector_element_basic_type(this); 5920 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 5921 __ sve_rbit(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($src$$reg)); 5922 %} 5923 ins_pipe(pipe_slow); 5924 %} 5925 5926 // The dst and src should use the same register to make sure the 5927 // inactive lanes in dst save the same elements as src. 5928 instruct vreverse_masked(vReg dst_src, pRegGov pg) %{ 5929 predicate(UseSVE > 0); 5930 match(Set dst_src (ReverseV dst_src pg)); 5931 ins_cost(SVE_COST); 5932 format %{ "sve_rbit $dst_src, $pg, $dst_src\t# vector (sve)" %} 5933 ins_encode %{ 5934 BasicType bt = Matcher::vector_element_basic_type(this); 5935 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 5936 __ sve_rbit(as_FloatRegister($dst_src$$reg), size, 5937 as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg)); 5938 %} 5939 ins_pipe(pipe_slow); 5940 %} 5941 5942 // -------------------------------- ReverseBytesV --------------------------------- 5943 5944 instruct vreverseBytes(vReg dst, vReg src) %{ 5945 predicate(UseSVE > 0 && 5946 !n->as_Vector()->is_predicated_vector()); 5947 match(Set dst (ReverseBytesV src)); 5948 ins_cost(SVE_COST); 5949 format %{ "sve_revb $dst, $src\t# vector (sve)" %} 5950 ins_encode %{ 5951 BasicType bt = Matcher::vector_element_basic_type(this); 5952 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 5953 if (bt == T_BYTE) { 5954 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) { 5955 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg)); 5956 } 5957 } else { 5958 __ sve_revb(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($src$$reg)); 5959 } 5960 %} 5961 ins_pipe(pipe_slow); 5962 %} 5963 5964 // The dst and src should use the same register to make sure the 5965 // inactive lanes in dst save the same elements as src. 5966 instruct vreverseBytes_masked(vReg dst_src, pRegGov pg) %{ 5967 predicate(UseSVE > 0); 5968 match(Set dst_src (ReverseBytesV dst_src pg)); 5969 ins_cost(SVE_COST); 5970 format %{ "sve_revb $dst_src, $pg, $dst_src\t# vector (sve)" %} 5971 ins_encode %{ 5972 BasicType bt = Matcher::vector_element_basic_type(this); 5973 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt); 5974 if (bt == T_BYTE) { 5975 // do nothing 5976 } else { 5977 __ sve_revb(as_FloatRegister($dst_src$$reg), size, 5978 as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg)); 5979 } 5980 %} 5981 ins_pipe(pipe_slow); 5982 %} 5983