1 //
2 // Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2020, 2021, Arm Limited. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // This file is automatically generated by running "m4 aarch64_sve_ad.m4". Do not edit ----
27
28 // AArch64 SVE Architecture Description File
29
30
31 // 4 bit signed offset -- for predicated load/store
32
33 operand vmemA_immIOffset4()
34 %{
35 // (esize / msize) = 1
36 predicate(Address::offset_ok_for_sve_immed(n->get_int(), 4,
37 Matcher::scalable_vector_reg_size(T_BYTE)));
38 match(ConI);
39
40 op_cost(0);
41 format %{ %}
42 interface(CONST_INTER);
43 %}
44
45 operand vmemA_immLOffset4()
46 %{
47 // (esize / msize) = 1
48 predicate(Address::offset_ok_for_sve_immed(n->get_long(), 4,
49 Matcher::scalable_vector_reg_size(T_BYTE)));
50 match(ConL);
51
52 op_cost(0);
53 format %{ %}
54 interface(CONST_INTER);
55 %}
56
57 operand vmemA_indOffI4(iRegP reg, vmemA_immIOffset4 off)
58 %{
59 constraint(ALLOC_IN_RC(ptr_reg));
60 match(AddP reg off);
61 op_cost(0);
62 format %{ "[$reg, $off]" %}
63 interface(MEMORY_INTER) %{
64 base($reg);
65 index(0xffffffff);
66 scale(0x0);
67 disp($off);
68 %}
69 %}
70
71 operand vmemA_indOffL4(iRegP reg, vmemA_immLOffset4 off)
72 %{
73 constraint(ALLOC_IN_RC(ptr_reg));
74 match(AddP reg off);
75 op_cost(0);
76 format %{ "[$reg, $off]" %}
77 interface(MEMORY_INTER) %{
78 base($reg);
79 index(0xffffffff);
80 scale(0x0);
81 disp($off);
82 %}
83 %}
84
85 // The indOff of vmemA is valid only when the vector element (load to/store from)
86 // size equals to memory element (load from/store to) size.
87 opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);
88
89 source_hpp %{
90 bool op_sve_supported(int opcode, int vlen, BasicType bt);
91 bool masked_op_sve_supported(int opcode, int vlen, BasicType bt);
92 %}
93
94 source %{
95
96 typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
97 PRegister Pg, const Address &adr);
98
99 // Predicated load/store, with optional ptrue to all elements of given predicate register.
100 static void loadStoreA_predicated(C2_MacroAssembler masm, bool is_store, FloatRegister reg,
101 PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
102 int opcode, Register base, int index, int size, int disp) {
103 sve_mem_insn_predicate insn;
104 int mesize = type2aelembytes(mem_elem_bt);
105 if (index == -1) {
106 assert(size == 0, "unsupported address mode: scale size = %d", size);
107 switch(mesize) {
108 case 1:
109 insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
110 break;
111 case 2:
112 insn = is_store ? &C2_MacroAssembler::sve_st1h : &C2_MacroAssembler::sve_ld1h;
113 break;
114 case 4:
115 insn = is_store ? &C2_MacroAssembler::sve_st1w : &C2_MacroAssembler::sve_ld1w;
116 break;
117 case 8:
118 insn = is_store ? &C2_MacroAssembler::sve_st1d : &C2_MacroAssembler::sve_ld1d;
119 break;
120 default:
121 assert(false, "unsupported");
122 ShouldNotReachHere();
123 }
124 int imm4 = disp / mesize / Matcher::scalable_vector_reg_size(vector_elem_bt);
125 (masm.*insn)(reg, Assembler::elemType_to_regVariant(vector_elem_bt), pg, Address(base, imm4));
126 } else {
127 assert(false, "unimplemented");
128 ShouldNotReachHere();
129 }
130 }
131
132 bool op_sve_supported(int opcode, int vlen, BasicType bt) {
133 int length_in_bytes = vlen * type2aelembytes(bt);
134 switch (opcode) {
135 case Op_MulAddVS2VI:
136 // No multiply reduction instructions
137 case Op_MulReductionVD:
138 case Op_MulReductionVF:
139 case Op_MulReductionVI:
140 case Op_MulReductionVL:
141 // Others
142 case Op_ExtractC:
143 case Op_ExtractUB:
144 return false;
145 // Vector API specific
146 case Op_VectorLoadShuffle:
147 case Op_VectorRearrange:
148 return vlen >= 4 && length_in_bytes <= MaxVectorSize;
149 case Op_LoadVector:
150 case Op_StoreVector:
151 return Matcher::vector_size_supported(bt, vlen);
152 default:
153 break;
154 }
155 // By default, we only support vector operations with no less than 8 bytes and 2 elements.
156 return 8 <= length_in_bytes && length_in_bytes <= MaxVectorSize && vlen >= 2;
157 }
158
159 bool masked_op_sve_supported(int opcode, int vlen, BasicType bt) {
160 if (opcode == Op_VectorRearrange) {
161 return false;
162 }
163 return op_sve_supported(opcode, vlen, bt);
164 }
165
166 %}
167
168 definitions %{
169 int_def SVE_COST (200, 200);
170 %}
171
172
173 // All SVE instructions
174
175 // vector load/store
176
177 // Unpredicated vector load/store
178 instruct loadV(vReg dst, vmemA mem) %{
179 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() >= 16 &&
180 n->as_LoadVector()->memory_size() == MaxVectorSize);
181 match(Set dst (LoadVector mem));
182 ins_cost(4 * SVE_COST);
183 format %{ "sve_ldr $dst, $mem\t# vector (sve)" %}
184 ins_encode %{
185 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
186 BasicType bt = Matcher::vector_element_basic_type(this);
187 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg, ptrue,
188 bt, bt, $mem->opcode(),
189 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
190 %}
191 ins_pipe(pipe_slow);
192 %}
193
194 instruct storeV(vReg src, vmemA mem) %{
195 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() >= 16 &&
196 n->as_StoreVector()->memory_size() == MaxVectorSize);
197 match(Set mem (StoreVector mem src));
198 ins_cost(4 * SVE_COST);
199 format %{ "sve_str $mem, $src\t# vector (sve)" %}
200 ins_encode %{
201 FloatRegister src_reg = as_FloatRegister($src$$reg);
202 BasicType bt = Matcher::vector_element_basic_type(this, $src);
203 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg, ptrue,
204 bt, bt, $mem->opcode(),
205 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
206 %}
207 ins_pipe(pipe_slow);
208 %}
209
210 // Load Vector (16 bits)
211 instruct loadV2_vreg(vReg dst, vmem2 mem)
212 %{
213 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 2);
214 match(Set dst (LoadVector mem));
215 ins_cost(4 * INSN_COST);
216 format %{ "ldrh $dst,$mem\t# vector (16 bits)" %}
217 ins_encode( aarch64_enc_ldrvH(dst, mem) );
218 ins_pipe(vload_reg_mem64);
219 %}
220
221 // Store Vector (16 bits)
222 instruct storeV2_vreg(vReg src, vmem2 mem)
223 %{
224 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 2);
225 match(Set mem (StoreVector mem src));
226 ins_cost(4 * INSN_COST);
227 format %{ "strh $mem,$src\t# vector (16 bits)" %}
228 ins_encode( aarch64_enc_strvH(src, mem) );
229 ins_pipe(vstore_reg_mem64);
230 %}
231
232 // Load Vector (32 bits)
233 instruct loadV4_vreg(vReg dst, vmem4 mem)
234 %{
235 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 4);
236 match(Set dst (LoadVector mem));
237 ins_cost(4 * INSN_COST);
238 format %{ "ldrs $dst,$mem\t# vector (32 bits)" %}
239 ins_encode( aarch64_enc_ldrvS(dst, mem) );
240 ins_pipe(vload_reg_mem64);
241 %}
242
243 // Store Vector (32 bits)
244 instruct storeV4_vreg(vReg src, vmem4 mem)
245 %{
246 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 4);
247 match(Set mem (StoreVector mem src));
248 ins_cost(4 * INSN_COST);
249 format %{ "strs $mem,$src\t# vector (32 bits)" %}
250 ins_encode( aarch64_enc_strvS(src, mem) );
251 ins_pipe(vstore_reg_mem64);
252 %}
253
254 // Load Vector (64 bits)
255 instruct loadV8_vreg(vReg dst, vmem8 mem)
256 %{
257 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 8);
258 match(Set dst (LoadVector mem));
259 ins_cost(4 * INSN_COST);
260 format %{ "ldrd $dst,$mem\t# vector (64 bits)" %}
261 ins_encode( aarch64_enc_ldrvD(dst, mem) );
262 ins_pipe(vload_reg_mem64);
263 %}
264
265 // Store Vector (64 bits)
266 instruct storeV8_vreg(vReg src, vmem8 mem)
267 %{
268 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 8);
269 match(Set mem (StoreVector mem src));
270 ins_cost(4 * INSN_COST);
271 format %{ "strd $mem,$src\t# vector (64 bits)" %}
272 ins_encode( aarch64_enc_strvD(src, mem) );
273 ins_pipe(vstore_reg_mem64);
274 %}
275
276 // Load Vector (128 bits)
277 instruct loadV16_vreg(vReg dst, vmem16 mem)
278 %{
279 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 16);
280 match(Set dst (LoadVector mem));
281 ins_cost(4 * INSN_COST);
282 format %{ "ldrq $dst,$mem\t# vector (128 bits)" %}
283 ins_encode( aarch64_enc_ldrvQ(dst, mem) );
284 ins_pipe(vload_reg_mem128);
285 %}
286
287 // Store Vector (128 bits)
288 instruct storeV16_vreg(vReg src, vmem16 mem)
289 %{
290 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 16);
291 match(Set mem (StoreVector mem src));
292 ins_cost(4 * INSN_COST);
293 format %{ "strq $mem,$src\t# vector (128 bits)" %}
294 ins_encode( aarch64_enc_strvQ(src, mem) );
295 ins_pipe(vstore_reg_mem128);
296 %}
297
298 // Predicated vector load/store, based on the vector length of the node.
299 // Only load/store values in the range of the memory_size. This is needed
300 // when the memory_size is lower than the hardware supported max vector size.
301 // And this might happen for Vector API mask vector load/store.
302 instruct loadV_partial(vReg dst, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
303 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() > 16 &&
304 n->as_LoadVector()->memory_size() < MaxVectorSize);
305 match(Set dst (LoadVector mem));
306 effect(TEMP pgtmp, KILL cr);
307 ins_cost(6 * SVE_COST);
308 format %{ "sve_whilelo_zr_imm $pgtmp, vector_length\n\t"
309 "sve_ldr $dst, $pgtmp, $mem\t# load vector partial" %}
310 ins_encode %{
311 BasicType bt = Matcher::vector_element_basic_type(this);
312 __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
313 Matcher::vector_length(this));
314 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
315 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg,
316 as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
317 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
318 %}
319 ins_pipe(pipe_slow);
320 %}
321
322 instruct storeV_partial(vReg src, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
323 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() > 16 &&
324 n->as_StoreVector()->memory_size() < MaxVectorSize);
325 match(Set mem (StoreVector mem src));
326 effect(TEMP pgtmp, KILL cr);
327 ins_cost(5 * SVE_COST);
328 format %{ "sve_whilelo_zr_imm $pgtmp, vector_length\n\t"
329 "sve_str $src, $pgtmp, $mem\t# store vector partial" %}
330 ins_encode %{
331 BasicType bt = Matcher::vector_element_basic_type(this, $src);
332 __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
333 Matcher::vector_length(this, $src));
334 FloatRegister src_reg = as_FloatRegister($src$$reg);
335 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg,
336 as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
337 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
338 %}
339 ins_pipe(pipe_slow);
340 %}
341
342 // vector load/store - predicated
343
344 instruct loadV_masked(vReg dst, vmemA mem, pRegGov pg) %{
345 predicate(UseSVE > 0 &&
346 n->as_LoadVector()->memory_size() == MaxVectorSize);
347 match(Set dst (LoadVectorMasked mem pg));
348 ins_cost(4 * SVE_COST);
349 format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated (sve)" %}
350 ins_encode %{
351 BasicType bt = Matcher::vector_element_basic_type(this);
352 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
353 as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
354 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
355 %}
356 ins_pipe(pipe_slow);
357 %}
358
359 instruct loadV_masked_partial(vReg dst, vmemA mem, pRegGov pg, pRegGov pgtmp, rFlagsReg cr) %{
360 predicate(UseSVE > 0 &&
361 n->as_LoadVector()->memory_size() < MaxVectorSize);
362 match(Set dst (LoadVectorMasked mem pg));
363 effect(TEMP pgtmp, KILL cr);
364 ins_cost(6 * SVE_COST);
365 format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated partial (sve)" %}
366 ins_encode %{
367 BasicType bt = Matcher::vector_element_basic_type(this);
368 __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
369 Matcher::vector_length(this));
370 __ sve_and(as_PRegister($pgtmp$$reg), as_PRegister($pgtmp$$reg),
371 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
372 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
373 as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
374 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
375 %}
376 ins_pipe(pipe_slow);
377 %}
378
379 instruct storeV_masked(vReg src, vmemA mem, pRegGov pg) %{
380 predicate(UseSVE > 0 &&
381 n->as_StoreVector()->memory_size() == MaxVectorSize);
382 match(Set mem (StoreVectorMasked mem (Binary src pg)));
383 ins_cost(4 * SVE_COST);
384 format %{ "sve_str $mem, $pg, $src\t# store vector predicated (sve)" %}
385 ins_encode %{
386 BasicType bt = Matcher::vector_element_basic_type(this, $src);
387 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
388 as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
389 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
390 %}
391 ins_pipe(pipe_slow);
392 %}
393
394 instruct storeV_masked_partial(vReg src, vmemA mem, pRegGov pg, pRegGov pgtmp, rFlagsReg cr) %{
395 predicate(UseSVE > 0 &&
396 n->as_StoreVector()->memory_size() < MaxVectorSize);
397 match(Set mem (StoreVectorMasked mem (Binary src pg)));
398 effect(TEMP pgtmp, KILL cr);
399 ins_cost(6 * SVE_COST);
400 format %{ "sve_str $mem, $pg, $src\t# store vector predicated partial (sve)" %}
401 ins_encode %{
402 BasicType bt = Matcher::vector_element_basic_type(this, $src);
403 __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
404 Matcher::vector_length(this, $src));
405 __ sve_and(as_PRegister($pgtmp$$reg), as_PRegister($pgtmp$$reg),
406 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
407 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
408 as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
409 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
410 %}
411 ins_pipe(pipe_slow);
412 %}
413
414 // maskAll
415
416 instruct vmaskAll_immI(pRegGov dst, immI src) %{
417 predicate(UseSVE > 0);
418 match(Set dst (MaskAll src));
419 ins_cost(SVE_COST);
420 format %{ "sve_ptrue/sve_pfalse $dst\t# mask all (sve) (B/H/S)" %}
421 ins_encode %{
422 int con = (int)$src$$constant;
423 if (con == 0) {
424 __ sve_pfalse(as_PRegister($dst$$reg));
425 } else {
426 assert(con == -1, "invalid constant value for mask");
427 BasicType bt = Matcher::vector_element_basic_type(this);
428 __ sve_ptrue(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt));
429 }
430 %}
431 ins_pipe(pipe_slow);
432 %}
433
434 instruct vmaskAllI(pRegGov dst, iRegIorL2I src, vReg tmp, rFlagsReg cr) %{
435 predicate(UseSVE > 0);
436 match(Set dst (MaskAll src));
437 effect(TEMP tmp, KILL cr);
438 ins_cost(2 * SVE_COST);
439 format %{ "sve_dup $tmp, $src\n\t"
440 "sve_cmpne $dst, $tmp, 0\t# mask all (sve) (B/H/S)" %}
441 ins_encode %{
442 BasicType bt = Matcher::vector_element_basic_type(this);
443 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
444 __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
445 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
446 %}
447 ins_pipe(pipe_slow);
448 %}
449
450 instruct vmaskAll_immL(pRegGov dst, immL src) %{
451 predicate(UseSVE > 0);
452 match(Set dst (MaskAll src));
453 ins_cost(SVE_COST);
454 format %{ "sve_ptrue/sve_pfalse $dst\t# mask all (sve) (D)" %}
455 ins_encode %{
456 long con = (long)$src$$constant;
457 if (con == 0) {
458 __ sve_pfalse(as_PRegister($dst$$reg));
459 } else {
460 assert(con == -1, "invalid constant value for mask");
461 BasicType bt = Matcher::vector_element_basic_type(this);
462 __ sve_ptrue(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt));
463 }
464 %}
465 ins_pipe(pipe_slow);
466 %}
467
468 instruct vmaskAllL(pRegGov dst, iRegL src, vReg tmp, rFlagsReg cr) %{
469 predicate(UseSVE > 0);
470 match(Set dst (MaskAll src));
471 effect(TEMP tmp, KILL cr);
472 ins_cost(2 * SVE_COST);
473 format %{ "sve_dup $tmp, $src\n\t"
474 "sve_cmpne $dst, $tmp, 0\t# mask all (sve) (D)" %}
475 ins_encode %{
476 BasicType bt = Matcher::vector_element_basic_type(this);
477 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
478 __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
479 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
480 %}
481 ins_pipe(pipe_slow);
482 %}
483
484 // mask logical and/or/xor
485
486 instruct vmask_and(pRegGov pd, pRegGov pn, pRegGov pm) %{
487 predicate(UseSVE > 0);
488 match(Set pd (AndVMask pn pm));
489 ins_cost(SVE_COST);
490 format %{ "sve_and $pd, $pn, $pm\t# predicate (sve)" %}
491 ins_encode %{
492 __ sve_and(as_PRegister($pd$$reg), ptrue,
493 as_PRegister($pn$$reg), as_PRegister($pm$$reg));
494 %}
495 ins_pipe(pipe_slow);
496 %}
497
498 instruct vmask_or(pRegGov pd, pRegGov pn, pRegGov pm) %{
499 predicate(UseSVE > 0);
500 match(Set pd (OrVMask pn pm));
501 ins_cost(SVE_COST);
502 format %{ "sve_orr $pd, $pn, $pm\t# predicate (sve)" %}
503 ins_encode %{
504 __ sve_orr(as_PRegister($pd$$reg), ptrue,
505 as_PRegister($pn$$reg), as_PRegister($pm$$reg));
506 %}
507 ins_pipe(pipe_slow);
508 %}
509
510 instruct vmask_xor(pRegGov pd, pRegGov pn, pRegGov pm) %{
511 predicate(UseSVE > 0);
512 match(Set pd (XorVMask pn pm));
513 ins_cost(SVE_COST);
514 format %{ "sve_eor $pd, $pn, $pm\t# predicate (sve)" %}
515 ins_encode %{
516 __ sve_eor(as_PRegister($pd$$reg), ptrue,
517 as_PRegister($pn$$reg), as_PRegister($pm$$reg));
518 %}
519 ins_pipe(pipe_slow);
520 %}
521
522 // mask logical and_not
523
524 instruct vmask_and_notI(pRegGov pd, pRegGov pn, pRegGov pm, immI_M1 m1) %{
525 predicate(UseSVE > 0);
526 match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
527 ins_cost(SVE_COST);
528 format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) (B/H/S)" %}
529 ins_encode %{
530 __ sve_bic(as_PRegister($pd$$reg), ptrue,
531 as_PRegister($pn$$reg), as_PRegister($pm$$reg));
532 %}
533 ins_pipe(pipe_slow);
534 %}
535
536 instruct vmask_and_notL(pRegGov pd, pRegGov pn, pRegGov pm, immL_M1 m1) %{
537 predicate(UseSVE > 0);
538 match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
539 ins_cost(SVE_COST);
540 format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) (D)" %}
541 ins_encode %{
542 __ sve_bic(as_PRegister($pd$$reg), ptrue,
543 as_PRegister($pn$$reg), as_PRegister($pm$$reg));
544 %}
545 ins_pipe(pipe_slow);
546 %}
547
548 // vector reinterpret
549
550 instruct reinterpret(vReg dst) %{
551 predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() ==
552 n->in(1)->bottom_type()->is_vect()->length_in_bytes()); // src == dst
553 match(Set dst (VectorReinterpret dst));
554 ins_cost(0);
555 format %{ "# reinterpret $dst\t# do nothing" %}
556 ins_encode %{
557 // empty
558 %}
559 ins_pipe(pipe_class_empty);
560 %}
561
562 instruct reinterpretResize(vReg dst, vReg src, pRegGov pgtmp, rFlagsReg cr) %{
563 predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() !=
564 n->in(1)->bottom_type()->is_vect()->length_in_bytes()); // src != dst
565 match(Set dst (VectorReinterpret src));
566 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
567 ins_cost(3 * SVE_COST);
568 format %{ "reinterpretResize $dst, $src\t# vector (sve)" %}
569 ins_encode %{
570 uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
571 uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
572 uint length_in_bytes_resize = length_in_bytes_src < length_in_bytes_dst ?
573 length_in_bytes_src : length_in_bytes_dst;
574 assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
575 "invalid vector length");
576 __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ B, length_in_bytes_resize);
577 __ sve_dup(as_FloatRegister($dst$$reg), __ B, 0);
578 __ sve_sel(as_FloatRegister($dst$$reg), __ B, as_PRegister($pgtmp$$reg),
579 as_FloatRegister($src$$reg), as_FloatRegister($dst$$reg));
580 %}
581 ins_pipe(pipe_slow);
582 %}
583
584 // vector mask reinterpret
585
586 instruct vmask_reinterpret_same_esize(pRegGov dst_src) %{
587 predicate(UseSVE > 0 &&
588 n->as_Vector()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
589 n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
590 match(Set dst_src (VectorReinterpret dst_src));
591 ins_cost(0);
592 format %{ "# vmask_reinterpret $dst_src\t# do nothing" %}
593 ins_encode %{
594 // empty
595 %}
596 ins_pipe(pipe_class_empty);
597 %}
598
599 instruct vmask_reinterpret_diff_esize(pRegGov dst, pRegGov src, vReg tmp, rFlagsReg cr) %{
600 predicate(UseSVE > 0 &&
601 n->as_Vector()->length() != n->in(1)->bottom_type()->is_vect()->length() &&
602 n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
603 match(Set dst (VectorReinterpret src));
604 effect(TEMP tmp, KILL cr);
605 ins_cost(2 * SVE_COST);
606 format %{ "# vmask_reinterpret $dst, $src\t# vector (sve)" %}
607 ins_encode %{
608 BasicType from_bt = Matcher::vector_element_basic_type(this, $src);
609 Assembler::SIMD_RegVariant from_size = __ elemType_to_regVariant(from_bt);
610 BasicType to_bt = Matcher::vector_element_basic_type(this);
611 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
612 __ sve_cpy(as_FloatRegister($tmp$$reg), from_size, as_PRegister($src$$reg), -1, false);
613 __ sve_cmp(Assembler::EQ, as_PRegister($dst$$reg), to_size, ptrue, as_FloatRegister($tmp$$reg), -1);
614 %}
615 ins_pipe(pipe_slow);
616 %}
617
618 // vector abs
619
620 instruct vabsB(vReg dst, vReg src) %{
621 predicate(UseSVE > 0 &&
622 !n->as_Vector()->is_predicated_vector());
623 match(Set dst (AbsVB src));
624 ins_cost(SVE_COST);
625 format %{ "sve_abs $dst, $src\t# vector (sve) (B)" %}
626 ins_encode %{
627 __ sve_abs(as_FloatRegister($dst$$reg), __ B,
628 ptrue, as_FloatRegister($src$$reg));
629 %}
630 ins_pipe(pipe_slow);
631 %}
632
633 instruct vabsS(vReg dst, vReg src) %{
634 predicate(UseSVE > 0 &&
635 !n->as_Vector()->is_predicated_vector());
636 match(Set dst (AbsVS src));
637 ins_cost(SVE_COST);
638 format %{ "sve_abs $dst, $src\t# vector (sve) (H)" %}
639 ins_encode %{
640 __ sve_abs(as_FloatRegister($dst$$reg), __ H,
641 ptrue, as_FloatRegister($src$$reg));
642 %}
643 ins_pipe(pipe_slow);
644 %}
645
646 instruct vabsI(vReg dst, vReg src) %{
647 predicate(UseSVE > 0 &&
648 !n->as_Vector()->is_predicated_vector());
649 match(Set dst (AbsVI src));
650 ins_cost(SVE_COST);
651 format %{ "sve_abs $dst, $src\t# vector (sve) (S)" %}
652 ins_encode %{
653 __ sve_abs(as_FloatRegister($dst$$reg), __ S,
654 ptrue, as_FloatRegister($src$$reg));
655 %}
656 ins_pipe(pipe_slow);
657 %}
658
659 instruct vabsL(vReg dst, vReg src) %{
660 predicate(UseSVE > 0 &&
661 !n->as_Vector()->is_predicated_vector());
662 match(Set dst (AbsVL src));
663 ins_cost(SVE_COST);
664 format %{ "sve_abs $dst, $src\t# vector (sve) (D)" %}
665 ins_encode %{
666 __ sve_abs(as_FloatRegister($dst$$reg), __ D,
667 ptrue, as_FloatRegister($src$$reg));
668 %}
669 ins_pipe(pipe_slow);
670 %}
671
672 instruct vabsF(vReg dst, vReg src) %{
673 predicate(UseSVE > 0 &&
674 !n->as_Vector()->is_predicated_vector());
675 match(Set dst (AbsVF src));
676 ins_cost(SVE_COST);
677 format %{ "sve_fabs $dst, $src\t# vector (sve) (S)" %}
678 ins_encode %{
679 __ sve_fabs(as_FloatRegister($dst$$reg), __ S,
680 ptrue, as_FloatRegister($src$$reg));
681 %}
682 ins_pipe(pipe_slow);
683 %}
684
685 instruct vabsD(vReg dst, vReg src) %{
686 predicate(UseSVE > 0 &&
687 !n->as_Vector()->is_predicated_vector());
688 match(Set dst (AbsVD src));
689 ins_cost(SVE_COST);
690 format %{ "sve_fabs $dst, $src\t# vector (sve) (D)" %}
691 ins_encode %{
692 __ sve_fabs(as_FloatRegister($dst$$reg), __ D,
693 ptrue, as_FloatRegister($src$$reg));
694 %}
695 ins_pipe(pipe_slow);
696 %}
697
698 // vector abs - predicated
699
700 instruct vabsB_masked(vReg dst_src, pRegGov pg) %{
701 predicate(UseSVE > 0);
702 match(Set dst_src (AbsVB dst_src pg));
703 ins_cost(SVE_COST);
704 format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (B)" %}
705 ins_encode %{
706 __ sve_abs(as_FloatRegister($dst_src$$reg), __ B,
707 as_PRegister($pg$$reg),
708 as_FloatRegister($dst_src$$reg));
709 %}
710 ins_pipe(pipe_slow);
711 %}
712
713 instruct vabsS_masked(vReg dst_src, pRegGov pg) %{
714 predicate(UseSVE > 0);
715 match(Set dst_src (AbsVS dst_src pg));
716 ins_cost(SVE_COST);
717 format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (H)" %}
718 ins_encode %{
719 __ sve_abs(as_FloatRegister($dst_src$$reg), __ H,
720 as_PRegister($pg$$reg),
721 as_FloatRegister($dst_src$$reg));
722 %}
723 ins_pipe(pipe_slow);
724 %}
725
726 instruct vabsI_masked(vReg dst_src, pRegGov pg) %{
727 predicate(UseSVE > 0);
728 match(Set dst_src (AbsVI dst_src pg));
729 ins_cost(SVE_COST);
730 format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
731 ins_encode %{
732 __ sve_abs(as_FloatRegister($dst_src$$reg), __ S,
733 as_PRegister($pg$$reg),
734 as_FloatRegister($dst_src$$reg));
735 %}
736 ins_pipe(pipe_slow);
737 %}
738
739 instruct vabsL_masked(vReg dst_src, pRegGov pg) %{
740 predicate(UseSVE > 0);
741 match(Set dst_src (AbsVL dst_src pg));
742 ins_cost(SVE_COST);
743 format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
744 ins_encode %{
745 __ sve_abs(as_FloatRegister($dst_src$$reg), __ D,
746 as_PRegister($pg$$reg),
747 as_FloatRegister($dst_src$$reg));
748 %}
749 ins_pipe(pipe_slow);
750 %}
751
752 instruct vabsF_masked(vReg dst_src, pRegGov pg) %{
753 predicate(UseSVE > 0);
754 match(Set dst_src (AbsVF dst_src pg));
755 ins_cost(SVE_COST);
756 format %{ "sve_fabs $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
757 ins_encode %{
758 __ sve_fabs(as_FloatRegister($dst_src$$reg), __ S,
759 as_PRegister($pg$$reg),
760 as_FloatRegister($dst_src$$reg));
761 %}
762 ins_pipe(pipe_slow);
763 %}
764
765 instruct vabsD_masked(vReg dst_src, pRegGov pg) %{
766 predicate(UseSVE > 0);
767 match(Set dst_src (AbsVD dst_src pg));
768 ins_cost(SVE_COST);
769 format %{ "sve_fabs $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
770 ins_encode %{
771 __ sve_fabs(as_FloatRegister($dst_src$$reg), __ D,
772 as_PRegister($pg$$reg),
773 as_FloatRegister($dst_src$$reg));
774 %}
775 ins_pipe(pipe_slow);
776 %}
777
778 // vector add
779
780 instruct vaddB(vReg dst, vReg src1, vReg src2) %{
781 predicate(UseSVE > 0);
782 match(Set dst (AddVB src1 src2));
783 ins_cost(SVE_COST);
784 format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (B)" %}
785 ins_encode %{
786 __ sve_add(as_FloatRegister($dst$$reg), __ B,
787 as_FloatRegister($src1$$reg),
788 as_FloatRegister($src2$$reg));
789 %}
790 ins_pipe(pipe_slow);
791 %}
792
793 instruct vaddS(vReg dst, vReg src1, vReg src2) %{
794 predicate(UseSVE > 0);
795 match(Set dst (AddVS src1 src2));
796 ins_cost(SVE_COST);
797 format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (H)" %}
798 ins_encode %{
799 __ sve_add(as_FloatRegister($dst$$reg), __ H,
800 as_FloatRegister($src1$$reg),
801 as_FloatRegister($src2$$reg));
802 %}
803 ins_pipe(pipe_slow);
804 %}
805
806 instruct vaddI(vReg dst, vReg src1, vReg src2) %{
807 predicate(UseSVE > 0);
808 match(Set dst (AddVI src1 src2));
809 ins_cost(SVE_COST);
810 format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (S)" %}
811 ins_encode %{
812 __ sve_add(as_FloatRegister($dst$$reg), __ S,
813 as_FloatRegister($src1$$reg),
814 as_FloatRegister($src2$$reg));
815 %}
816 ins_pipe(pipe_slow);
817 %}
818
819 instruct vaddL(vReg dst, vReg src1, vReg src2) %{
820 predicate(UseSVE > 0);
821 match(Set dst (AddVL src1 src2));
822 ins_cost(SVE_COST);
823 format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (D)" %}
824 ins_encode %{
825 __ sve_add(as_FloatRegister($dst$$reg), __ D,
826 as_FloatRegister($src1$$reg),
827 as_FloatRegister($src2$$reg));
828 %}
829 ins_pipe(pipe_slow);
830 %}
831
832 instruct vaddF(vReg dst, vReg src1, vReg src2) %{
833 predicate(UseSVE > 0);
834 match(Set dst (AddVF src1 src2));
835 ins_cost(SVE_COST);
836 format %{ "sve_fadd $dst, $src1, $src2\t # vector (sve) (S)" %}
837 ins_encode %{
838 __ sve_fadd(as_FloatRegister($dst$$reg), __ S,
839 as_FloatRegister($src1$$reg),
840 as_FloatRegister($src2$$reg));
841 %}
842 ins_pipe(pipe_slow);
843 %}
844
845 instruct vaddD(vReg dst, vReg src1, vReg src2) %{
846 predicate(UseSVE > 0);
847 match(Set dst (AddVD src1 src2));
848 ins_cost(SVE_COST);
849 format %{ "sve_fadd $dst, $src1, $src2\t # vector (sve) (D)" %}
850 ins_encode %{
851 __ sve_fadd(as_FloatRegister($dst$$reg), __ D,
852 as_FloatRegister($src1$$reg),
853 as_FloatRegister($src2$$reg));
854 %}
855 ins_pipe(pipe_slow);
856 %}
857
858 // vector add - predicated
859
860 instruct vaddB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
861 predicate(UseSVE > 0);
862 match(Set dst_src1 (AddVB (Binary dst_src1 src2) pg));
863 ins_cost(SVE_COST);
864 format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
865 ins_encode %{
866 __ sve_add(as_FloatRegister($dst_src1$$reg), __ B,
867 as_PRegister($pg$$reg),
868 as_FloatRegister($src2$$reg));
869 %}
870 ins_pipe(pipe_slow);
871 %}
872
873 instruct vaddS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
874 predicate(UseSVE > 0);
875 match(Set dst_src1 (AddVS (Binary dst_src1 src2) pg));
876 ins_cost(SVE_COST);
877 format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
878 ins_encode %{
879 __ sve_add(as_FloatRegister($dst_src1$$reg), __ H,
880 as_PRegister($pg$$reg),
881 as_FloatRegister($src2$$reg));
882 %}
883 ins_pipe(pipe_slow);
884 %}
885
886 instruct vaddI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
887 predicate(UseSVE > 0);
888 match(Set dst_src1 (AddVI (Binary dst_src1 src2) pg));
889 ins_cost(SVE_COST);
890 format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
891 ins_encode %{
892 __ sve_add(as_FloatRegister($dst_src1$$reg), __ S,
893 as_PRegister($pg$$reg),
894 as_FloatRegister($src2$$reg));
895 %}
896 ins_pipe(pipe_slow);
897 %}
898
899 instruct vaddL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
900 predicate(UseSVE > 0);
901 match(Set dst_src1 (AddVL (Binary dst_src1 src2) pg));
902 ins_cost(SVE_COST);
903 format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
904 ins_encode %{
905 __ sve_add(as_FloatRegister($dst_src1$$reg), __ D,
906 as_PRegister($pg$$reg),
907 as_FloatRegister($src2$$reg));
908 %}
909 ins_pipe(pipe_slow);
910 %}
911
912 instruct vaddF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
913 predicate(UseSVE > 0);
914 match(Set dst_src1 (AddVF (Binary dst_src1 src2) pg));
915 ins_cost(SVE_COST);
916 format %{ "sve_fadd $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
917 ins_encode %{
918 __ sve_fadd(as_FloatRegister($dst_src1$$reg), __ S,
919 as_PRegister($pg$$reg),
920 as_FloatRegister($src2$$reg));
921 %}
922 ins_pipe(pipe_slow);
923 %}
924
925 instruct vaddD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
926 predicate(UseSVE > 0);
927 match(Set dst_src1 (AddVD (Binary dst_src1 src2) pg));
928 ins_cost(SVE_COST);
929 format %{ "sve_fadd $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
930 ins_encode %{
931 __ sve_fadd(as_FloatRegister($dst_src1$$reg), __ D,
932 as_PRegister($pg$$reg),
933 as_FloatRegister($src2$$reg));
934 %}
935 ins_pipe(pipe_slow);
936 %}
937
938 // vector and
939
940 instruct vand(vReg dst, vReg src1, vReg src2) %{
941 predicate(UseSVE > 0);
942 match(Set dst (AndV src1 src2));
943 ins_cost(SVE_COST);
944 format %{ "sve_and $dst, $src1, $src2\t# vector (sve)" %}
945 ins_encode %{
946 __ sve_and(as_FloatRegister($dst$$reg),
947 as_FloatRegister($src1$$reg),
948 as_FloatRegister($src2$$reg));
949 %}
950 ins_pipe(pipe_slow);
951 %}
952
953 // vector or
954
955 instruct vor(vReg dst, vReg src1, vReg src2) %{
956 predicate(UseSVE > 0);
957 match(Set dst (OrV src1 src2));
958 ins_cost(SVE_COST);
959 format %{ "sve_orr $dst, $src1, $src2\t# vector (sve)" %}
960 ins_encode %{
961 __ sve_orr(as_FloatRegister($dst$$reg),
962 as_FloatRegister($src1$$reg),
963 as_FloatRegister($src2$$reg));
964 %}
965 ins_pipe(pipe_slow);
966 %}
967
968 // vector xor
969
970 instruct vxor(vReg dst, vReg src1, vReg src2) %{
971 predicate(UseSVE > 0);
972 match(Set dst (XorV src1 src2));
973 ins_cost(SVE_COST);
974 format %{ "sve_eor $dst, $src1, $src2\t# vector (sve)" %}
975 ins_encode %{
976 __ sve_eor(as_FloatRegister($dst$$reg),
977 as_FloatRegister($src1$$reg),
978 as_FloatRegister($src2$$reg));
979 %}
980 ins_pipe(pipe_slow);
981 %}
982
983 // vector and - predicated
984
985 instruct vand_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
986 predicate(UseSVE > 0);
987 match(Set dst_src1 (AndV (Binary dst_src1 src2) pg));
988 ins_cost(SVE_COST);
989 format %{ "sve_and $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
990 ins_encode %{
991 BasicType bt = Matcher::vector_element_basic_type(this);
992 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
993 __ sve_and(as_FloatRegister($dst_src1$$reg), size,
994 as_PRegister($pg$$reg),
995 as_FloatRegister($src2$$reg));
996 %}
997 ins_pipe(pipe_slow);
998 %}
999
1000 // vector or - predicated
1001
1002 instruct vor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1003 predicate(UseSVE > 0);
1004 match(Set dst_src1 (OrV (Binary dst_src1 src2) pg));
1005 ins_cost(SVE_COST);
1006 format %{ "sve_orr $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
1007 ins_encode %{
1008 BasicType bt = Matcher::vector_element_basic_type(this);
1009 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1010 __ sve_orr(as_FloatRegister($dst_src1$$reg), size,
1011 as_PRegister($pg$$reg),
1012 as_FloatRegister($src2$$reg));
1013 %}
1014 ins_pipe(pipe_slow);
1015 %}
1016
1017 // vector xor - predicated
1018
1019 instruct vxor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1020 predicate(UseSVE > 0);
1021 match(Set dst_src1 (XorV (Binary dst_src1 src2) pg));
1022 ins_cost(SVE_COST);
1023 format %{ "sve_eor $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
1024 ins_encode %{
1025 BasicType bt = Matcher::vector_element_basic_type(this);
1026 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1027 __ sve_eor(as_FloatRegister($dst_src1$$reg), size,
1028 as_PRegister($pg$$reg),
1029 as_FloatRegister($src2$$reg));
1030 %}
1031 ins_pipe(pipe_slow);
1032 %}
1033
1034 // vector not
1035
1036 instruct vnotI(vReg dst, vReg src, immI_M1 m1) %{
1037 predicate(UseSVE > 0);
1038 match(Set dst (XorV src (ReplicateB m1)));
1039 match(Set dst (XorV src (ReplicateS m1)));
1040 match(Set dst (XorV src (ReplicateI m1)));
1041 ins_cost(SVE_COST);
1042 format %{ "sve_not $dst, $src\t# vector (sve) B/H/S" %}
1043 ins_encode %{
1044 __ sve_not(as_FloatRegister($dst$$reg), __ D,
1045 ptrue, as_FloatRegister($src$$reg));
1046 %}
1047 ins_pipe(pipe_slow);
1048 %}
1049
1050 instruct vnotL(vReg dst, vReg src, immL_M1 m1) %{
1051 predicate(UseSVE > 0);
1052 match(Set dst (XorV src (ReplicateL m1)));
1053 ins_cost(SVE_COST);
1054 format %{ "sve_not $dst, $src\t# vector (sve) D" %}
1055 ins_encode %{
1056 __ sve_not(as_FloatRegister($dst$$reg), __ D,
1057 ptrue, as_FloatRegister($src$$reg));
1058 %}
1059 ins_pipe(pipe_slow);
1060 %}
1061
1062 // vector and_not
1063
1064 instruct vand_notI(vReg dst, vReg src1, vReg src2, immI_M1 m1) %{
1065 predicate(UseSVE > 0);
1066 match(Set dst (AndV src1 (XorV src2 (ReplicateB m1))));
1067 match(Set dst (AndV src1 (XorV src2 (ReplicateS m1))));
1068 match(Set dst (AndV src1 (XorV src2 (ReplicateI m1))));
1069 ins_cost(SVE_COST);
1070 format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) B/H/S" %}
1071 ins_encode %{
1072 __ sve_bic(as_FloatRegister($dst$$reg),
1073 as_FloatRegister($src1$$reg),
1074 as_FloatRegister($src2$$reg));
1075 %}
1076 ins_pipe(pipe_slow);
1077 %}
1078
1079 instruct vand_notL(vReg dst, vReg src1, vReg src2, immL_M1 m1) %{
1080 predicate(UseSVE > 0);
1081 match(Set dst (AndV src1 (XorV src2 (ReplicateL m1))));
1082 ins_cost(SVE_COST);
1083 format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) D" %}
1084 ins_encode %{
1085 __ sve_bic(as_FloatRegister($dst$$reg),
1086 as_FloatRegister($src1$$reg),
1087 as_FloatRegister($src2$$reg));
1088 %}
1089 ins_pipe(pipe_slow);
1090 %}
1091
1092 // vector float div
1093
1094 instruct vdivF(vReg dst_src1, vReg src2) %{
1095 predicate(UseSVE > 0);
1096 match(Set dst_src1 (DivVF dst_src1 src2));
1097 ins_cost(SVE_COST);
1098 format %{ "sve_fdiv $dst_src1, $dst_src1, $src2\t# vector (sve) (S)" %}
1099 ins_encode %{
1100 __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ S,
1101 ptrue, as_FloatRegister($src2$$reg));
1102 %}
1103 ins_pipe(pipe_slow);
1104 %}
1105
1106 instruct vdivD(vReg dst_src1, vReg src2) %{
1107 predicate(UseSVE > 0);
1108 match(Set dst_src1 (DivVD dst_src1 src2));
1109 ins_cost(SVE_COST);
1110 format %{ "sve_fdiv $dst_src1, $dst_src1, $src2\t# vector (sve) (D)" %}
1111 ins_encode %{
1112 __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ D,
1113 ptrue, as_FloatRegister($src2$$reg));
1114 %}
1115 ins_pipe(pipe_slow);
1116 %}
1117
1118 // vector float div - predicated
1119
1120 instruct vfdivF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1121 predicate(UseSVE > 0);
1122 match(Set dst_src1 (DivVF (Binary dst_src1 src2) pg));
1123 ins_cost(SVE_COST);
1124 format %{ "sve_fdiv $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
1125 ins_encode %{
1126 __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ S,
1127 as_PRegister($pg$$reg),
1128 as_FloatRegister($src2$$reg));
1129 %}
1130 ins_pipe(pipe_slow);
1131 %}
1132
1133 instruct vfdivD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1134 predicate(UseSVE > 0);
1135 match(Set dst_src1 (DivVD (Binary dst_src1 src2) pg));
1136 ins_cost(SVE_COST);
1137 format %{ "sve_fdiv $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
1138 ins_encode %{
1139 __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ D,
1140 as_PRegister($pg$$reg),
1141 as_FloatRegister($src2$$reg));
1142 %}
1143 ins_pipe(pipe_slow);
1144 %}
1145
1146 // vector min/max
1147
1148 instruct vmin(vReg dst_src1, vReg src2) %{
1149 predicate(UseSVE > 0);
1150 match(Set dst_src1 (MinV dst_src1 src2));
1151 ins_cost(SVE_COST);
1152 format %{ "sve_min $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
1153 ins_encode %{
1154 BasicType bt = Matcher::vector_element_basic_type(this);
1155 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1156 if (is_floating_point_type(bt)) {
1157 __ sve_fmin(as_FloatRegister($dst_src1$$reg), size,
1158 ptrue, as_FloatRegister($src2$$reg));
1159 } else {
1160 assert(is_integral_type(bt), "unsupported type");
1161 __ sve_smin(as_FloatRegister($dst_src1$$reg), size,
1162 ptrue, as_FloatRegister($src2$$reg));
1163 }
1164 %}
1165 ins_pipe(pipe_slow);
1166 %}
1167
1168 instruct vmax(vReg dst_src1, vReg src2) %{
1169 predicate(UseSVE > 0);
1170 match(Set dst_src1 (MaxV dst_src1 src2));
1171 ins_cost(SVE_COST);
1172 format %{ "sve_max $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
1173 ins_encode %{
1174 BasicType bt = Matcher::vector_element_basic_type(this);
1175 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1176 if (is_floating_point_type(bt)) {
1177 __ sve_fmax(as_FloatRegister($dst_src1$$reg), size,
1178 ptrue, as_FloatRegister($src2$$reg));
1179 } else {
1180 assert(is_integral_type(bt), "unsupported type");
1181 __ sve_smax(as_FloatRegister($dst_src1$$reg), size,
1182 ptrue, as_FloatRegister($src2$$reg));
1183 }
1184 %}
1185 ins_pipe(pipe_slow);
1186 %}
1187
1188 // vector min/max - predicated
1189
1190 instruct vmin_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1191 predicate(UseSVE > 0);
1192 match(Set dst_src1 (MinV (Binary dst_src1 src2) pg));
1193 ins_cost(SVE_COST);
1194 format %{ "sve_min $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %}
1195 ins_encode %{
1196 BasicType bt = Matcher::vector_element_basic_type(this);
1197 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1198 if (is_floating_point_type(bt)) {
1199 __ sve_fmin(as_FloatRegister($dst_src1$$reg), size,
1200 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1201 } else {
1202 assert(is_integral_type(bt), "unsupported type");
1203 __ sve_smin(as_FloatRegister($dst_src1$$reg), size,
1204 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1205 }
1206 %}
1207 ins_pipe(pipe_slow);
1208 %}
1209
1210 instruct vmax_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1211 predicate(UseSVE > 0);
1212 match(Set dst_src1 (MaxV (Binary dst_src1 src2) pg));
1213 ins_cost(SVE_COST);
1214 format %{ "sve_max $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %}
1215 ins_encode %{
1216 BasicType bt = Matcher::vector_element_basic_type(this);
1217 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1218 if (is_floating_point_type(bt)) {
1219 __ sve_fmax(as_FloatRegister($dst_src1$$reg), size,
1220 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1221 } else {
1222 assert(is_integral_type(bt), "unsupported type");
1223 __ sve_smax(as_FloatRegister($dst_src1$$reg), size,
1224 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1225 }
1226 %}
1227 ins_pipe(pipe_slow);
1228 %}
1229
1230 // vector fmla
1231
1232 // dst_src1 = dst_src1 + src2 * src3
1233 instruct vfmlaF(vReg dst_src1, vReg src2, vReg src3) %{
1234 predicate(UseFMA && UseSVE > 0);
1235 match(Set dst_src1 (FmaVF dst_src1 (Binary src2 src3)));
1236 ins_cost(SVE_COST);
1237 format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1238 ins_encode %{
1239 __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ S,
1240 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1241 %}
1242 ins_pipe(pipe_slow);
1243 %}
1244
1245 // dst_src1 = dst_src1 + src2 * src3
1246 instruct vfmlaD(vReg dst_src1, vReg src2, vReg src3) %{
1247 predicate(UseFMA && UseSVE > 0);
1248 match(Set dst_src1 (FmaVD dst_src1 (Binary src2 src3)));
1249 ins_cost(SVE_COST);
1250 format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1251 ins_encode %{
1252 __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ D,
1253 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1254 %}
1255 ins_pipe(pipe_slow);
1256 %}
1257
1258 // vector fmla - predicated
1259
1260 // dst_src1 = dst_src1 * src2 + src3
1261 instruct vfmlaF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1262 predicate(UseFMA && UseSVE > 0);
1263 match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary src3 pg)));
1264 ins_cost(SVE_COST);
1265 format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) (S)" %}
1266 ins_encode %{
1267 __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
1268 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1269 %}
1270 ins_pipe(pipe_slow);
1271 %}
1272
1273 // dst_src1 = dst_src1 * src2 + src3
1274 instruct vfmlaD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1275 predicate(UseFMA && UseSVE > 0);
1276 match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary src3 pg)));
1277 ins_cost(SVE_COST);
1278 format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) (D)" %}
1279 ins_encode %{
1280 __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
1281 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1282 %}
1283 ins_pipe(pipe_slow);
1284 %}
1285
1286 // vector fmls
1287
1288 // dst_src1 = dst_src1 + -src2 * src3
1289 // dst_src1 = dst_src1 + src2 * -src3
1290 instruct vfmlsF(vReg dst_src1, vReg src2, vReg src3) %{
1291 predicate(UseFMA && UseSVE > 0);
1292 match(Set dst_src1 (FmaVF dst_src1 (Binary (NegVF src2) src3)));
1293 match(Set dst_src1 (FmaVF dst_src1 (Binary src2 (NegVF src3))));
1294 ins_cost(SVE_COST);
1295 format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1296 ins_encode %{
1297 __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ S,
1298 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1299 %}
1300 ins_pipe(pipe_slow);
1301 %}
1302
1303 // dst_src1 = dst_src1 + -src2 * src3
1304 // dst_src1 = dst_src1 + src2 * -src3
1305 instruct vfmlsD(vReg dst_src1, vReg src2, vReg src3) %{
1306 predicate(UseFMA && UseSVE > 0);
1307 match(Set dst_src1 (FmaVD dst_src1 (Binary (NegVD src2) src3)));
1308 match(Set dst_src1 (FmaVD dst_src1 (Binary src2 (NegVD src3))));
1309 ins_cost(SVE_COST);
1310 format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1311 ins_encode %{
1312 __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ D,
1313 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1314 %}
1315 ins_pipe(pipe_slow);
1316 %}
1317
1318 // vector fnmla
1319
1320 // dst_src1 = -dst_src1 + -src2 * src3
1321 // dst_src1 = -dst_src1 + src2 * -src3
1322 instruct vfnmlaF(vReg dst_src1, vReg src2, vReg src3) %{
1323 predicate(UseFMA && UseSVE > 0);
1324 match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary (NegVF src2) src3)));
1325 match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 (NegVF src3))));
1326 ins_cost(SVE_COST);
1327 format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1328 ins_encode %{
1329 __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ S,
1330 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1331 %}
1332 ins_pipe(pipe_slow);
1333 %}
1334
1335 // dst_src1 = -dst_src1 + -src2 * src3
1336 // dst_src1 = -dst_src1 + src2 * -src3
1337 instruct vfnmlaD(vReg dst_src1, vReg src2, vReg src3) %{
1338 predicate(UseFMA && UseSVE > 0);
1339 match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary (NegVD src2) src3)));
1340 match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 (NegVD src3))));
1341 ins_cost(SVE_COST);
1342 format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1343 ins_encode %{
1344 __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ D,
1345 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1346 %}
1347 ins_pipe(pipe_slow);
1348 %}
1349
1350 // vector fnmls
1351
1352 // dst_src1 = -dst_src1 + src2 * src3
1353 instruct vfnmlsF(vReg dst_src1, vReg src2, vReg src3) %{
1354 predicate(UseFMA && UseSVE > 0);
1355 match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 src3)));
1356 ins_cost(SVE_COST);
1357 format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1358 ins_encode %{
1359 __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ S,
1360 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1361 %}
1362 ins_pipe(pipe_slow);
1363 %}
1364
1365 // dst_src1 = -dst_src1 + src2 * src3
1366 instruct vfnmlsD(vReg dst_src1, vReg src2, vReg src3) %{
1367 predicate(UseFMA && UseSVE > 0);
1368 match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 src3)));
1369 ins_cost(SVE_COST);
1370 format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1371 ins_encode %{
1372 __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ D,
1373 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1374 %}
1375 ins_pipe(pipe_slow);
1376 %}
1377
1378 // vector mla
1379
1380 // dst_src1 = dst_src1 + src2 * src3
1381 instruct vmlaB(vReg dst_src1, vReg src2, vReg src3)
1382 %{
1383 predicate(UseSVE > 0);
1384 match(Set dst_src1 (AddVB dst_src1 (MulVB src2 src3)));
1385 ins_cost(SVE_COST);
1386 format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (B)" %}
1387 ins_encode %{
1388 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ B,
1389 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1390 %}
1391 ins_pipe(pipe_slow);
1392 %}
1393
1394 // dst_src1 = dst_src1 + src2 * src3
1395 instruct vmlaS(vReg dst_src1, vReg src2, vReg src3)
1396 %{
1397 predicate(UseSVE > 0);
1398 match(Set dst_src1 (AddVS dst_src1 (MulVS src2 src3)));
1399 ins_cost(SVE_COST);
1400 format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (H)" %}
1401 ins_encode %{
1402 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ H,
1403 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1404 %}
1405 ins_pipe(pipe_slow);
1406 %}
1407
1408 // dst_src1 = dst_src1 + src2 * src3
1409 instruct vmlaI(vReg dst_src1, vReg src2, vReg src3)
1410 %{
1411 predicate(UseSVE > 0);
1412 match(Set dst_src1 (AddVI dst_src1 (MulVI src2 src3)));
1413 ins_cost(SVE_COST);
1414 format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (S)" %}
1415 ins_encode %{
1416 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ S,
1417 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1418 %}
1419 ins_pipe(pipe_slow);
1420 %}
1421
1422 // dst_src1 = dst_src1 + src2 * src3
1423 instruct vmlaL(vReg dst_src1, vReg src2, vReg src3)
1424 %{
1425 predicate(UseSVE > 0);
1426 match(Set dst_src1 (AddVL dst_src1 (MulVL src2 src3)));
1427 ins_cost(SVE_COST);
1428 format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (D)" %}
1429 ins_encode %{
1430 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ D,
1431 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1432 %}
1433 ins_pipe(pipe_slow);
1434 %}
1435
1436 // vector mls
1437
1438 // dst_src1 = dst_src1 - src2 * src3
1439 instruct vmlsB(vReg dst_src1, vReg src2, vReg src3)
1440 %{
1441 predicate(UseSVE > 0);
1442 match(Set dst_src1 (SubVB dst_src1 (MulVB src2 src3)));
1443 ins_cost(SVE_COST);
1444 format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (B)" %}
1445 ins_encode %{
1446 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ B,
1447 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1448 %}
1449 ins_pipe(pipe_slow);
1450 %}
1451
1452 // dst_src1 = dst_src1 - src2 * src3
1453 instruct vmlsS(vReg dst_src1, vReg src2, vReg src3)
1454 %{
1455 predicate(UseSVE > 0);
1456 match(Set dst_src1 (SubVS dst_src1 (MulVS src2 src3)));
1457 ins_cost(SVE_COST);
1458 format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (H)" %}
1459 ins_encode %{
1460 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ H,
1461 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1462 %}
1463 ins_pipe(pipe_slow);
1464 %}
1465
1466 // dst_src1 = dst_src1 - src2 * src3
1467 instruct vmlsI(vReg dst_src1, vReg src2, vReg src3)
1468 %{
1469 predicate(UseSVE > 0);
1470 match(Set dst_src1 (SubVI dst_src1 (MulVI src2 src3)));
1471 ins_cost(SVE_COST);
1472 format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (S)" %}
1473 ins_encode %{
1474 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ S,
1475 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1476 %}
1477 ins_pipe(pipe_slow);
1478 %}
1479
1480 // dst_src1 = dst_src1 - src2 * src3
1481 instruct vmlsL(vReg dst_src1, vReg src2, vReg src3)
1482 %{
1483 predicate(UseSVE > 0);
1484 match(Set dst_src1 (SubVL dst_src1 (MulVL src2 src3)));
1485 ins_cost(SVE_COST);
1486 format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (D)" %}
1487 ins_encode %{
1488 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ D,
1489 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1490 %}
1491 ins_pipe(pipe_slow);
1492 %}
1493
1494 // vector mul
1495
1496 instruct vmulB(vReg dst_src1, vReg src2) %{
1497 predicate(UseSVE > 0);
1498 match(Set dst_src1 (MulVB dst_src1 src2));
1499 ins_cost(SVE_COST);
1500 format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (B)" %}
1501 ins_encode %{
1502 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ B,
1503 ptrue, as_FloatRegister($src2$$reg));
1504 %}
1505 ins_pipe(pipe_slow);
1506 %}
1507
1508 instruct vmulS(vReg dst_src1, vReg src2) %{
1509 predicate(UseSVE > 0);
1510 match(Set dst_src1 (MulVS dst_src1 src2));
1511 ins_cost(SVE_COST);
1512 format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (H)" %}
1513 ins_encode %{
1514 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ H,
1515 ptrue, as_FloatRegister($src2$$reg));
1516 %}
1517 ins_pipe(pipe_slow);
1518 %}
1519
1520 instruct vmulI(vReg dst_src1, vReg src2) %{
1521 predicate(UseSVE > 0);
1522 match(Set dst_src1 (MulVI dst_src1 src2));
1523 ins_cost(SVE_COST);
1524 format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (S)" %}
1525 ins_encode %{
1526 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ S,
1527 ptrue, as_FloatRegister($src2$$reg));
1528 %}
1529 ins_pipe(pipe_slow);
1530 %}
1531
1532 instruct vmulL(vReg dst_src1, vReg src2) %{
1533 predicate(UseSVE > 0);
1534 match(Set dst_src1 (MulVL dst_src1 src2));
1535 ins_cost(SVE_COST);
1536 format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (D)" %}
1537 ins_encode %{
1538 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ D,
1539 ptrue, as_FloatRegister($src2$$reg));
1540 %}
1541 ins_pipe(pipe_slow);
1542 %}
1543
1544 instruct vmulF(vReg dst, vReg src1, vReg src2) %{
1545 predicate(UseSVE > 0);
1546 match(Set dst (MulVF src1 src2));
1547 ins_cost(SVE_COST);
1548 format %{ "sve_fmul $dst, $src1, $src2\t # vector (sve) (S)" %}
1549 ins_encode %{
1550 __ sve_fmul(as_FloatRegister($dst$$reg), __ S,
1551 as_FloatRegister($src1$$reg),
1552 as_FloatRegister($src2$$reg));
1553 %}
1554 ins_pipe(pipe_slow);
1555 %}
1556
1557 instruct vmulD(vReg dst, vReg src1, vReg src2) %{
1558 predicate(UseSVE > 0);
1559 match(Set dst (MulVD src1 src2));
1560 ins_cost(SVE_COST);
1561 format %{ "sve_fmul $dst, $src1, $src2\t # vector (sve) (D)" %}
1562 ins_encode %{
1563 __ sve_fmul(as_FloatRegister($dst$$reg), __ D,
1564 as_FloatRegister($src1$$reg),
1565 as_FloatRegister($src2$$reg));
1566 %}
1567 ins_pipe(pipe_slow);
1568 %}
1569
1570 // vector mul - predicated
1571
1572 instruct vmulB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1573 predicate(UseSVE > 0);
1574 match(Set dst_src1 (MulVB (Binary dst_src1 src2) pg));
1575 ins_cost(SVE_COST);
1576 format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
1577 ins_encode %{
1578 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ B,
1579 as_PRegister($pg$$reg),
1580 as_FloatRegister($src2$$reg));
1581 %}
1582 ins_pipe(pipe_slow);
1583 %}
1584
1585 instruct vmulS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1586 predicate(UseSVE > 0);
1587 match(Set dst_src1 (MulVS (Binary dst_src1 src2) pg));
1588 ins_cost(SVE_COST);
1589 format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
1590 ins_encode %{
1591 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ H,
1592 as_PRegister($pg$$reg),
1593 as_FloatRegister($src2$$reg));
1594 %}
1595 ins_pipe(pipe_slow);
1596 %}
1597
1598 instruct vmulI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1599 predicate(UseSVE > 0);
1600 match(Set dst_src1 (MulVI (Binary dst_src1 src2) pg));
1601 ins_cost(SVE_COST);
1602 format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
1603 ins_encode %{
1604 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ S,
1605 as_PRegister($pg$$reg),
1606 as_FloatRegister($src2$$reg));
1607 %}
1608 ins_pipe(pipe_slow);
1609 %}
1610
1611 instruct vmulL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1612 predicate(UseSVE > 0);
1613 match(Set dst_src1 (MulVL (Binary dst_src1 src2) pg));
1614 ins_cost(SVE_COST);
1615 format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
1616 ins_encode %{
1617 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ D,
1618 as_PRegister($pg$$reg),
1619 as_FloatRegister($src2$$reg));
1620 %}
1621 ins_pipe(pipe_slow);
1622 %}
1623
1624 instruct vmulF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1625 predicate(UseSVE > 0);
1626 match(Set dst_src1 (MulVF (Binary dst_src1 src2) pg));
1627 ins_cost(SVE_COST);
1628 format %{ "sve_fmul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
1629 ins_encode %{
1630 __ sve_fmul(as_FloatRegister($dst_src1$$reg), __ S,
1631 as_PRegister($pg$$reg),
1632 as_FloatRegister($src2$$reg));
1633 %}
1634 ins_pipe(pipe_slow);
1635 %}
1636
1637 instruct vmulD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1638 predicate(UseSVE > 0);
1639 match(Set dst_src1 (MulVD (Binary dst_src1 src2) pg));
1640 ins_cost(SVE_COST);
1641 format %{ "sve_fmul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
1642 ins_encode %{
1643 __ sve_fmul(as_FloatRegister($dst_src1$$reg), __ D,
1644 as_PRegister($pg$$reg),
1645 as_FloatRegister($src2$$reg));
1646 %}
1647 ins_pipe(pipe_slow);
1648 %}
1649
1650 // vector fneg
1651
1652 instruct vnegF(vReg dst, vReg src) %{
1653 predicate(UseSVE > 0 &&
1654 !n->as_Vector()->is_predicated_vector());
1655 match(Set dst (NegVF src));
1656 ins_cost(SVE_COST);
1657 format %{ "sve_fneg $dst, $src\t# vector (sve) (S)" %}
1658 ins_encode %{
1659 __ sve_fneg(as_FloatRegister($dst$$reg), __ S,
1660 ptrue, as_FloatRegister($src$$reg));
1661 %}
1662 ins_pipe(pipe_slow);
1663 %}
1664
1665 instruct vnegD(vReg dst, vReg src) %{
1666 predicate(UseSVE > 0 &&
1667 !n->as_Vector()->is_predicated_vector());
1668 match(Set dst (NegVD src));
1669 ins_cost(SVE_COST);
1670 format %{ "sve_fneg $dst, $src\t# vector (sve) (D)" %}
1671 ins_encode %{
1672 __ sve_fneg(as_FloatRegister($dst$$reg), __ D,
1673 ptrue, as_FloatRegister($src$$reg));
1674 %}
1675 ins_pipe(pipe_slow);
1676 %}
1677
1678 // vector fneg - predicated
1679
1680 instruct vnegF_masked(vReg dst_src, pRegGov pg) %{
1681 predicate(UseSVE > 0);
1682 match(Set dst_src (NegVF dst_src pg));
1683 ins_cost(SVE_COST);
1684 format %{ "sve_fneg $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
1685 ins_encode %{
1686 __ sve_fneg(as_FloatRegister($dst_src$$reg), __ S,
1687 as_PRegister($pg$$reg),
1688 as_FloatRegister($dst_src$$reg));
1689 %}
1690 ins_pipe(pipe_slow);
1691 %}
1692
1693 instruct vnegD_masked(vReg dst_src, pRegGov pg) %{
1694 predicate(UseSVE > 0);
1695 match(Set dst_src (NegVD dst_src pg));
1696 ins_cost(SVE_COST);
1697 format %{ "sve_fneg $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
1698 ins_encode %{
1699 __ sve_fneg(as_FloatRegister($dst_src$$reg), __ D,
1700 as_PRegister($pg$$reg),
1701 as_FloatRegister($dst_src$$reg));
1702 %}
1703 ins_pipe(pipe_slow);
1704 %}
1705
1706 // popcount vector
1707
1708 instruct vpopcountI(vReg dst, vReg src) %{
1709 predicate(UseSVE > 0);
1710 match(Set dst (PopCountVI src));
1711 format %{ "sve_cnt $dst, $src\t# vector (sve) (S)\n\t" %}
1712 ins_encode %{
1713 __ sve_cnt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg));
1714 %}
1715 ins_pipe(pipe_slow);
1716 %}
1717
1718 // vector mask compare
1719
1720 instruct vmaskcmp(pRegGov dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %{
1721 predicate(UseSVE > 0);
1722 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
1723 effect(KILL cr);
1724 ins_cost(SVE_COST);
1725 format %{ "sve_cmp $dst, $src1, $src2\t# vector mask cmp (sve)" %}
1726 ins_encode %{
1727 BasicType bt = Matcher::vector_element_basic_type(this);
1728 __ sve_compare(as_PRegister($dst$$reg), bt, ptrue, as_FloatRegister($src1$$reg),
1729 as_FloatRegister($src2$$reg), (int)$cond$$constant);
1730 %}
1731 ins_pipe(pipe_slow);
1732 %}
1733
1734 instruct vmaskcmp_masked(pRegGov dst, vReg src1, vReg src2, immI cond, pRegGov pg, rFlagsReg cr) %{
1735 predicate(UseSVE > 0);
1736 match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond pg)));
1737 effect(KILL cr);
1738 ins_cost(SVE_COST);
1739 format %{ "sve_cmp $dst, $pg, $src1, $src2\t# vector mask cmp (sve)" %}
1740 ins_encode %{
1741 BasicType bt = Matcher::vector_element_basic_type(this);
1742 __ sve_compare(as_PRegister($dst$$reg), bt, as_PRegister($pg$$reg), as_FloatRegister($src1$$reg),
1743 as_FloatRegister($src2$$reg), (int)$cond$$constant);
1744 %}
1745 ins_pipe(pipe_slow);
1746 %}
1747
1748 // vector blend
1749
1750 instruct vblend(vReg dst, vReg src1, vReg src2, pRegGov pg) %{
1751 predicate(UseSVE > 0);
1752 match(Set dst (VectorBlend (Binary src1 src2) pg));
1753 ins_cost(SVE_COST);
1754 format %{ "sve_sel $dst, $pg, $src2, $src1\t# vector blend (sve)" %}
1755 ins_encode %{
1756 Assembler::SIMD_RegVariant size =
1757 __ elemType_to_regVariant(Matcher::vector_element_basic_type(this));
1758 __ sve_sel(as_FloatRegister($dst$$reg), size, as_PRegister($pg$$reg),
1759 as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
1760 %}
1761 ins_pipe(pipe_slow);
1762 %}
1763
1764 // vector load mask
1765
1766 instruct vloadmaskB(pRegGov dst, vReg src, rFlagsReg cr) %{
1767 predicate(UseSVE > 0 &&
1768 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
1769 match(Set dst (VectorLoadMask src));
1770 effect(KILL cr);
1771 ins_cost(SVE_COST);
1772 format %{ "vloadmaskB $dst, $src\t# vector load mask (sve) (B)" %}
1773 ins_encode %{
1774 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ B,
1775 ptrue, as_FloatRegister($src$$reg), 0);
1776 %}
1777 ins_pipe(pipe_slow);
1778 %}
1779
1780 instruct vloadmask_extend(pRegGov dst, vReg src, vReg tmp, rFlagsReg cr) %{
1781 predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() != T_BYTE);
1782 match(Set dst (VectorLoadMask src));
1783 effect(TEMP tmp, KILL cr);
1784 ins_cost(3 * SVE_COST);
1785 format %{ "vloadmask $dst, $src\t# vector load mask (sve) (H/S/D)" %}
1786 ins_encode %{
1787 BasicType bt = Matcher::vector_element_basic_type(this);
1788 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1789 __ sve_vector_extend(as_FloatRegister($tmp$$reg), size, as_FloatRegister($src$$reg), __ B);
1790 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
1791 %}
1792 ins_pipe(pipe_slow);
1793 %}
1794
1795 // vector store mask
1796
1797 instruct vstoremaskB(vReg dst, pRegGov src, immI_1 size) %{
1798 predicate(UseSVE > 0);
1799 match(Set dst (VectorStoreMask src size));
1800 ins_cost(SVE_COST);
1801 format %{ "vstoremask $dst, $src\t# vector store mask (sve) (B)" %}
1802 ins_encode %{
1803 __ sve_cpy(as_FloatRegister($dst$$reg), __ B, as_PRegister($src$$reg), 1, false);
1804 %}
1805 ins_pipe(pipe_slow);
1806 %}
1807
1808 instruct vstoremask_narrow(vReg dst, pRegGov src, vReg tmp, immI_gt_1 size) %{
1809 predicate(UseSVE > 0);
1810 match(Set dst (VectorStoreMask src size));
1811 effect(TEMP_DEF dst, TEMP tmp);
1812 ins_cost(3 * SVE_COST);
1813 format %{ "vstoremask $dst, $src\t# vector store mask (sve) (H/S/D)" %}
1814 ins_encode %{
1815 Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant((int)$size$$constant);
1816 __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($src$$reg), 1, false);
1817 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
1818 as_FloatRegister($dst$$reg), size, as_FloatRegister($tmp$$reg));
1819 %}
1820 ins_pipe(pipe_slow);
1821 %}
1822
1823 // Combine LoadVector+VectorLoadMask when the vector element type is not T_BYTE
1824
1825 instruct vloadmask_loadV(pRegGov dst, indirect mem, vReg tmp, rFlagsReg cr) %{
1826 predicate(UseSVE > 0 &&
1827 n->as_Vector()->length_in_bytes() == MaxVectorSize &&
1828 type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
1829 match(Set dst (VectorLoadMask (LoadVector mem)));
1830 effect(TEMP tmp, KILL cr);
1831 ins_cost(3 * SVE_COST);
1832 format %{ "sve_ld1b $tmp, $mem\n\t"
1833 "sve_cmpne $dst, $tmp, 0\t# load vector mask (sve) (H/S/D)" %}
1834 ins_encode %{
1835 // Load mask values which are boolean type, and extend them to the
1836 // expected vector element type. Convert the vector to predicate.
1837 BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
1838 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($tmp$$reg),
1839 ptrue, T_BOOLEAN, to_vect_bt, $mem->opcode(),
1840 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1841 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ elemType_to_regVariant(to_vect_bt),
1842 ptrue, as_FloatRegister($tmp$$reg), 0);
1843 %}
1844 ins_pipe(pipe_slow);
1845 %}
1846
1847 instruct vloadmask_loadV_partial(pRegGov dst, indirect mem, vReg vtmp, pRegGov ptmp, rFlagsReg cr) %{
1848 predicate(UseSVE > 0 &&
1849 n->as_Vector()->length_in_bytes() > 16 &&
1850 n->as_Vector()->length_in_bytes() < MaxVectorSize &&
1851 type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
1852 match(Set dst (VectorLoadMask (LoadVector mem)));
1853 effect(TEMP vtmp, TEMP ptmp, KILL cr);
1854 ins_cost(6 * SVE_COST);
1855 format %{ "vloadmask_loadV $dst, $mem\t# load vector mask partial (sve) (H/S/D)" %}
1856 ins_encode %{
1857 // Load valid mask values which are boolean type, and extend them to the
1858 // expected vector element type. Convert the vector to predicate.
1859 BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
1860 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(to_vect_bt);
1861 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this));
1862 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($vtmp$$reg),
1863 as_PRegister($ptmp$$reg), T_BOOLEAN, to_vect_bt, $mem->opcode(),
1864 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1865 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($vtmp$$reg), 0);
1866 %}
1867 ins_pipe(pipe_slow);
1868 %}
1869
1870 // Combine VectorStoreMask+StoreVector when the vector element type is not T_BYTE
1871
1872 instruct storeV_vstoremask(indirect mem, pRegGov src, vReg tmp, immI_gt_1 esize) %{
1873 predicate(UseSVE > 0 &&
1874 Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
1875 match(Set mem (StoreVector mem (VectorStoreMask src esize)));
1876 effect(TEMP tmp);
1877 ins_cost(3 * SVE_COST);
1878 format %{ "sve_cpy $tmp, $src, 1\n\t"
1879 "sve_st1b $tmp, $mem\t# store vector mask (sve) (H/S/D)" %}
1880 ins_encode %{
1881 BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
1882 assert(type2aelembytes(from_vect_bt) == (int)$esize$$constant, "unsupported type.");
1883 Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
1884 __ sve_cpy(as_FloatRegister($tmp$$reg), size, as_PRegister($src$$reg), 1, false);
1885 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($tmp$$reg),
1886 ptrue, T_BOOLEAN, from_vect_bt, $mem->opcode(),
1887 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1888 %}
1889 ins_pipe(pipe_slow);
1890 %}
1891
1892 instruct storeV_vstoremask_partial(indirect mem, pRegGov src, vReg vtmp,
1893 immI_gt_1 esize, pRegGov ptmp, rFlagsReg cr) %{
1894 predicate(UseSVE > 0 &&
1895 n->as_StoreVector()->memory_size() > 16 &&
1896 type2aelembytes(n->as_StoreVector()->vect_type()->element_basic_type()) > 1 &&
1897 Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
1898 match(Set mem (StoreVector mem (VectorStoreMask src esize)));
1899 effect(TEMP vtmp, TEMP ptmp, KILL cr);
1900 format %{ "storeV_vstoremask $src, $mem\t# store vector mask partial (sve) (H/S/D)" %}
1901 ins_cost(6 * SVE_COST);
1902 ins_encode %{
1903 // Convert the valid src predicate to vector, and store the vector
1904 // elements as boolean values.
1905 BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
1906 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(from_vect_bt);
1907 __ sve_cpy(as_FloatRegister($vtmp$$reg), size, as_PRegister($src$$reg), 1, false);
1908 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
1909 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($vtmp$$reg),
1910 as_PRegister($ptmp$$reg), T_BOOLEAN, from_vect_bt, $mem->opcode(),
1911 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1912 %}
1913 ins_pipe(pipe_slow);
1914 %}
1915
1916 // vector add reduction
1917
1918 instruct reduce_addI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
1919 predicate(UseSVE > 0 &&
1920 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1921 match(Set dst (AddReductionVI src1 src2));
1922 effect(TEMP_DEF dst, TEMP tmp);
1923 ins_cost(SVE_COST);
1924 format %{ "sve_reduce_addI $dst, $src1, $src2\t# addI reduction (sve) (may extend)" %}
1925 ins_encode %{
1926 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1927 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1928 $src1$$Register, as_FloatRegister($src2$$reg),
1929 ptrue, as_FloatRegister($tmp$$reg));
1930 %}
1931 ins_pipe(pipe_slow);
1932 %}
1933
1934 instruct reduce_addL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
1935 predicate(UseSVE > 0 &&
1936 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1937 match(Set dst (AddReductionVL src1 src2));
1938 effect(TEMP_DEF dst, TEMP tmp);
1939 ins_cost(SVE_COST);
1940 format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction (sve)" %}
1941 ins_encode %{
1942 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1943 $src1$$Register, as_FloatRegister($src2$$reg),
1944 ptrue, as_FloatRegister($tmp$$reg));
1945 %}
1946 ins_pipe(pipe_slow);
1947 %}
1948
1949 instruct reduce_addF(vRegF src1_dst, vReg src2) %{
1950 predicate(UseSVE > 0 &&
1951 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1952 match(Set src1_dst (AddReductionVF src1_dst src2));
1953 ins_cost(SVE_COST);
1954 format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (S)" %}
1955 ins_encode %{
1956 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
1957 ptrue, as_FloatRegister($src2$$reg));
1958 %}
1959 ins_pipe(pipe_slow);
1960 %}
1961
1962 instruct reduce_addD(vRegD src1_dst, vReg src2) %{
1963 predicate(UseSVE > 0 &&
1964 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1965 match(Set src1_dst (AddReductionVD src1_dst src2));
1966 ins_cost(SVE_COST);
1967 format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (D)" %}
1968 ins_encode %{
1969 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
1970 ptrue, as_FloatRegister($src2$$reg));
1971 %}
1972 ins_pipe(pipe_slow);
1973 %}
1974
1975 instruct reduce_addI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1976 pRegGov ptmp, rFlagsReg cr) %{
1977 predicate(UseSVE > 0 &&
1978 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1979 match(Set dst (AddReductionVI src1 src2));
1980 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1981 ins_cost(2 * SVE_COST);
1982 format %{ "sve_reduce_addI $dst, $src1, $src2\t# addI reduction partial (sve) (may extend)" %}
1983 ins_encode %{
1984 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1985 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1986 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1987 Matcher::vector_length(this, $src2));
1988 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1989 $src1$$Register, as_FloatRegister($src2$$reg),
1990 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1991 %}
1992 ins_pipe(pipe_slow);
1993 %}
1994
1995 instruct reduce_addL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1996 pRegGov ptmp, rFlagsReg cr) %{
1997 predicate(UseSVE > 0 &&
1998 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1999 match(Set dst (AddReductionVL src1 src2));
2000 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2001 ins_cost(2 * SVE_COST);
2002 format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction partial (sve)" %}
2003 ins_encode %{
2004 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2005 Matcher::vector_length(this, $src2));
2006 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2007 $src1$$Register, as_FloatRegister($src2$$reg),
2008 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2009 %}
2010 ins_pipe(pipe_slow);
2011 %}
2012
2013 instruct reduce_addF_partial(vRegF src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
2014 predicate(UseSVE > 0 &&
2015 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2016 match(Set src1_dst (AddReductionVF src1_dst src2));
2017 ins_cost(SVE_COST);
2018 effect(TEMP ptmp, KILL cr);
2019 format %{ "sve_reduce_addF $src1_dst, $src1_dst, $src2\t# addF reduction partial (sve) (S)" %}
2020 ins_encode %{
2021 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2022 Matcher::vector_length(this, $src2));
2023 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
2024 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2025 %}
2026 ins_pipe(pipe_slow);
2027 %}
2028
2029 instruct reduce_addD_partial(vRegD src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
2030 predicate(UseSVE > 0 &&
2031 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2032 match(Set src1_dst (AddReductionVD src1_dst src2));
2033 ins_cost(SVE_COST);
2034 effect(TEMP ptmp, KILL cr);
2035 format %{ "sve_reduce_addD $src1_dst, $src1_dst, $src2\t# addD reduction partial (sve) (D)" %}
2036 ins_encode %{
2037 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2038 Matcher::vector_length(this, $src2));
2039 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
2040 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2041 %}
2042 ins_pipe(pipe_slow);
2043 %}
2044
2045 // vector add reduction - predicated
2046
2047 instruct reduce_addI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2048 predicate(UseSVE > 0 &&
2049 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2050 match(Set dst (AddReductionVI (Binary src1 src2) pg));
2051 effect(TEMP_DEF dst, TEMP tmp);
2052 ins_cost(SVE_COST);
2053 format %{ "sve_reduce_addI $dst, $src1, $pg, $src2\t# addI reduction predicated (sve) (may extend)" %}
2054 ins_encode %{
2055 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2056 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2057 $src1$$Register, as_FloatRegister($src2$$reg),
2058 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2059 %}
2060 ins_pipe(pipe_slow);
2061 %}
2062
2063 instruct reduce_addL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2064 predicate(UseSVE > 0 &&
2065 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2066 match(Set dst (AddReductionVL (Binary src1 src2) pg));
2067 effect(TEMP_DEF dst, TEMP tmp);
2068 ins_cost(SVE_COST);
2069 format %{ "sve_reduce_addL $dst, $src1, $pg, $src2\t# addL reduction predicated (sve)" %}
2070 ins_encode %{
2071 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2072 $src1$$Register, as_FloatRegister($src2$$reg),
2073 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2074 %}
2075 ins_pipe(pipe_slow);
2076 %}
2077
2078 instruct reduce_addF_masked(vRegF src1_dst, vReg src2, pRegGov pg) %{
2079 predicate(UseSVE > 0 &&
2080 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2081 match(Set src1_dst (AddReductionVF (Binary src1_dst src2) pg));
2082 ins_cost(SVE_COST);
2083 format %{ "sve_reduce_addF $src1_dst, $pg, $src2\t# addF reduction predicated (sve)" %}
2084 ins_encode %{
2085 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
2086 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
2087 %}
2088 ins_pipe(pipe_slow);
2089 %}
2090
2091 instruct reduce_addD_masked(vRegD src1_dst, vReg src2, pRegGov pg) %{
2092 predicate(UseSVE > 0 &&
2093 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2094 match(Set src1_dst (AddReductionVD (Binary src1_dst src2) pg));
2095 ins_cost(SVE_COST);
2096 format %{ "sve_reduce_addD $src1_dst, $pg, $src2\t# addD reduction predicated (sve)" %}
2097 ins_encode %{
2098 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
2099 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
2100 %}
2101 ins_pipe(pipe_slow);
2102 %}
2103
2104 instruct reduce_addI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2105 pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2106 predicate(UseSVE > 0 &&
2107 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2108 match(Set dst (AddReductionVI (Binary src1 src2) pg));
2109 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2110 ins_cost(3 * SVE_COST);
2111 format %{ "sve_reduce_addI $dst, $src1, $pg, $src2\t# addI reduction predicated partial (sve) (may extend)" %}
2112 ins_encode %{
2113 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2114 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2115 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2116 Matcher::vector_length(this, $src2));
2117 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2118 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2119 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2120 $src1$$Register, as_FloatRegister($src2$$reg),
2121 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2122 %}
2123 ins_pipe(pipe_slow);
2124 %}
2125
2126 instruct reduce_addL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2127 pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2128 predicate(UseSVE > 0 &&
2129 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2130 match(Set dst (AddReductionVL (Binary src1 src2) pg));
2131 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2132 ins_cost(3 * SVE_COST);
2133 format %{ "sve_reduce_addL $dst, $src1, $pg, $src2\t# addL reduction predicated partial (sve)" %}
2134 ins_encode %{
2135 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2136 Matcher::vector_length(this, $src2));
2137 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2138 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2139 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2140 $src1$$Register, as_FloatRegister($src2$$reg),
2141 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2142 %}
2143 ins_pipe(pipe_slow);
2144 %}
2145
2146 instruct reduce_addF_masked_partial(vRegF src1_dst, vReg src2, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2147 predicate(UseSVE > 0 &&
2148 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2149 match(Set src1_dst (AddReductionVF (Binary src1_dst src2) pg));
2150 effect(TEMP ptmp, KILL cr);
2151 ins_cost(SVE_COST);
2152 format %{ "sve_reduce_addF $src1_dst, $pg, $src2\t# addF reduction predicated partial (sve)" %}
2153 ins_encode %{
2154 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2155 Matcher::vector_length(this, $src2));
2156 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2157 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2158 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
2159 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2160 %}
2161 ins_pipe(pipe_slow);
2162 %}
2163
2164 instruct reduce_addD_masked_partial(vRegD src1_dst, vReg src2, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2165 predicate(UseSVE > 0 &&
2166 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2167 match(Set src1_dst (AddReductionVD (Binary src1_dst src2) pg));
2168 effect(TEMP ptmp, KILL cr);
2169 ins_cost(SVE_COST);
2170 format %{ "sve_reduce_addD $src1_dst, $pg, $src2\t# addD reduction predicated partial (sve)" %}
2171 ins_encode %{
2172 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2173 Matcher::vector_length(this, $src2));
2174 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2175 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2176 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
2177 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2178 %}
2179 ins_pipe(pipe_slow);
2180 %}
2181
2182 // vector and reduction
2183
2184 instruct reduce_andI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
2185 predicate(UseSVE > 0 &&
2186 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2187 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2188 match(Set dst (AndReductionV src1 src2));
2189 effect(TEMP_DEF dst, TEMP tmp);
2190 ins_cost(SVE_COST);
2191 format %{ "sve_reduce_andI $dst, $src1, $src2\t# andI reduction (sve) (may extend)" %}
2192 ins_encode %{
2193 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2194 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2195 $src1$$Register, as_FloatRegister($src2$$reg),
2196 ptrue, as_FloatRegister($tmp$$reg));
2197 %}
2198 ins_pipe(pipe_slow);
2199 %}
2200
2201 instruct reduce_andL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
2202 predicate(UseSVE > 0 &&
2203 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2204 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2205 match(Set dst (AndReductionV src1 src2));
2206 effect(TEMP_DEF dst, TEMP tmp);
2207 ins_cost(SVE_COST);
2208 format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction (sve)" %}
2209 ins_encode %{
2210 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2211 $src1$$Register, as_FloatRegister($src2$$reg),
2212 ptrue, as_FloatRegister($tmp$$reg));
2213 %}
2214 ins_pipe(pipe_slow);
2215 %}
2216
2217 instruct reduce_andI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2218 pRegGov ptmp, rFlagsReg cr) %{
2219 predicate(UseSVE > 0 &&
2220 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2221 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2222 match(Set dst (AndReductionV src1 src2));
2223 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2224 ins_cost(2 * SVE_COST);
2225 format %{ "sve_reduce_andI $dst, $src1, $src2\t# andI reduction partial (sve) (may extend)" %}
2226 ins_encode %{
2227 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2228 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2229 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2230 Matcher::vector_length(this, $src2));
2231 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2232 $src1$$Register, as_FloatRegister($src2$$reg),
2233 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2234 %}
2235 ins_pipe(pipe_slow);
2236 %}
2237
2238 instruct reduce_andL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2239 pRegGov ptmp, rFlagsReg cr) %{
2240 predicate(UseSVE > 0 &&
2241 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2242 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2243 match(Set dst (AndReductionV src1 src2));
2244 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2245 ins_cost(2 * SVE_COST);
2246 format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction partial (sve)" %}
2247 ins_encode %{
2248 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2249 Matcher::vector_length(this, $src2));
2250 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2251 $src1$$Register, as_FloatRegister($src2$$reg),
2252 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2253 %}
2254 ins_pipe(pipe_slow);
2255 %}
2256
2257 // vector and reduction - predicated
2258
2259 instruct reduce_andI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2260 predicate(UseSVE > 0 &&
2261 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2262 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2263 match(Set dst (AndReductionV (Binary src1 src2) pg));
2264 effect(TEMP_DEF dst, TEMP tmp);
2265 ins_cost(SVE_COST);
2266 format %{ "sve_reduce_andI $dst, $src1, $pg, $src2\t# andI reduction predicated (sve) (may extend)" %}
2267 ins_encode %{
2268 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2269 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2270 $src1$$Register, as_FloatRegister($src2$$reg),
2271 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2272 %}
2273 ins_pipe(pipe_slow);
2274 %}
2275
2276 instruct reduce_andL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2277 predicate(UseSVE > 0 &&
2278 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2279 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2280 match(Set dst (AndReductionV (Binary src1 src2) pg));
2281 effect(TEMP_DEF dst, TEMP tmp);
2282 ins_cost(SVE_COST);
2283 format %{ "sve_reduce_andL $dst, $src1, $pg, $src2\t# andL reduction predicated (sve)" %}
2284 ins_encode %{
2285 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2286 $src1$$Register, as_FloatRegister($src2$$reg),
2287 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2288 %}
2289 ins_pipe(pipe_slow);
2290 %}
2291
2292 instruct reduce_andI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2293 pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2294 predicate(UseSVE > 0 &&
2295 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2296 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2297 match(Set dst (AndReductionV (Binary src1 src2) pg));
2298 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2299 ins_cost(3 * SVE_COST);
2300 format %{ "sve_reduce_andI $dst, $src1, $pg, $src2\t# andI reduction predicated partial (sve) (may extend)" %}
2301 ins_encode %{
2302 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2303 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2304 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2305 Matcher::vector_length(this, $src2));
2306 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2307 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2308 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2309 $src1$$Register, as_FloatRegister($src2$$reg),
2310 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2311 %}
2312 ins_pipe(pipe_slow);
2313 %}
2314
2315 instruct reduce_andL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2316 pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2317 predicate(UseSVE > 0 &&
2318 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2319 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2320 match(Set dst (AndReductionV (Binary src1 src2) pg));
2321 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2322 ins_cost(3 * SVE_COST);
2323 format %{ "sve_reduce_andL $dst, $src1, $pg, $src2\t# andL reduction predicated partial (sve)" %}
2324 ins_encode %{
2325 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2326 Matcher::vector_length(this, $src2));
2327 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2328 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2329 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2330 $src1$$Register, as_FloatRegister($src2$$reg),
2331 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2332 %}
2333 ins_pipe(pipe_slow);
2334 %}
2335
2336 // vector or reduction
2337
2338 instruct reduce_orI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
2339 predicate(UseSVE > 0 &&
2340 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2341 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2342 match(Set dst (OrReductionV src1 src2));
2343 effect(TEMP_DEF dst, TEMP tmp);
2344 ins_cost(SVE_COST);
2345 format %{ "sve_reduce_orI $dst, $src1, $src2\t# orI reduction (sve) (may extend)" %}
2346 ins_encode %{
2347 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2348 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2349 $src1$$Register, as_FloatRegister($src2$$reg),
2350 ptrue, as_FloatRegister($tmp$$reg));
2351 %}
2352 ins_pipe(pipe_slow);
2353 %}
2354
2355 instruct reduce_orL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
2356 predicate(UseSVE > 0 &&
2357 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2358 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2359 match(Set dst (OrReductionV src1 src2));
2360 effect(TEMP_DEF dst, TEMP tmp);
2361 ins_cost(SVE_COST);
2362 format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction (sve)" %}
2363 ins_encode %{
2364 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2365 $src1$$Register, as_FloatRegister($src2$$reg),
2366 ptrue, as_FloatRegister($tmp$$reg));
2367 %}
2368 ins_pipe(pipe_slow);
2369 %}
2370
2371 instruct reduce_orI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2372 pRegGov ptmp, rFlagsReg cr) %{
2373 predicate(UseSVE > 0 &&
2374 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2375 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2376 match(Set dst (OrReductionV src1 src2));
2377 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2378 ins_cost(2 * SVE_COST);
2379 format %{ "sve_reduce_orI $dst, $src1, $src2\t# orI reduction partial (sve) (may extend)" %}
2380 ins_encode %{
2381 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2382 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2383 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2384 Matcher::vector_length(this, $src2));
2385 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2386 $src1$$Register, as_FloatRegister($src2$$reg),
2387 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2388 %}
2389 ins_pipe(pipe_slow);
2390 %}
2391
2392 instruct reduce_orL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2393 pRegGov ptmp, rFlagsReg cr) %{
2394 predicate(UseSVE > 0 &&
2395 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2396 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2397 match(Set dst (OrReductionV src1 src2));
2398 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2399 ins_cost(2 * SVE_COST);
2400 format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction partial (sve)" %}
2401 ins_encode %{
2402 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2403 Matcher::vector_length(this, $src2));
2404 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2405 $src1$$Register, as_FloatRegister($src2$$reg),
2406 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2407 %}
2408 ins_pipe(pipe_slow);
2409 %}
2410
2411 // vector or reduction - predicated
2412
2413 instruct reduce_orI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2414 predicate(UseSVE > 0 &&
2415 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2416 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2417 match(Set dst (OrReductionV (Binary src1 src2) pg));
2418 effect(TEMP_DEF dst, TEMP tmp);
2419 ins_cost(SVE_COST);
2420 format %{ "sve_reduce_orI $dst, $src1, $pg, $src2\t# orI reduction predicated (sve) (may extend)" %}
2421 ins_encode %{
2422 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2423 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2424 $src1$$Register, as_FloatRegister($src2$$reg),
2425 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2426 %}
2427 ins_pipe(pipe_slow);
2428 %}
2429
2430 instruct reduce_orL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2431 predicate(UseSVE > 0 &&
2432 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2433 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2434 match(Set dst (OrReductionV (Binary src1 src2) pg));
2435 effect(TEMP_DEF dst, TEMP tmp);
2436 ins_cost(SVE_COST);
2437 format %{ "sve_reduce_orL $dst, $src1, $pg, $src2\t# orL reduction predicated (sve)" %}
2438 ins_encode %{
2439 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2440 $src1$$Register, as_FloatRegister($src2$$reg),
2441 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2442 %}
2443 ins_pipe(pipe_slow);
2444 %}
2445
2446 instruct reduce_orI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2447 pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2448 predicate(UseSVE > 0 &&
2449 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2450 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2451 match(Set dst (OrReductionV (Binary src1 src2) pg));
2452 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2453 ins_cost(3 * SVE_COST);
2454 format %{ "sve_reduce_orI $dst, $src1, $pg, $src2\t# orI reduction predicated partial (sve) (may extend)" %}
2455 ins_encode %{
2456 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2457 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2458 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2459 Matcher::vector_length(this, $src2));
2460 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2461 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2462 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2463 $src1$$Register, as_FloatRegister($src2$$reg),
2464 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2465 %}
2466 ins_pipe(pipe_slow);
2467 %}
2468
2469 instruct reduce_orL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2470 pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2471 predicate(UseSVE > 0 &&
2472 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2473 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2474 match(Set dst (OrReductionV (Binary src1 src2) pg));
2475 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2476 ins_cost(3 * SVE_COST);
2477 format %{ "sve_reduce_orL $dst, $src1, $pg, $src2\t# orL reduction predicated partial (sve)" %}
2478 ins_encode %{
2479 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2480 Matcher::vector_length(this, $src2));
2481 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2482 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2483 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2484 $src1$$Register, as_FloatRegister($src2$$reg),
2485 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2486 %}
2487 ins_pipe(pipe_slow);
2488 %}
2489
2490 // vector xor reduction
2491
2492 instruct reduce_eorI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
2493 predicate(UseSVE > 0 &&
2494 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2495 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2496 match(Set dst (XorReductionV src1 src2));
2497 effect(TEMP_DEF dst, TEMP tmp);
2498 ins_cost(SVE_COST);
2499 format %{ "sve_reduce_eorI $dst, $src1, $src2\t# eorI reduction (sve) (may extend)" %}
2500 ins_encode %{
2501 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2502 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2503 $src1$$Register, as_FloatRegister($src2$$reg),
2504 ptrue, as_FloatRegister($tmp$$reg));
2505 %}
2506 ins_pipe(pipe_slow);
2507 %}
2508
2509 instruct reduce_eorL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
2510 predicate(UseSVE > 0 &&
2511 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2512 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2513 match(Set dst (XorReductionV src1 src2));
2514 effect(TEMP_DEF dst, TEMP tmp);
2515 ins_cost(SVE_COST);
2516 format %{ "sve_reduce_eorL $dst, $src1, $src2\t# eorL reduction (sve)" %}
2517 ins_encode %{
2518 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2519 $src1$$Register, as_FloatRegister($src2$$reg),
2520 ptrue, as_FloatRegister($tmp$$reg));
2521 %}
2522 ins_pipe(pipe_slow);
2523 %}
2524
2525 instruct reduce_eorI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2526 pRegGov ptmp, rFlagsReg cr) %{
2527 predicate(UseSVE > 0 &&
2528 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2529 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2530 match(Set dst (XorReductionV src1 src2));
2531 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2532 ins_cost(2 * SVE_COST);
2533 format %{ "sve_reduce_eorI $dst, $src1, $src2\t# eorI reduction partial (sve) (may extend)" %}
2534 ins_encode %{
2535 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2536 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2537 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2538 Matcher::vector_length(this, $src2));
2539 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2540 $src1$$Register, as_FloatRegister($src2$$reg),
2541 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2542 %}
2543 ins_pipe(pipe_slow);
2544 %}
2545
2546 instruct reduce_eorL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2547 pRegGov ptmp, rFlagsReg cr) %{
2548 predicate(UseSVE > 0 &&
2549 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2550 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2551 match(Set dst (XorReductionV src1 src2));
2552 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2553 ins_cost(2 * SVE_COST);
2554 format %{ "sve_reduce_eorL $dst, $src1, $src2\t# eorL reduction partial (sve)" %}
2555 ins_encode %{
2556 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2557 Matcher::vector_length(this, $src2));
2558 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2559 $src1$$Register, as_FloatRegister($src2$$reg),
2560 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2561 %}
2562 ins_pipe(pipe_slow);
2563 %}
2564
2565 // vector xor reduction - predicated
2566
2567 instruct reduce_eorI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2568 predicate(UseSVE > 0 &&
2569 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2570 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2571 match(Set dst (XorReductionV (Binary src1 src2) pg));
2572 effect(TEMP_DEF dst, TEMP tmp);
2573 ins_cost(SVE_COST);
2574 format %{ "sve_reduce_eorI $dst, $src1, $pg, $src2\t# eorI reduction predicated (sve) (may extend)" %}
2575 ins_encode %{
2576 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2577 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2578 $src1$$Register, as_FloatRegister($src2$$reg),
2579 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2580 %}
2581 ins_pipe(pipe_slow);
2582 %}
2583
2584 instruct reduce_eorL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2585 predicate(UseSVE > 0 &&
2586 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2587 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2588 match(Set dst (XorReductionV (Binary src1 src2) pg));
2589 effect(TEMP_DEF dst, TEMP tmp);
2590 ins_cost(SVE_COST);
2591 format %{ "sve_reduce_eorL $dst, $src1, $pg, $src2\t# eorL reduction predicated (sve)" %}
2592 ins_encode %{
2593 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2594 $src1$$Register, as_FloatRegister($src2$$reg),
2595 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2596 %}
2597 ins_pipe(pipe_slow);
2598 %}
2599
2600 instruct reduce_eorI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2601 pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2602 predicate(UseSVE > 0 &&
2603 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2604 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2605 match(Set dst (XorReductionV (Binary src1 src2) pg));
2606 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2607 ins_cost(3 * SVE_COST);
2608 format %{ "sve_reduce_eorI $dst, $src1, $pg, $src2\t# eorI reduction predicated partial (sve) (may extend)" %}
2609 ins_encode %{
2610 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2611 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2612 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2613 Matcher::vector_length(this, $src2));
2614 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2615 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2616 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2617 $src1$$Register, as_FloatRegister($src2$$reg),
2618 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2619 %}
2620 ins_pipe(pipe_slow);
2621 %}
2622
2623 instruct reduce_eorL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2624 pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2625 predicate(UseSVE > 0 &&
2626 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2627 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2628 match(Set dst (XorReductionV (Binary src1 src2) pg));
2629 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2630 ins_cost(3 * SVE_COST);
2631 format %{ "sve_reduce_eorL $dst, $src1, $pg, $src2\t# eorL reduction predicated partial (sve)" %}
2632 ins_encode %{
2633 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2634 Matcher::vector_length(this, $src2));
2635 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2636 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2637 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2638 $src1$$Register, as_FloatRegister($src2$$reg),
2639 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2640 %}
2641 ins_pipe(pipe_slow);
2642 %}
2643
2644 // vector max reduction
2645
2646 instruct reduce_maxI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
2647 predicate(UseSVE > 0 &&
2648 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2649 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2650 is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
2651 match(Set dst (MaxReductionV src1 src2));
2652 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2653 ins_cost(SVE_COST);
2654 format %{ "sve_reduce_maxI $dst, $src1, $src2\t# maxI reduction (sve)" %}
2655 ins_encode %{
2656 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2657 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2658 $src1$$Register, as_FloatRegister($src2$$reg),
2659 ptrue, as_FloatRegister($tmp$$reg));
2660 %}
2661 ins_pipe(pipe_slow);
2662 %}
2663
2664 instruct reduce_maxL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
2665 predicate(UseSVE > 0 &&
2666 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2667 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2668 match(Set dst (MaxReductionV src1 src2));
2669 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2670 ins_cost(SVE_COST);
2671 format %{ "sve_reduce_maxL $dst, $src1, $src2\t# maxL reduction (sve)" %}
2672 ins_encode %{
2673 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2674 $src1$$Register, as_FloatRegister($src2$$reg),
2675 ptrue, as_FloatRegister($tmp$$reg));
2676 %}
2677 ins_pipe(pipe_slow);
2678 %}
2679
2680 instruct reduce_maxI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2681 pRegGov ptmp, rFlagsReg cr) %{
2682 predicate(UseSVE > 0 &&
2683 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2684 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2685 is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
2686 match(Set dst (MaxReductionV src1 src2));
2687 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2688 ins_cost(2 * SVE_COST);
2689 format %{ "sve_reduce_maxI $dst, $src1, $src2\t# maxI reduction partial (sve)" %}
2690 ins_encode %{
2691 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2692 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2693 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2694 Matcher::vector_length(this, $src2));
2695 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2696 $src1$$Register, as_FloatRegister($src2$$reg),
2697 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2698 %}
2699 ins_pipe(pipe_slow);
2700 %}
2701
2702 instruct reduce_maxL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2703 pRegGov ptmp, rFlagsReg cr) %{
2704 predicate(UseSVE > 0 &&
2705 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2706 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2707 match(Set dst (MaxReductionV src1 src2));
2708 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2709 ins_cost(2 * SVE_COST);
2710 format %{ "sve_reduce_maxL $dst, $src1, $src2\t# maxL reduction partial (sve)" %}
2711 ins_encode %{
2712 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2713 Matcher::vector_length(this, $src2));
2714 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2715 $src1$$Register, as_FloatRegister($src2$$reg),
2716 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2717 %}
2718 ins_pipe(pipe_slow);
2719 %}
2720
2721 instruct reduce_maxF(vRegF dst, vRegF src1, vReg src2) %{
2722 predicate(UseSVE > 0 &&
2723 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
2724 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2725 match(Set dst (MaxReductionV src1 src2));
2726 ins_cost(INSN_COST);
2727 effect(TEMP_DEF dst);
2728 format %{ "sve_reduce_maxF $dst, $src1, $src2\t# maxF reduction (sve)" %}
2729 ins_encode %{
2730 __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src2$$reg));
2731 __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2732 %}
2733 ins_pipe(pipe_slow);
2734 %}
2735
2736 instruct reduce_maxF_partial(vRegF dst, vRegF src1, vReg src2,
2737 pRegGov ptmp, rFlagsReg cr) %{
2738 predicate(UseSVE > 0 &&
2739 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
2740 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2741 match(Set dst (MaxReductionV src1 src2));
2742 ins_cost(INSN_COST);
2743 effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
2744 format %{ "sve_reduce_maxF $dst, $src1, $src2\t# maxF reduction partial (sve)" %}
2745 ins_encode %{
2746 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2747 Matcher::vector_length(this, $src2));
2748 __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2749 __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2750 %}
2751 ins_pipe(pipe_slow);
2752 %}
2753
2754 instruct reduce_maxD(vRegD dst, vRegD src1, vReg src2) %{
2755 predicate(UseSVE > 0 &&
2756 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
2757 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2758 match(Set dst (MaxReductionV src1 src2));
2759 ins_cost(INSN_COST);
2760 effect(TEMP_DEF dst);
2761 format %{ "sve_reduce_maxD $dst, $src1, $src2\t# maxD reduction (sve)" %}
2762 ins_encode %{
2763 __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
2764 __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2765 %}
2766 ins_pipe(pipe_slow);
2767 %}
2768
2769 instruct reduce_maxD_partial(vRegD dst, vRegD src1, vReg src2,
2770 pRegGov ptmp, rFlagsReg cr) %{
2771 predicate(UseSVE > 0 &&
2772 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
2773 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2774 match(Set dst (MaxReductionV src1 src2));
2775 ins_cost(INSN_COST);
2776 effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
2777 format %{ "sve_reduce_maxD $dst, $src1, $src2\t# maxD reduction partial (sve)" %}
2778 ins_encode %{
2779 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2780 Matcher::vector_length(this, $src2));
2781 __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2782 __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2783 %}
2784 ins_pipe(pipe_slow);
2785 %}
2786
2787 // vector max reduction - predicated
2788
2789 instruct reduce_maxI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp,
2790 pRegGov pg, rFlagsReg cr) %{
2791 predicate(UseSVE > 0 &&
2792 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2793 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2794 is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
2795 match(Set dst (MaxReductionV (Binary src1 src2) pg));
2796 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2797 ins_cost(SVE_COST);
2798 format %{ "sve_reduce_maxI $dst, $src1, $pg, $src2\t# maxI reduction predicated (sve)" %}
2799 ins_encode %{
2800 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2801 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2802 $src1$$Register, as_FloatRegister($src2$$reg),
2803 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2804 %}
2805 ins_pipe(pipe_slow);
2806 %}
2807
2808 instruct reduce_maxL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp,
2809 pRegGov pg, rFlagsReg cr) %{
2810 predicate(UseSVE > 0 &&
2811 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2812 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2813 match(Set dst (MaxReductionV (Binary src1 src2) pg));
2814 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2815 ins_cost(SVE_COST);
2816 format %{ "sve_reduce_maxL $dst, $src1, $pg, $src2\t# maxL reduction predicated (sve)" %}
2817 ins_encode %{
2818 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2819 $src1$$Register, as_FloatRegister($src2$$reg),
2820 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2821 %}
2822 ins_pipe(pipe_slow);
2823 %}
2824
2825 instruct reduce_maxI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2826 pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2827 predicate(UseSVE > 0 &&
2828 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2829 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2830 is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
2831 match(Set dst (MaxReductionV (Binary src1 src2) pg));
2832 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2833 ins_cost(3 * SVE_COST);
2834 format %{ "sve_reduce_maxI $dst, $src1, $pg, $src2\t# maxI reduction predicated partial (sve)" %}
2835 ins_encode %{
2836 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2837 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2838 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2839 Matcher::vector_length(this, $src2));
2840 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2841 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2842 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2843 $src1$$Register, as_FloatRegister($src2$$reg),
2844 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2845 %}
2846 ins_pipe(pipe_slow);
2847 %}
2848
2849 instruct reduce_maxL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2850 pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2851 predicate(UseSVE > 0 &&
2852 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2853 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2854 match(Set dst (MaxReductionV (Binary src1 src2) pg));
2855 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2856 ins_cost(3 * SVE_COST);
2857 format %{ "sve_reduce_maxL $dst, $src1, $pg, $src2\t# maxL reduction predicated partial (sve)" %}
2858 ins_encode %{
2859 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2860 Matcher::vector_length(this, $src2));
2861 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2862 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2863 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2864 $src1$$Register, as_FloatRegister($src2$$reg),
2865 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2866 %}
2867 ins_pipe(pipe_slow);
2868 %}
2869
2870 instruct reduce_maxF_masked(vRegF dst, vRegF src1, vReg src2, pRegGov pg) %{
2871 predicate(UseSVE > 0 &&
2872 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
2873 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2874 match(Set dst (MaxReductionV (Binary src1 src2) pg));
2875 ins_cost(SVE_COST);
2876 format %{ "sve_reduce_maxF $dst, $src1, $pg, $src2\t# maxF reduction predicated (sve)" %}
2877 ins_encode %{
2878 __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
2879 __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2880 %}
2881 ins_pipe(pipe_slow);
2882 %}
2883
2884 instruct reduce_maxD_masked(vRegD dst, vRegD src1, vReg src2, pRegGov pg) %{
2885 predicate(UseSVE > 0 &&
2886 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
2887 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2888 match(Set dst (MaxReductionV (Binary src1 src2) pg));
2889 ins_cost(SVE_COST);
2890 format %{ "sve_reduce_maxD $dst, $src1, $pg, $src2\t# maxD reduction predicated (sve)" %}
2891 ins_encode %{
2892 __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
2893 __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2894 %}
2895 ins_pipe(pipe_slow);
2896 %}
2897
2898 instruct reduce_maxF_masked_partial(vRegF dst, vRegF src1, vReg src2, pRegGov pg,
2899 pRegGov ptmp, rFlagsReg cr) %{
2900 predicate(UseSVE > 0 &&
2901 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
2902 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2903 match(Set dst (MaxReductionV (Binary src1 src2) pg));
2904 effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
2905 ins_cost(3 * SVE_COST);
2906 format %{ "sve_reduce_maxF $dst, $src1, $pg, $src2\t# maxF reduction predicated partial (sve)" %}
2907 ins_encode %{
2908 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2909 Matcher::vector_length(this, $src2));
2910 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2911 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2912 __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S,
2913 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2914 __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2915 %}
2916 ins_pipe(pipe_slow);
2917 %}
2918
2919 instruct reduce_maxD_masked_partial(vRegD dst, vRegD src1, vReg src2, pRegGov pg,
2920 pRegGov ptmp, rFlagsReg cr) %{
2921 predicate(UseSVE > 0 &&
2922 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
2923 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2924 match(Set dst (MaxReductionV (Binary src1 src2) pg));
2925 effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
2926 ins_cost(3 * SVE_COST);
2927 format %{ "sve_reduce_maxD $dst, $src1, $pg, $src2\t# maxD reduction predicated partial (sve)" %}
2928 ins_encode %{
2929 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2930 Matcher::vector_length(this, $src2));
2931 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2932 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2933 __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D,
2934 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2935 __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
2936 %}
2937 ins_pipe(pipe_slow);
2938 %}
2939
2940 // vector min reduction
2941
2942 instruct reduce_minI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
2943 predicate(UseSVE > 0 &&
2944 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2945 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2946 is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
2947 match(Set dst (MinReductionV src1 src2));
2948 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2949 ins_cost(SVE_COST);
2950 format %{ "sve_reduce_minI $dst, $src1, $src2\t# minI reduction (sve)" %}
2951 ins_encode %{
2952 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2953 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2954 $src1$$Register, as_FloatRegister($src2$$reg),
2955 ptrue, as_FloatRegister($tmp$$reg));
2956 %}
2957 ins_pipe(pipe_slow);
2958 %}
2959
2960 instruct reduce_minL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
2961 predicate(UseSVE > 0 &&
2962 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2963 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2964 match(Set dst (MinReductionV src1 src2));
2965 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2966 ins_cost(SVE_COST);
2967 format %{ "sve_reduce_minL $dst, $src1, $src2\t# minL reduction (sve)" %}
2968 ins_encode %{
2969 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2970 $src1$$Register, as_FloatRegister($src2$$reg),
2971 ptrue, as_FloatRegister($tmp$$reg));
2972 %}
2973 ins_pipe(pipe_slow);
2974 %}
2975
2976 instruct reduce_minI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2977 pRegGov ptmp, rFlagsReg cr) %{
2978 predicate(UseSVE > 0 &&
2979 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2980 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2981 is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
2982 match(Set dst (MinReductionV src1 src2));
2983 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2984 ins_cost(2 * SVE_COST);
2985 format %{ "sve_reduce_minI $dst, $src1, $src2\t# minI reduction partial (sve)" %}
2986 ins_encode %{
2987 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2988 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2989 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
2990 Matcher::vector_length(this, $src2));
2991 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2992 $src1$$Register, as_FloatRegister($src2$$reg),
2993 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2994 %}
2995 ins_pipe(pipe_slow);
2996 %}
2997
2998 instruct reduce_minL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2999 pRegGov ptmp, rFlagsReg cr) %{
3000 predicate(UseSVE > 0 &&
3001 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
3002 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3003 match(Set dst (MinReductionV src1 src2));
3004 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
3005 ins_cost(2 * SVE_COST);
3006 format %{ "sve_reduce_minL $dst, $src1, $src2\t# minL reduction partial (sve)" %}
3007 ins_encode %{
3008 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
3009 Matcher::vector_length(this, $src2));
3010 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
3011 $src1$$Register, as_FloatRegister($src2$$reg),
3012 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
3013 %}
3014 ins_pipe(pipe_slow);
3015 %}
3016
3017 instruct reduce_minF(vRegF dst, vRegF src1, vReg src2) %{
3018 predicate(UseSVE > 0 &&
3019 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3020 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3021 match(Set dst (MinReductionV src1 src2));
3022 ins_cost(INSN_COST);
3023 effect(TEMP_DEF dst);
3024 format %{ "sve_reduce_minF $dst, $src1, $src2\t# minF reduction (sve)" %}
3025 ins_encode %{
3026 __ sve_fminv(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src2$$reg));
3027 __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3028 %}
3029 ins_pipe(pipe_slow);
3030 %}
3031
3032 instruct reduce_minF_partial(vRegF dst, vRegF src1, vReg src2,
3033 pRegGov ptmp, rFlagsReg cr) %{
3034 predicate(UseSVE > 0 &&
3035 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3036 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3037 match(Set dst (MinReductionV src1 src2));
3038 ins_cost(INSN_COST);
3039 effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3040 format %{ "sve_reduce_minF $dst, $src1, $src2\t# minF reduction partial (sve)" %}
3041 ins_encode %{
3042 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
3043 Matcher::vector_length(this, $src2));
3044 __ sve_fminv(as_FloatRegister($dst$$reg), __ S, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
3045 __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3046 %}
3047 ins_pipe(pipe_slow);
3048 %}
3049
3050 instruct reduce_minD(vRegD dst, vRegD src1, vReg src2) %{
3051 predicate(UseSVE > 0 &&
3052 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3053 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3054 match(Set dst (MinReductionV src1 src2));
3055 ins_cost(INSN_COST);
3056 effect(TEMP_DEF dst);
3057 format %{ "sve_reduce_minD $dst, $src1, $src2\t# minD reduction (sve)" %}
3058 ins_encode %{
3059 __ sve_fminv(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
3060 __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3061 %}
3062 ins_pipe(pipe_slow);
3063 %}
3064
3065 instruct reduce_minD_partial(vRegD dst, vRegD src1, vReg src2,
3066 pRegGov ptmp, rFlagsReg cr) %{
3067 predicate(UseSVE > 0 &&
3068 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3069 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3070 match(Set dst (MinReductionV src1 src2));
3071 ins_cost(INSN_COST);
3072 effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3073 format %{ "sve_reduce_minD $dst, $src1, $src2\t# minD reduction partial (sve)" %}
3074 ins_encode %{
3075 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
3076 Matcher::vector_length(this, $src2));
3077 __ sve_fminv(as_FloatRegister($dst$$reg), __ D, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
3078 __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3079 %}
3080 ins_pipe(pipe_slow);
3081 %}
3082
3083 // vector min reduction - predicated
3084
3085 instruct reduce_minI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp,
3086 pRegGov pg, rFlagsReg cr) %{
3087 predicate(UseSVE > 0 &&
3088 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
3089 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
3090 is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
3091 match(Set dst (MinReductionV (Binary src1 src2) pg));
3092 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3093 ins_cost(SVE_COST);
3094 format %{ "sve_reduce_minI $dst, $src1, $pg, $src2\t# minI reduction predicated (sve)" %}
3095 ins_encode %{
3096 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
3097 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
3098 $src1$$Register, as_FloatRegister($src2$$reg),
3099 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
3100 %}
3101 ins_pipe(pipe_slow);
3102 %}
3103
3104 instruct reduce_minL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp,
3105 pRegGov pg, rFlagsReg cr) %{
3106 predicate(UseSVE > 0 &&
3107 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
3108 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3109 match(Set dst (MinReductionV (Binary src1 src2) pg));
3110 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3111 ins_cost(SVE_COST);
3112 format %{ "sve_reduce_minL $dst, $src1, $pg, $src2\t# minL reduction predicated (sve)" %}
3113 ins_encode %{
3114 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
3115 $src1$$Register, as_FloatRegister($src2$$reg),
3116 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
3117 %}
3118 ins_pipe(pipe_slow);
3119 %}
3120
3121 instruct reduce_minI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
3122 pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
3123 predicate(UseSVE > 0 &&
3124 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
3125 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
3126 is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
3127 match(Set dst (MinReductionV (Binary src1 src2) pg));
3128 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
3129 ins_cost(3 * SVE_COST);
3130 format %{ "sve_reduce_minI $dst, $src1, $pg, $src2\t# minI reduction predicated partial (sve)" %}
3131 ins_encode %{
3132 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
3133 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
3134 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
3135 Matcher::vector_length(this, $src2));
3136 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
3137 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
3138 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
3139 $src1$$Register, as_FloatRegister($src2$$reg),
3140 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
3141 %}
3142 ins_pipe(pipe_slow);
3143 %}
3144
3145 instruct reduce_minL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
3146 pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
3147 predicate(UseSVE > 0 &&
3148 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
3149 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3150 match(Set dst (MinReductionV (Binary src1 src2) pg));
3151 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
3152 ins_cost(3 * SVE_COST);
3153 format %{ "sve_reduce_minL $dst, $src1, $pg, $src2\t# minL reduction predicated partial (sve)" %}
3154 ins_encode %{
3155 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
3156 Matcher::vector_length(this, $src2));
3157 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
3158 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
3159 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
3160 $src1$$Register, as_FloatRegister($src2$$reg),
3161 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
3162 %}
3163 ins_pipe(pipe_slow);
3164 %}
3165
3166 instruct reduce_minF_masked(vRegF dst, vRegF src1, vReg src2, pRegGov pg) %{
3167 predicate(UseSVE > 0 &&
3168 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3169 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3170 match(Set dst (MinReductionV (Binary src1 src2) pg));
3171 ins_cost(SVE_COST);
3172 format %{ "sve_reduce_minF $dst, $src1, $pg, $src2\t# minF reduction predicated (sve)" %}
3173 ins_encode %{
3174 __ sve_fminv(as_FloatRegister($dst$$reg), __ S, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
3175 __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3176 %}
3177 ins_pipe(pipe_slow);
3178 %}
3179
3180 instruct reduce_minD_masked(vRegD dst, vRegD src1, vReg src2, pRegGov pg) %{
3181 predicate(UseSVE > 0 &&
3182 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3183 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3184 match(Set dst (MinReductionV (Binary src1 src2) pg));
3185 ins_cost(SVE_COST);
3186 format %{ "sve_reduce_minD $dst, $src1, $pg, $src2\t# minD reduction predicated (sve)" %}
3187 ins_encode %{
3188 __ sve_fminv(as_FloatRegister($dst$$reg), __ D, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
3189 __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3190 %}
3191 ins_pipe(pipe_slow);
3192 %}
3193
3194 instruct reduce_minF_masked_partial(vRegF dst, vRegF src1, vReg src2, pRegGov pg,
3195 pRegGov ptmp, rFlagsReg cr) %{
3196 predicate(UseSVE > 0 &&
3197 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3198 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3199 match(Set dst (MinReductionV (Binary src1 src2) pg));
3200 effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3201 ins_cost(3 * SVE_COST);
3202 format %{ "sve_reduce_minF $dst, $src1, $pg, $src2\t# minF reduction predicated partial (sve)" %}
3203 ins_encode %{
3204 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
3205 Matcher::vector_length(this, $src2));
3206 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
3207 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
3208 __ sve_fminv(as_FloatRegister($dst$$reg), __ S,
3209 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
3210 __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3211 %}
3212 ins_pipe(pipe_slow);
3213 %}
3214
3215 instruct reduce_minD_masked_partial(vRegD dst, vRegD src1, vReg src2, pRegGov pg,
3216 pRegGov ptmp, rFlagsReg cr) %{
3217 predicate(UseSVE > 0 &&
3218 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3219 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3220 match(Set dst (MinReductionV (Binary src1 src2) pg));
3221 effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3222 ins_cost(3 * SVE_COST);
3223 format %{ "sve_reduce_minD $dst, $src1, $pg, $src2\t# minD reduction predicated partial (sve)" %}
3224 ins_encode %{
3225 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
3226 Matcher::vector_length(this, $src2));
3227 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
3228 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
3229 __ sve_fminv(as_FloatRegister($dst$$reg), __ D,
3230 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
3231 __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3232 %}
3233 ins_pipe(pipe_slow);
3234 %}
3235
3236 // vector Math.rint, floor, ceil
3237
3238 instruct vroundD(vReg dst, vReg src, immI rmode) %{
3239 predicate(UseSVE > 0 &&
3240 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
3241 match(Set dst (RoundDoubleModeV src rmode));
3242 format %{ "sve_frint $dst, $src, $rmode\t# vector (sve) (D)" %}
3243 ins_encode %{
3244 switch ($rmode$$constant) {
3245 case RoundDoubleModeNode::rmode_rint:
3246 __ sve_frintn(as_FloatRegister($dst$$reg), __ D,
3247 ptrue, as_FloatRegister($src$$reg));
3248 break;
3249 case RoundDoubleModeNode::rmode_floor:
3250 __ sve_frintm(as_FloatRegister($dst$$reg), __ D,
3251 ptrue, as_FloatRegister($src$$reg));
3252 break;
3253 case RoundDoubleModeNode::rmode_ceil:
3254 __ sve_frintp(as_FloatRegister($dst$$reg), __ D,
3255 ptrue, as_FloatRegister($src$$reg));
3256 break;
3257 }
3258 %}
3259 ins_pipe(pipe_slow);
3260 %}
3261
3262 // vector replicate
3263
3264 instruct replicateB(vReg dst, iRegIorL2I src) %{
3265 predicate(UseSVE > 0);
3266 match(Set dst (ReplicateB src));
3267 ins_cost(SVE_COST);
3268 format %{ "sve_dup $dst, $src\t# vector (sve) (B)" %}
3269 ins_encode %{
3270 __ sve_dup(as_FloatRegister($dst$$reg), __ B, as_Register($src$$reg));
3271 %}
3272 ins_pipe(pipe_slow);
3273 %}
3274
3275 instruct replicateS(vReg dst, iRegIorL2I src) %{
3276 predicate(UseSVE > 0);
3277 match(Set dst (ReplicateS src));
3278 ins_cost(SVE_COST);
3279 format %{ "sve_dup $dst, $src\t# vector (sve) (H)" %}
3280 ins_encode %{
3281 __ sve_dup(as_FloatRegister($dst$$reg), __ H, as_Register($src$$reg));
3282 %}
3283 ins_pipe(pipe_slow);
3284 %}
3285
3286 instruct replicateI(vReg dst, iRegIorL2I src) %{
3287 predicate(UseSVE > 0);
3288 match(Set dst (ReplicateI src));
3289 ins_cost(SVE_COST);
3290 format %{ "sve_dup $dst, $src\t# vector (sve) (S)" %}
3291 ins_encode %{
3292 __ sve_dup(as_FloatRegister($dst$$reg), __ S, as_Register($src$$reg));
3293 %}
3294 ins_pipe(pipe_slow);
3295 %}
3296
3297 instruct replicateL(vReg dst, iRegL src) %{
3298 predicate(UseSVE > 0);
3299 match(Set dst (ReplicateL src));
3300 ins_cost(SVE_COST);
3301 format %{ "sve_dup $dst, $src\t# vector (sve) (D)" %}
3302 ins_encode %{
3303 __ sve_dup(as_FloatRegister($dst$$reg), __ D, as_Register($src$$reg));
3304 %}
3305 ins_pipe(pipe_slow);
3306 %}
3307
3308 instruct replicateB_imm8(vReg dst, immI8 con) %{
3309 predicate(UseSVE > 0);
3310 match(Set dst (ReplicateB con));
3311 ins_cost(SVE_COST);
3312 format %{ "sve_dup $dst, $con\t# vector (sve) (B)" %}
3313 ins_encode %{
3314 __ sve_dup(as_FloatRegister($dst$$reg), __ B, $con$$constant);
3315 %}
3316 ins_pipe(pipe_slow);
3317 %}
3318
3319 instruct replicateS_imm8(vReg dst, immI8_shift8 con) %{
3320 predicate(UseSVE > 0);
3321 match(Set dst (ReplicateS con));
3322 ins_cost(SVE_COST);
3323 format %{ "sve_dup $dst, $con\t# vector (sve) (H)" %}
3324 ins_encode %{
3325 __ sve_dup(as_FloatRegister($dst$$reg), __ H, $con$$constant);
3326 %}
3327 ins_pipe(pipe_slow);
3328 %}
3329
3330 instruct replicateI_imm8(vReg dst, immI8_shift8 con) %{
3331 predicate(UseSVE > 0);
3332 match(Set dst (ReplicateI con));
3333 ins_cost(SVE_COST);
3334 format %{ "sve_dup $dst, $con\t# vector (sve) (S)" %}
3335 ins_encode %{
3336 __ sve_dup(as_FloatRegister($dst$$reg), __ S, $con$$constant);
3337 %}
3338 ins_pipe(pipe_slow);
3339 %}
3340
3341 instruct replicateL_imm8(vReg dst, immL8_shift8 con) %{
3342 predicate(UseSVE > 0);
3343 match(Set dst (ReplicateL con));
3344 ins_cost(SVE_COST);
3345 format %{ "sve_dup $dst, $con\t# vector (sve) (D)" %}
3346 ins_encode %{
3347 __ sve_dup(as_FloatRegister($dst$$reg), __ D, $con$$constant);
3348 %}
3349 ins_pipe(pipe_slow);
3350 %}
3351
3352 instruct replicateF(vReg dst, vRegF src) %{
3353 predicate(UseSVE > 0);
3354 match(Set dst (ReplicateF src));
3355 ins_cost(SVE_COST);
3356 format %{ "sve_cpy $dst, $src\t# vector (sve) (S)" %}
3357 ins_encode %{
3358 __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
3359 ptrue, as_FloatRegister($src$$reg));
3360 %}
3361 ins_pipe(pipe_slow);
3362 %}
3363
3364 instruct replicateD(vReg dst, vRegD src) %{
3365 predicate(UseSVE > 0);
3366 match(Set dst (ReplicateD src));
3367 ins_cost(SVE_COST);
3368 format %{ "sve_cpy $dst, $src\t# vector (sve) (D)" %}
3369 ins_encode %{
3370 __ sve_cpy(as_FloatRegister($dst$$reg), __ D,
3371 ptrue, as_FloatRegister($src$$reg));
3372 %}
3373 ins_pipe(pipe_slow);
3374 %}
3375
3376 // vector shift
3377
3378 instruct vasrB(vReg dst, vReg shift) %{
3379 predicate(UseSVE > 0);
3380 match(Set dst (RShiftVB dst shift));
3381 ins_cost(SVE_COST);
3382 format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (B)" %}
3383 ins_encode %{
3384 __ sve_asr(as_FloatRegister($dst$$reg), __ B,
3385 ptrue, as_FloatRegister($shift$$reg));
3386 %}
3387 ins_pipe(pipe_slow);
3388 %}
3389
3390 instruct vasrS(vReg dst, vReg shift) %{
3391 predicate(UseSVE > 0);
3392 match(Set dst (RShiftVS dst shift));
3393 ins_cost(SVE_COST);
3394 format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (H)" %}
3395 ins_encode %{
3396 __ sve_asr(as_FloatRegister($dst$$reg), __ H,
3397 ptrue, as_FloatRegister($shift$$reg));
3398 %}
3399 ins_pipe(pipe_slow);
3400 %}
3401
3402 instruct vasrI(vReg dst, vReg shift) %{
3403 predicate(UseSVE > 0);
3404 match(Set dst (RShiftVI dst shift));
3405 ins_cost(SVE_COST);
3406 format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (S)" %}
3407 ins_encode %{
3408 __ sve_asr(as_FloatRegister($dst$$reg), __ S,
3409 ptrue, as_FloatRegister($shift$$reg));
3410 %}
3411 ins_pipe(pipe_slow);
3412 %}
3413
3414 instruct vasrL(vReg dst, vReg shift) %{
3415 predicate(UseSVE > 0);
3416 match(Set dst (RShiftVL dst shift));
3417 ins_cost(SVE_COST);
3418 format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (D)" %}
3419 ins_encode %{
3420 __ sve_asr(as_FloatRegister($dst$$reg), __ D,
3421 ptrue, as_FloatRegister($shift$$reg));
3422 %}
3423 ins_pipe(pipe_slow);
3424 %}
3425
3426 instruct vlslB(vReg dst, vReg shift) %{
3427 predicate(UseSVE > 0);
3428 match(Set dst (LShiftVB dst shift));
3429 ins_cost(SVE_COST);
3430 format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (B)" %}
3431 ins_encode %{
3432 __ sve_lsl(as_FloatRegister($dst$$reg), __ B,
3433 ptrue, as_FloatRegister($shift$$reg));
3434 %}
3435 ins_pipe(pipe_slow);
3436 %}
3437
3438 instruct vlslS(vReg dst, vReg shift) %{
3439 predicate(UseSVE > 0);
3440 match(Set dst (LShiftVS dst shift));
3441 ins_cost(SVE_COST);
3442 format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (H)" %}
3443 ins_encode %{
3444 __ sve_lsl(as_FloatRegister($dst$$reg), __ H,
3445 ptrue, as_FloatRegister($shift$$reg));
3446 %}
3447 ins_pipe(pipe_slow);
3448 %}
3449
3450 instruct vlslI(vReg dst, vReg shift) %{
3451 predicate(UseSVE > 0);
3452 match(Set dst (LShiftVI dst shift));
3453 ins_cost(SVE_COST);
3454 format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (S)" %}
3455 ins_encode %{
3456 __ sve_lsl(as_FloatRegister($dst$$reg), __ S,
3457 ptrue, as_FloatRegister($shift$$reg));
3458 %}
3459 ins_pipe(pipe_slow);
3460 %}
3461
3462 instruct vlslL(vReg dst, vReg shift) %{
3463 predicate(UseSVE > 0);
3464 match(Set dst (LShiftVL dst shift));
3465 ins_cost(SVE_COST);
3466 format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (D)" %}
3467 ins_encode %{
3468 __ sve_lsl(as_FloatRegister($dst$$reg), __ D,
3469 ptrue, as_FloatRegister($shift$$reg));
3470 %}
3471 ins_pipe(pipe_slow);
3472 %}
3473
3474 instruct vlsrB(vReg dst, vReg shift) %{
3475 predicate(UseSVE > 0);
3476 match(Set dst (URShiftVB dst shift));
3477 ins_cost(SVE_COST);
3478 format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (B)" %}
3479 ins_encode %{
3480 __ sve_lsr(as_FloatRegister($dst$$reg), __ B,
3481 ptrue, as_FloatRegister($shift$$reg));
3482 %}
3483 ins_pipe(pipe_slow);
3484 %}
3485
3486 instruct vlsrS(vReg dst, vReg shift) %{
3487 predicate(UseSVE > 0);
3488 match(Set dst (URShiftVS dst shift));
3489 ins_cost(SVE_COST);
3490 format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (H)" %}
3491 ins_encode %{
3492 __ sve_lsr(as_FloatRegister($dst$$reg), __ H,
3493 ptrue, as_FloatRegister($shift$$reg));
3494 %}
3495 ins_pipe(pipe_slow);
3496 %}
3497
3498 instruct vlsrI(vReg dst, vReg shift) %{
3499 predicate(UseSVE > 0);
3500 match(Set dst (URShiftVI dst shift));
3501 ins_cost(SVE_COST);
3502 format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (S)" %}
3503 ins_encode %{
3504 __ sve_lsr(as_FloatRegister($dst$$reg), __ S,
3505 ptrue, as_FloatRegister($shift$$reg));
3506 %}
3507 ins_pipe(pipe_slow);
3508 %}
3509
3510 instruct vlsrL(vReg dst, vReg shift) %{
3511 predicate(UseSVE > 0);
3512 match(Set dst (URShiftVL dst shift));
3513 ins_cost(SVE_COST);
3514 format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (D)" %}
3515 ins_encode %{
3516 __ sve_lsr(as_FloatRegister($dst$$reg), __ D,
3517 ptrue, as_FloatRegister($shift$$reg));
3518 %}
3519 ins_pipe(pipe_slow);
3520 %}
3521
3522 instruct vasrB_imm(vReg dst, vReg src, immI shift) %{
3523 predicate(UseSVE > 0);
3524 match(Set dst (RShiftVB src (RShiftCntV shift)));
3525 ins_cost(SVE_COST);
3526 format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (B)" %}
3527 ins_encode %{
3528 int con = (int)$shift$$constant;
3529 if (con == 0) {
3530 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3531 as_FloatRegister($src$$reg));
3532 return;
3533 }
3534 if (con >= 8) con = 7;
3535 __ sve_asr(as_FloatRegister($dst$$reg), __ B,
3536 as_FloatRegister($src$$reg), con);
3537 %}
3538 ins_pipe(pipe_slow);
3539 %}
3540
3541 instruct vasrS_imm(vReg dst, vReg src, immI shift) %{
3542 predicate(UseSVE > 0);
3543 match(Set dst (RShiftVS src (RShiftCntV shift)));
3544 ins_cost(SVE_COST);
3545 format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (H)" %}
3546 ins_encode %{
3547 int con = (int)$shift$$constant;
3548 if (con == 0) {
3549 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3550 as_FloatRegister($src$$reg));
3551 return;
3552 }
3553 if (con >= 16) con = 15;
3554 __ sve_asr(as_FloatRegister($dst$$reg), __ H,
3555 as_FloatRegister($src$$reg), con);
3556 %}
3557 ins_pipe(pipe_slow);
3558 %}
3559
3560 instruct vasrI_imm(vReg dst, vReg src, immI shift) %{
3561 predicate(UseSVE > 0);
3562 match(Set dst (RShiftVI src (RShiftCntV shift)));
3563 ins_cost(SVE_COST);
3564 format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (S)" %}
3565 ins_encode %{
3566 int con = (int)$shift$$constant;
3567 if (con == 0) {
3568 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3569 as_FloatRegister($src$$reg));
3570 return;
3571 }
3572 __ sve_asr(as_FloatRegister($dst$$reg), __ S,
3573 as_FloatRegister($src$$reg), con);
3574 %}
3575 ins_pipe(pipe_slow);
3576 %}
3577
3578 instruct vasrL_imm(vReg dst, vReg src, immI shift) %{
3579 predicate(UseSVE > 0);
3580 match(Set dst (RShiftVL src (RShiftCntV shift)));
3581 ins_cost(SVE_COST);
3582 format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (D)" %}
3583 ins_encode %{
3584 int con = (int)$shift$$constant;
3585 if (con == 0) {
3586 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3587 as_FloatRegister($src$$reg));
3588 return;
3589 }
3590 __ sve_asr(as_FloatRegister($dst$$reg), __ D,
3591 as_FloatRegister($src$$reg), con);
3592 %}
3593 ins_pipe(pipe_slow);
3594 %}
3595
3596 instruct vlsrB_imm(vReg dst, vReg src, immI shift) %{
3597 predicate(UseSVE > 0);
3598 match(Set dst (URShiftVB src (RShiftCntV shift)));
3599 ins_cost(SVE_COST);
3600 format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (B)" %}
3601 ins_encode %{
3602 int con = (int)$shift$$constant;
3603 if (con == 0) {
3604 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3605 as_FloatRegister($src$$reg));
3606 return;
3607 }
3608 if (con >= 8) {
3609 __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3610 as_FloatRegister($src$$reg));
3611 return;
3612 }
3613 __ sve_lsr(as_FloatRegister($dst$$reg), __ B,
3614 as_FloatRegister($src$$reg), con);
3615 %}
3616 ins_pipe(pipe_slow);
3617 %}
3618
3619 instruct vlsrS_imm(vReg dst, vReg src, immI shift) %{
3620 predicate(UseSVE > 0);
3621 match(Set dst (URShiftVS src (RShiftCntV shift)));
3622 ins_cost(SVE_COST);
3623 format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (H)" %}
3624 ins_encode %{
3625 int con = (int)$shift$$constant;
3626 if (con == 0) {
3627 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3628 as_FloatRegister($src$$reg));
3629 return;
3630 }
3631 if (con >= 16) {
3632 __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3633 as_FloatRegister($src$$reg));
3634 return;
3635 }
3636 __ sve_lsr(as_FloatRegister($dst$$reg), __ H,
3637 as_FloatRegister($src$$reg), con);
3638 %}
3639 ins_pipe(pipe_slow);
3640 %}
3641
3642 instruct vlsrI_imm(vReg dst, vReg src, immI shift) %{
3643 predicate(UseSVE > 0);
3644 match(Set dst (URShiftVI src (RShiftCntV shift)));
3645 ins_cost(SVE_COST);
3646 format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (S)" %}
3647 ins_encode %{
3648 int con = (int)$shift$$constant;
3649 if (con == 0) {
3650 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3651 as_FloatRegister($src$$reg));
3652 return;
3653 }
3654 __ sve_lsr(as_FloatRegister($dst$$reg), __ S,
3655 as_FloatRegister($src$$reg), con);
3656 %}
3657 ins_pipe(pipe_slow);
3658 %}
3659
3660 instruct vlsrL_imm(vReg dst, vReg src, immI shift) %{
3661 predicate(UseSVE > 0);
3662 match(Set dst (URShiftVL src (RShiftCntV shift)));
3663 ins_cost(SVE_COST);
3664 format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (D)" %}
3665 ins_encode %{
3666 int con = (int)$shift$$constant;
3667 if (con == 0) {
3668 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3669 as_FloatRegister($src$$reg));
3670 return;
3671 }
3672 __ sve_lsr(as_FloatRegister($dst$$reg), __ D,
3673 as_FloatRegister($src$$reg), con);
3674 %}
3675 ins_pipe(pipe_slow);
3676 %}
3677
3678 instruct vlslB_imm(vReg dst, vReg src, immI shift) %{
3679 predicate(UseSVE > 0);
3680 match(Set dst (LShiftVB src (LShiftCntV shift)));
3681 ins_cost(SVE_COST);
3682 format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (B)" %}
3683 ins_encode %{
3684 int con = (int)$shift$$constant;
3685 if (con >= 8) {
3686 __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3687 as_FloatRegister($src$$reg));
3688 return;
3689 }
3690 __ sve_lsl(as_FloatRegister($dst$$reg), __ B,
3691 as_FloatRegister($src$$reg), con);
3692 %}
3693 ins_pipe(pipe_slow);
3694 %}
3695
3696 instruct vlslS_imm(vReg dst, vReg src, immI shift) %{
3697 predicate(UseSVE > 0);
3698 match(Set dst (LShiftVS src (LShiftCntV shift)));
3699 ins_cost(SVE_COST);
3700 format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (H)" %}
3701 ins_encode %{
3702 int con = (int)$shift$$constant;
3703 if (con >= 16) {
3704 __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3705 as_FloatRegister($src$$reg));
3706 return;
3707 }
3708 __ sve_lsl(as_FloatRegister($dst$$reg), __ H,
3709 as_FloatRegister($src$$reg), con);
3710 %}
3711 ins_pipe(pipe_slow);
3712 %}
3713
3714 instruct vlslI_imm(vReg dst, vReg src, immI shift) %{
3715 predicate(UseSVE > 0);
3716 match(Set dst (LShiftVI src (LShiftCntV shift)));
3717 ins_cost(SVE_COST);
3718 format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (S)" %}
3719 ins_encode %{
3720 int con = (int)$shift$$constant;
3721 __ sve_lsl(as_FloatRegister($dst$$reg), __ S,
3722 as_FloatRegister($src$$reg), con);
3723 %}
3724 ins_pipe(pipe_slow);
3725 %}
3726
3727 instruct vlslL_imm(vReg dst, vReg src, immI shift) %{
3728 predicate(UseSVE > 0);
3729 match(Set dst (LShiftVL src (LShiftCntV shift)));
3730 ins_cost(SVE_COST);
3731 format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (D)" %}
3732 ins_encode %{
3733 int con = (int)$shift$$constant;
3734 __ sve_lsl(as_FloatRegister($dst$$reg), __ D,
3735 as_FloatRegister($src$$reg), con);
3736 %}
3737 ins_pipe(pipe_slow);
3738 %}
3739
3740 instruct vshiftcntB(vReg dst, iRegIorL2I cnt) %{
3741 predicate(UseSVE > 0 &&
3742 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE));
3743 match(Set dst (LShiftCntV cnt));
3744 match(Set dst (RShiftCntV cnt));
3745 format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (B)" %}
3746 ins_encode %{
3747 __ sve_dup(as_FloatRegister($dst$$reg), __ B, as_Register($cnt$$reg));
3748 %}
3749 ins_pipe(pipe_slow);
3750 %}
3751
3752 instruct vshiftcntS(vReg dst, iRegIorL2I cnt) %{
3753 predicate(UseSVE > 0 &&
3754 (n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
3755 (n->bottom_type()->is_vect()->element_basic_type() == T_CHAR)));
3756 match(Set dst (LShiftCntV cnt));
3757 match(Set dst (RShiftCntV cnt));
3758 format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (H)" %}
3759 ins_encode %{
3760 __ sve_dup(as_FloatRegister($dst$$reg), __ H, as_Register($cnt$$reg));
3761 %}
3762 ins_pipe(pipe_slow);
3763 %}
3764
3765 instruct vshiftcntI(vReg dst, iRegIorL2I cnt) %{
3766 predicate(UseSVE > 0 &&
3767 (n->bottom_type()->is_vect()->element_basic_type() == T_INT));
3768 match(Set dst (LShiftCntV cnt));
3769 match(Set dst (RShiftCntV cnt));
3770 format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (S)" %}
3771 ins_encode %{
3772 __ sve_dup(as_FloatRegister($dst$$reg), __ S, as_Register($cnt$$reg));
3773 %}
3774 ins_pipe(pipe_slow);
3775 %}
3776
3777 instruct vshiftcntL(vReg dst, iRegIorL2I cnt) %{
3778 predicate(UseSVE > 0 &&
3779 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
3780 match(Set dst (LShiftCntV cnt));
3781 match(Set dst (RShiftCntV cnt));
3782 format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (D)" %}
3783 ins_encode %{
3784 __ sve_dup(as_FloatRegister($dst$$reg), __ D, as_Register($cnt$$reg));
3785 %}
3786 ins_pipe(pipe_slow);
3787 %}
3788
3789 // vector shift - predicated
3790
3791 instruct vasrB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3792 predicate(UseSVE > 0);
3793 match(Set dst_src1 (RShiftVB (Binary dst_src1 src2) pg));
3794 ins_cost(SVE_COST);
3795 format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
3796 ins_encode %{
3797 __ sve_asr(as_FloatRegister($dst_src1$$reg), __ B,
3798 as_PRegister($pg$$reg),
3799 as_FloatRegister($src2$$reg));
3800 %}
3801 ins_pipe(pipe_slow);
3802 %}
3803
3804 instruct vasrS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3805 predicate(UseSVE > 0);
3806 match(Set dst_src1 (RShiftVS (Binary dst_src1 src2) pg));
3807 ins_cost(SVE_COST);
3808 format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
3809 ins_encode %{
3810 __ sve_asr(as_FloatRegister($dst_src1$$reg), __ H,
3811 as_PRegister($pg$$reg),
3812 as_FloatRegister($src2$$reg));
3813 %}
3814 ins_pipe(pipe_slow);
3815 %}
3816
3817 instruct vasrI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3818 predicate(UseSVE > 0);
3819 match(Set dst_src1 (RShiftVI (Binary dst_src1 src2) pg));
3820 ins_cost(SVE_COST);
3821 format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
3822 ins_encode %{
3823 __ sve_asr(as_FloatRegister($dst_src1$$reg), __ S,
3824 as_PRegister($pg$$reg),
3825 as_FloatRegister($src2$$reg));
3826 %}
3827 ins_pipe(pipe_slow);
3828 %}
3829
3830 instruct vasrL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3831 predicate(UseSVE > 0);
3832 match(Set dst_src1 (RShiftVL (Binary dst_src1 src2) pg));
3833 ins_cost(SVE_COST);
3834 format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
3835 ins_encode %{
3836 __ sve_asr(as_FloatRegister($dst_src1$$reg), __ D,
3837 as_PRegister($pg$$reg),
3838 as_FloatRegister($src2$$reg));
3839 %}
3840 ins_pipe(pipe_slow);
3841 %}
3842
3843 instruct vlslB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3844 predicate(UseSVE > 0);
3845 match(Set dst_src1 (LShiftVB (Binary dst_src1 src2) pg));
3846 ins_cost(SVE_COST);
3847 format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
3848 ins_encode %{
3849 __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ B,
3850 as_PRegister($pg$$reg),
3851 as_FloatRegister($src2$$reg));
3852 %}
3853 ins_pipe(pipe_slow);
3854 %}
3855
3856 instruct vlslS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3857 predicate(UseSVE > 0);
3858 match(Set dst_src1 (LShiftVS (Binary dst_src1 src2) pg));
3859 ins_cost(SVE_COST);
3860 format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
3861 ins_encode %{
3862 __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ H,
3863 as_PRegister($pg$$reg),
3864 as_FloatRegister($src2$$reg));
3865 %}
3866 ins_pipe(pipe_slow);
3867 %}
3868
3869 instruct vlslI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3870 predicate(UseSVE > 0);
3871 match(Set dst_src1 (LShiftVI (Binary dst_src1 src2) pg));
3872 ins_cost(SVE_COST);
3873 format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
3874 ins_encode %{
3875 __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ S,
3876 as_PRegister($pg$$reg),
3877 as_FloatRegister($src2$$reg));
3878 %}
3879 ins_pipe(pipe_slow);
3880 %}
3881
3882 instruct vlslL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3883 predicate(UseSVE > 0);
3884 match(Set dst_src1 (LShiftVL (Binary dst_src1 src2) pg));
3885 ins_cost(SVE_COST);
3886 format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
3887 ins_encode %{
3888 __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ D,
3889 as_PRegister($pg$$reg),
3890 as_FloatRegister($src2$$reg));
3891 %}
3892 ins_pipe(pipe_slow);
3893 %}
3894
3895 instruct vlsrB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3896 predicate(UseSVE > 0);
3897 match(Set dst_src1 (URShiftVB (Binary dst_src1 src2) pg));
3898 ins_cost(SVE_COST);
3899 format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
3900 ins_encode %{
3901 __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ B,
3902 as_PRegister($pg$$reg),
3903 as_FloatRegister($src2$$reg));
3904 %}
3905 ins_pipe(pipe_slow);
3906 %}
3907
3908 instruct vlsrS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3909 predicate(UseSVE > 0);
3910 match(Set dst_src1 (URShiftVS (Binary dst_src1 src2) pg));
3911 ins_cost(SVE_COST);
3912 format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
3913 ins_encode %{
3914 __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ H,
3915 as_PRegister($pg$$reg),
3916 as_FloatRegister($src2$$reg));
3917 %}
3918 ins_pipe(pipe_slow);
3919 %}
3920
3921 instruct vlsrI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3922 predicate(UseSVE > 0);
3923 match(Set dst_src1 (URShiftVI (Binary dst_src1 src2) pg));
3924 ins_cost(SVE_COST);
3925 format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
3926 ins_encode %{
3927 __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ S,
3928 as_PRegister($pg$$reg),
3929 as_FloatRegister($src2$$reg));
3930 %}
3931 ins_pipe(pipe_slow);
3932 %}
3933
3934 instruct vlsrL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3935 predicate(UseSVE > 0);
3936 match(Set dst_src1 (URShiftVL (Binary dst_src1 src2) pg));
3937 ins_cost(SVE_COST);
3938 format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
3939 ins_encode %{
3940 __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ D,
3941 as_PRegister($pg$$reg),
3942 as_FloatRegister($src2$$reg));
3943 %}
3944 ins_pipe(pipe_slow);
3945 %}
3946
3947 instruct vasrB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
3948 predicate(UseSVE > 0);
3949 match(Set dst_src (RShiftVB (Binary dst_src (RShiftCntV shift)) pg));
3950 ins_cost(SVE_COST);
3951 format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %}
3952 ins_encode %{
3953 int con = (int)$shift$$constant;
3954 assert(con > 0 && con < 8, "invalid shift immediate");
3955 __ sve_asr(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con);
3956 %}
3957 ins_pipe(pipe_slow);
3958 %}
3959
3960 instruct vasrS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
3961 predicate(UseSVE > 0);
3962 match(Set dst_src (RShiftVS (Binary dst_src (RShiftCntV shift)) pg));
3963 ins_cost(SVE_COST);
3964 format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %}
3965 ins_encode %{
3966 int con = (int)$shift$$constant;
3967 assert(con > 0 && con < 16, "invalid shift immediate");
3968 __ sve_asr(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con);
3969 %}
3970 ins_pipe(pipe_slow);
3971 %}
3972
3973 instruct vasrI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
3974 predicate(UseSVE > 0);
3975 match(Set dst_src (RShiftVI (Binary dst_src (RShiftCntV shift)) pg));
3976 ins_cost(SVE_COST);
3977 format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %}
3978 ins_encode %{
3979 int con = (int)$shift$$constant;
3980 assert(con > 0 && con < 32, "invalid shift immediate");
3981 __ sve_asr(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con);
3982 %}
3983 ins_pipe(pipe_slow);
3984 %}
3985
3986 instruct vasrL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
3987 predicate(UseSVE > 0);
3988 match(Set dst_src (RShiftVL (Binary dst_src (RShiftCntV shift)) pg));
3989 ins_cost(SVE_COST);
3990 format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %}
3991 ins_encode %{
3992 int con = (int)$shift$$constant;
3993 assert(con > 0 && con < 64, "invalid shift immediate");
3994 __ sve_asr(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con);
3995 %}
3996 ins_pipe(pipe_slow);
3997 %}
3998
3999 instruct vlsrB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4000 predicate(UseSVE > 0);
4001 match(Set dst_src (URShiftVB (Binary dst_src (RShiftCntV shift)) pg));
4002 ins_cost(SVE_COST);
4003 format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %}
4004 ins_encode %{
4005 int con = (int)$shift$$constant;
4006 assert(con > 0 && con < 8, "invalid shift immediate");
4007 __ sve_lsr(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con);
4008 %}
4009 ins_pipe(pipe_slow);
4010 %}
4011
4012 instruct vlsrS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4013 predicate(UseSVE > 0);
4014 match(Set dst_src (URShiftVS (Binary dst_src (RShiftCntV shift)) pg));
4015 ins_cost(SVE_COST);
4016 format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %}
4017 ins_encode %{
4018 int con = (int)$shift$$constant;
4019 assert(con > 0 && con < 16, "invalid shift immediate");
4020 __ sve_lsr(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con);
4021 %}
4022 ins_pipe(pipe_slow);
4023 %}
4024
4025 instruct vlsrI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4026 predicate(UseSVE > 0);
4027 match(Set dst_src (URShiftVI (Binary dst_src (RShiftCntV shift)) pg));
4028 ins_cost(SVE_COST);
4029 format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %}
4030 ins_encode %{
4031 int con = (int)$shift$$constant;
4032 assert(con > 0 && con < 32, "invalid shift immediate");
4033 __ sve_lsr(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con);
4034 %}
4035 ins_pipe(pipe_slow);
4036 %}
4037
4038 instruct vlsrL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4039 predicate(UseSVE > 0);
4040 match(Set dst_src (URShiftVL (Binary dst_src (RShiftCntV shift)) pg));
4041 ins_cost(SVE_COST);
4042 format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %}
4043 ins_encode %{
4044 int con = (int)$shift$$constant;
4045 assert(con > 0 && con < 64, "invalid shift immediate");
4046 __ sve_lsr(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con);
4047 %}
4048 ins_pipe(pipe_slow);
4049 %}
4050
4051 instruct vlslB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4052 predicate(UseSVE > 0);
4053 match(Set dst_src (LShiftVB (Binary dst_src (LShiftCntV shift)) pg));
4054 ins_cost(SVE_COST);
4055 format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %}
4056 ins_encode %{
4057 int con = (int)$shift$$constant;
4058 assert(con >= 0 && con < 8, "invalid shift immediate");
4059 __ sve_lsl(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con);
4060 %}
4061 ins_pipe(pipe_slow);
4062 %}
4063
4064 instruct vlslS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4065 predicate(UseSVE > 0);
4066 match(Set dst_src (LShiftVS (Binary dst_src (LShiftCntV shift)) pg));
4067 ins_cost(SVE_COST);
4068 format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %}
4069 ins_encode %{
4070 int con = (int)$shift$$constant;
4071 assert(con >= 0 && con < 16, "invalid shift immediate");
4072 __ sve_lsl(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con);
4073 %}
4074 ins_pipe(pipe_slow);
4075 %}
4076
4077 instruct vlslI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4078 predicate(UseSVE > 0);
4079 match(Set dst_src (LShiftVI (Binary dst_src (LShiftCntV shift)) pg));
4080 ins_cost(SVE_COST);
4081 format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %}
4082 ins_encode %{
4083 int con = (int)$shift$$constant;
4084 assert(con >= 0 && con < 32, "invalid shift immediate");
4085 __ sve_lsl(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con);
4086 %}
4087 ins_pipe(pipe_slow);
4088 %}
4089
4090 instruct vlslL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4091 predicate(UseSVE > 0);
4092 match(Set dst_src (LShiftVL (Binary dst_src (LShiftCntV shift)) pg));
4093 ins_cost(SVE_COST);
4094 format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %}
4095 ins_encode %{
4096 int con = (int)$shift$$constant;
4097 assert(con >= 0 && con < 64, "invalid shift immediate");
4098 __ sve_lsl(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con);
4099 %}
4100 ins_pipe(pipe_slow);
4101 %}
4102
4103 // vector sqrt
4104
4105 instruct vsqrtF(vReg dst, vReg src) %{
4106 predicate(UseSVE > 0 &&
4107 !n->as_Vector()->is_predicated_vector());
4108 match(Set dst (SqrtVF src));
4109 ins_cost(SVE_COST);
4110 format %{ "sve_fsqrt $dst, $src\t# vector (sve) (S)" %}
4111 ins_encode %{
4112 __ sve_fsqrt(as_FloatRegister($dst$$reg), __ S,
4113 ptrue, as_FloatRegister($src$$reg));
4114 %}
4115 ins_pipe(pipe_slow);
4116 %}
4117
4118 instruct vsqrtD(vReg dst, vReg src) %{
4119 predicate(UseSVE > 0 &&
4120 !n->as_Vector()->is_predicated_vector());
4121 match(Set dst (SqrtVD src));
4122 ins_cost(SVE_COST);
4123 format %{ "sve_fsqrt $dst, $src\t# vector (sve) (D)" %}
4124 ins_encode %{
4125 __ sve_fsqrt(as_FloatRegister($dst$$reg), __ D,
4126 ptrue, as_FloatRegister($src$$reg));
4127 %}
4128 ins_pipe(pipe_slow);
4129 %}
4130
4131 // vector sqrt - predicated
4132
4133 instruct vsqrtF_masked(vReg dst_src, pRegGov pg) %{
4134 predicate(UseSVE > 0);
4135 match(Set dst_src (SqrtVF dst_src pg));
4136 ins_cost(SVE_COST);
4137 format %{ "sve_fsqrt $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
4138 ins_encode %{
4139 __ sve_fsqrt(as_FloatRegister($dst_src$$reg), __ S,
4140 as_PRegister($pg$$reg),
4141 as_FloatRegister($dst_src$$reg));
4142 %}
4143 ins_pipe(pipe_slow);
4144 %}
4145
4146 instruct vsqrtD_masked(vReg dst_src, pRegGov pg) %{
4147 predicate(UseSVE > 0);
4148 match(Set dst_src (SqrtVD dst_src pg));
4149 ins_cost(SVE_COST);
4150 format %{ "sve_fsqrt $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
4151 ins_encode %{
4152 __ sve_fsqrt(as_FloatRegister($dst_src$$reg), __ D,
4153 as_PRegister($pg$$reg),
4154 as_FloatRegister($dst_src$$reg));
4155 %}
4156 ins_pipe(pipe_slow);
4157 %}
4158
4159 // vector sub
4160
4161 instruct vsubB(vReg dst, vReg src1, vReg src2) %{
4162 predicate(UseSVE > 0);
4163 match(Set dst (SubVB src1 src2));
4164 ins_cost(SVE_COST);
4165 format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (B)" %}
4166 ins_encode %{
4167 __ sve_sub(as_FloatRegister($dst$$reg), __ B,
4168 as_FloatRegister($src1$$reg),
4169 as_FloatRegister($src2$$reg));
4170 %}
4171 ins_pipe(pipe_slow);
4172 %}
4173
4174 instruct vsubS(vReg dst, vReg src1, vReg src2) %{
4175 predicate(UseSVE > 0);
4176 match(Set dst (SubVS src1 src2));
4177 ins_cost(SVE_COST);
4178 format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (H)" %}
4179 ins_encode %{
4180 __ sve_sub(as_FloatRegister($dst$$reg), __ H,
4181 as_FloatRegister($src1$$reg),
4182 as_FloatRegister($src2$$reg));
4183 %}
4184 ins_pipe(pipe_slow);
4185 %}
4186
4187 instruct vsubI(vReg dst, vReg src1, vReg src2) %{
4188 predicate(UseSVE > 0);
4189 match(Set dst (SubVI src1 src2));
4190 ins_cost(SVE_COST);
4191 format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (S)" %}
4192 ins_encode %{
4193 __ sve_sub(as_FloatRegister($dst$$reg), __ S,
4194 as_FloatRegister($src1$$reg),
4195 as_FloatRegister($src2$$reg));
4196 %}
4197 ins_pipe(pipe_slow);
4198 %}
4199
4200 instruct vsubL(vReg dst, vReg src1, vReg src2) %{
4201 predicate(UseSVE > 0);
4202 match(Set dst (SubVL src1 src2));
4203 ins_cost(SVE_COST);
4204 format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (D)" %}
4205 ins_encode %{
4206 __ sve_sub(as_FloatRegister($dst$$reg), __ D,
4207 as_FloatRegister($src1$$reg),
4208 as_FloatRegister($src2$$reg));
4209 %}
4210 ins_pipe(pipe_slow);
4211 %}
4212
4213 instruct vsubF(vReg dst, vReg src1, vReg src2) %{
4214 predicate(UseSVE > 0);
4215 match(Set dst (SubVF src1 src2));
4216 ins_cost(SVE_COST);
4217 format %{ "sve_fsub $dst, $src1, $src2\t # vector (sve) (S)" %}
4218 ins_encode %{
4219 __ sve_fsub(as_FloatRegister($dst$$reg), __ S,
4220 as_FloatRegister($src1$$reg),
4221 as_FloatRegister($src2$$reg));
4222 %}
4223 ins_pipe(pipe_slow);
4224 %}
4225
4226 instruct vsubD(vReg dst, vReg src1, vReg src2) %{
4227 predicate(UseSVE > 0);
4228 match(Set dst (SubVD src1 src2));
4229 ins_cost(SVE_COST);
4230 format %{ "sve_fsub $dst, $src1, $src2\t # vector (sve) (D)" %}
4231 ins_encode %{
4232 __ sve_fsub(as_FloatRegister($dst$$reg), __ D,
4233 as_FloatRegister($src1$$reg),
4234 as_FloatRegister($src2$$reg));
4235 %}
4236 ins_pipe(pipe_slow);
4237 %}
4238
4239 // vector sub - predicated
4240
4241 instruct vsubB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4242 predicate(UseSVE > 0);
4243 match(Set dst_src1 (SubVB (Binary dst_src1 src2) pg));
4244 ins_cost(SVE_COST);
4245 format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
4246 ins_encode %{
4247 __ sve_sub(as_FloatRegister($dst_src1$$reg), __ B,
4248 as_PRegister($pg$$reg),
4249 as_FloatRegister($src2$$reg));
4250 %}
4251 ins_pipe(pipe_slow);
4252 %}
4253
4254 instruct vsubS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4255 predicate(UseSVE > 0);
4256 match(Set dst_src1 (SubVS (Binary dst_src1 src2) pg));
4257 ins_cost(SVE_COST);
4258 format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
4259 ins_encode %{
4260 __ sve_sub(as_FloatRegister($dst_src1$$reg), __ H,
4261 as_PRegister($pg$$reg),
4262 as_FloatRegister($src2$$reg));
4263 %}
4264 ins_pipe(pipe_slow);
4265 %}
4266
4267 instruct vsubI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4268 predicate(UseSVE > 0);
4269 match(Set dst_src1 (SubVI (Binary dst_src1 src2) pg));
4270 ins_cost(SVE_COST);
4271 format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
4272 ins_encode %{
4273 __ sve_sub(as_FloatRegister($dst_src1$$reg), __ S,
4274 as_PRegister($pg$$reg),
4275 as_FloatRegister($src2$$reg));
4276 %}
4277 ins_pipe(pipe_slow);
4278 %}
4279
4280 instruct vsubL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4281 predicate(UseSVE > 0);
4282 match(Set dst_src1 (SubVL (Binary dst_src1 src2) pg));
4283 ins_cost(SVE_COST);
4284 format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
4285 ins_encode %{
4286 __ sve_sub(as_FloatRegister($dst_src1$$reg), __ D,
4287 as_PRegister($pg$$reg),
4288 as_FloatRegister($src2$$reg));
4289 %}
4290 ins_pipe(pipe_slow);
4291 %}
4292
4293 instruct vsubF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4294 predicate(UseSVE > 0);
4295 match(Set dst_src1 (SubVF (Binary dst_src1 src2) pg));
4296 ins_cost(SVE_COST);
4297 format %{ "sve_fsub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
4298 ins_encode %{
4299 __ sve_fsub(as_FloatRegister($dst_src1$$reg), __ S,
4300 as_PRegister($pg$$reg),
4301 as_FloatRegister($src2$$reg));
4302 %}
4303 ins_pipe(pipe_slow);
4304 %}
4305
4306 instruct vsubD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4307 predicate(UseSVE > 0);
4308 match(Set dst_src1 (SubVD (Binary dst_src1 src2) pg));
4309 ins_cost(SVE_COST);
4310 format %{ "sve_fsub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
4311 ins_encode %{
4312 __ sve_fsub(as_FloatRegister($dst_src1$$reg), __ D,
4313 as_PRegister($pg$$reg),
4314 as_FloatRegister($src2$$reg));
4315 %}
4316 ins_pipe(pipe_slow);
4317 %}
4318
4319 // ------------------------------ Vector mask cast --------------------------
4320
4321 instruct vmaskcast(pRegGov dst_src) %{
4322 predicate(UseSVE > 0 &&
4323 n->bottom_type()->is_vect()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
4324 n->bottom_type()->is_vect()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
4325 match(Set dst_src (VectorMaskCast dst_src));
4326 ins_cost(0);
4327 format %{ "vmaskcast $dst_src\t# empty (sve)" %}
4328 ins_encode %{
4329 // empty
4330 %}
4331 ins_pipe(pipe_class_empty);
4332 %}
4333
4334 instruct vmaskcast_extend(pRegGov dst, pReg src)
4335 %{
4336 predicate(UseSVE > 0 &&
4337 (Matcher::vector_length_in_bytes(n) == 2 * Matcher::vector_length_in_bytes(n->in(1)) ||
4338 Matcher::vector_length_in_bytes(n) == 4 * Matcher::vector_length_in_bytes(n->in(1)) ||
4339 Matcher::vector_length_in_bytes(n) == 8 * Matcher::vector_length_in_bytes(n->in(1))));
4340 match(Set dst (VectorMaskCast src));
4341 ins_cost(SVE_COST * 3);
4342 format %{ "sve_vmaskcast_extend $dst, $src\t# extend predicate $src" %}
4343 ins_encode %{
4344 __ sve_vmaskcast_extend(as_PRegister($dst$$reg), as_PRegister($src$$reg),
4345 Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
4346 %}
4347 ins_pipe(pipe_slow);
4348 %}
4349
4350 instruct vmaskcast_narrow(pRegGov dst, pReg src)
4351 %{
4352 predicate(UseSVE > 0 &&
4353 (Matcher::vector_length_in_bytes(n) * 2 == Matcher::vector_length_in_bytes(n->in(1)) ||
4354 Matcher::vector_length_in_bytes(n) * 4 == Matcher::vector_length_in_bytes(n->in(1)) ||
4355 Matcher::vector_length_in_bytes(n) * 8 == Matcher::vector_length_in_bytes(n->in(1))));
4356 match(Set dst (VectorMaskCast src));
4357 ins_cost(SVE_COST * 3);
4358 format %{ "sve_vmaskcast_narrow $dst, $src\t# narrow predicate $src" %}
4359 ins_encode %{
4360 __ sve_vmaskcast_narrow(as_PRegister($dst$$reg), as_PRegister($src$$reg),
4361 Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
4362 %}
4363 ins_pipe(pipe_slow);
4364 %}
4365
4366 // ------------------------------ Vector cast -------------------------------
4367
4368 instruct vcvtBtoX_extend(vReg dst, vReg src)
4369 %{
4370 predicate(UseSVE > 0);
4371 match(Set dst (VectorCastB2X src));
4372 ins_cost(2 * SVE_COST);
4373 format %{ "sve_vectorcast_b2x $dst, $src\t# convert B to X vector (extend)" %}
4374 ins_encode %{
4375 BasicType to_bt = Matcher::vector_element_basic_type(this);
4376 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4377 __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ B);
4378 if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
4379 __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
4380 }
4381 %}
4382 ins_pipe(pipe_slow);
4383 %}
4384
4385 instruct vcvtStoB(vReg dst, vReg src, vReg tmp)
4386 %{
4387 predicate(UseSVE > 0 &&
4388 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
4389 match(Set dst (VectorCastS2X src));
4390 effect(TEMP tmp);
4391 ins_cost(2 * SVE_COST);
4392 format %{ "sve_vectorcast_s2b $dst, $src\t# convert H to B vector" %}
4393 ins_encode %{
4394 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
4395 as_FloatRegister($src$$reg), __ H, as_FloatRegister($tmp$$reg));
4396 %}
4397 ins_pipe(pipe_slow);
4398 %}
4399
4400 instruct vcvtStoX_extend(vReg dst, vReg src)
4401 %{
4402 predicate(UseSVE > 0 &&
4403 type2aelembytes(Matcher::vector_element_basic_type(n)) > 2);
4404 match(Set dst (VectorCastS2X src));
4405 ins_cost(2 * SVE_COST);
4406 format %{ "sve_vectorcast_s2x $dst, $src\t# convert H to X vector (extend)" %}
4407 ins_encode %{
4408 BasicType to_bt = Matcher::vector_element_basic_type(this);
4409 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4410 __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ H);
4411 if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
4412 __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
4413 }
4414 %}
4415 ins_pipe(pipe_slow);
4416 %}
4417
4418 instruct vcvtItoB(vReg dst, vReg src, vReg tmp)
4419 %{
4420 predicate(UseSVE > 0 &&
4421 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
4422 match(Set dst (VectorCastI2X src));
4423 effect(TEMP_DEF dst, TEMP tmp);
4424 ins_cost(3 * SVE_COST);
4425 format %{ "sve_vectorcast_i2b $dst, $src\t# convert I to B vector" %}
4426 ins_encode %{
4427 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
4428 as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
4429 %}
4430 ins_pipe(pipe_slow);
4431 %}
4432
4433 instruct vcvtItoS(vReg dst, vReg src, vReg tmp)
4434 %{
4435 predicate(UseSVE > 0 &&
4436 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
4437 match(Set dst (VectorCastI2X src));
4438 effect(TEMP tmp);
4439 ins_cost(2 * SVE_COST);
4440 format %{ "sve_vectorcast_i2s $dst, $src\t# convert I to H vector" %}
4441 ins_encode %{
4442 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ H,
4443 as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
4444 %}
4445 ins_pipe(pipe_slow);
4446 %}
4447
4448 instruct vcvtItoL(vReg dst, vReg src)
4449 %{
4450 predicate(UseSVE > 0 &&
4451 n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
4452 match(Set dst (VectorCastI2X src));
4453 ins_cost(SVE_COST);
4454 format %{ "sve_vectorcast_i2l $dst, $src\t# convert I to L vector" %}
4455 ins_encode %{
4456 __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
4457 %}
4458 ins_pipe(pipe_slow);
4459 %}
4460
4461 instruct vcvtItoF(vReg dst, vReg src)
4462 %{
4463 predicate(UseSVE > 0 &&
4464 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4465 match(Set dst (VectorCastI2X src));
4466 ins_cost(SVE_COST);
4467 format %{ "sve_vectorcast_i2f $dst, $src\t# convert I to F vector" %}
4468 ins_encode %{
4469 __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
4470 %}
4471 ins_pipe(pipe_slow);
4472 %}
4473
4474 instruct vcvtItoD(vReg dst, vReg src)
4475 %{
4476 predicate(UseSVE > 0 &&
4477 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
4478 match(Set dst (VectorCastI2X src));
4479 ins_cost(2 * SVE_COST);
4480 format %{ "sve_vectorcast_i2d $dst, $src\t# convert I to D vector" %}
4481 ins_encode %{
4482 __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
4483 __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ D);
4484 %}
4485 ins_pipe(pipe_slow);
4486 %}
4487
4488 instruct vcvtLtoX_narrow(vReg dst, vReg src, vReg tmp)
4489 %{
4490 predicate(UseSVE > 0 && is_integral_type(Matcher::vector_element_basic_type(n)));
4491 match(Set dst (VectorCastL2X src));
4492 effect(TEMP_DEF dst, TEMP tmp);
4493 ins_cost(2 * SVE_COST);
4494 format %{ "sve_vectorcast_l2x $dst, $src\t# convert L to B/H/S vector (narrow)" %}
4495 ins_encode %{
4496 BasicType to_bt = Matcher::vector_element_basic_type(this);
4497 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4498 __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
4499 as_FloatRegister($src$$reg), __ D, as_FloatRegister($tmp$$reg));
4500 %}
4501 ins_pipe(pipe_slow);
4502 %}
4503
4504 instruct vcvtLtoF(vReg dst, vReg src, vReg tmp)
4505 %{
4506 predicate(UseSVE > 0 &&
4507 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4508 match(Set dst (VectorCastL2X src));
4509 effect(TEMP_DEF dst, TEMP tmp);
4510 ins_cost(3 * SVE_COST);
4511 format %{ "sve_vectorcast_l2f $dst, $src\t# convert L to F vector" %}
4512 ins_encode %{
4513 __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
4514 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
4515 as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
4516
4517 %}
4518 ins_pipe(pipe_slow);
4519 %}
4520
4521 instruct vcvtLtoD(vReg dst, vReg src)
4522 %{
4523 predicate(UseSVE > 0 &&
4524 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
4525 match(Set dst (VectorCastL2X src));
4526 ins_cost(SVE_COST);
4527 format %{ "sve_vectorcast_l2d $dst, $src\t# convert L to D vector" %}
4528 ins_encode %{
4529 __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
4530 %}
4531 ins_pipe(pipe_slow);
4532 %}
4533
4534 instruct vcvtFtoX_narrow(vReg dst, vReg src, vReg tmp)
4535 %{
4536 predicate(UseSVE > 0 &&
4537 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4538 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT));
4539 match(Set dst (VectorCastF2X src));
4540 effect(TEMP_DEF dst, TEMP tmp);
4541 ins_cost(3 * SVE_COST);
4542 format %{ "sve_vectorcast_f2x $dst, $src\t# convert F to B/H vector" %}
4543 ins_encode %{
4544 BasicType to_bt = Matcher::vector_element_basic_type(this);
4545 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4546 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
4547 __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
4548 as_FloatRegister($dst$$reg), __ S, as_FloatRegister($tmp$$reg));
4549 %}
4550 ins_pipe(pipe_slow);
4551 %}
4552
4553 instruct vcvtFtoX_extend(vReg dst, vReg src)
4554 %{
4555 predicate(UseSVE > 0 &&
4556 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
4557 n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
4558 match(Set dst (VectorCastF2X src));
4559 ins_cost(SVE_COST);
4560 format %{ "sve_vectorcast_f2x $dst, $src\t# convert F to I/L vector" %}
4561 ins_encode %{
4562 BasicType to_bt = Matcher::vector_element_basic_type(this);
4563 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
4564 if (to_bt == T_LONG) {
4565 __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($dst$$reg), __ S);
4566 }
4567 %}
4568 ins_pipe(pipe_slow);
4569 %}
4570
4571 instruct vcvtFtoD(vReg dst, vReg src)
4572 %{
4573 predicate(UseSVE > 0 &&
4574 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
4575 match(Set dst (VectorCastF2X src));
4576 ins_cost(2 * SVE_COST);
4577 format %{ "sve_vectorcast_f2d $dst, $dst\t# convert F to D vector" %}
4578 ins_encode %{
4579 __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
4580 __ sve_fcvt(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);
4581 %}
4582 ins_pipe(pipe_slow);
4583 %}
4584
4585 instruct vcvtDtoX_narrow(vReg dst, vReg src, vReg tmp)
4586 %{
4587 predicate(UseSVE > 0 &&
4588 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4589 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
4590 n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4591 match(Set dst (VectorCastD2X src));
4592 effect(TEMP_DEF dst, TEMP tmp);
4593 ins_cost(3 * SVE_COST);
4594 format %{ "sve_vectorcast_d2x $dst, $src\t# convert D to X vector (narrow)" %}
4595 ins_encode %{
4596 BasicType to_bt = Matcher::vector_element_basic_type(this);
4597 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4598 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
4599 __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
4600 as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
4601 %}
4602 ins_pipe(pipe_slow);
4603 %}
4604
4605 instruct vcvtDtoL(vReg dst, vReg src)
4606 %{
4607 predicate(UseSVE > 0 &&
4608 n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
4609 match(Set dst (VectorCastD2X src));
4610 ins_cost(SVE_COST);
4611 format %{ "sve_vectorcast_d2l $dst, $src\t# convert D to L vector" %}
4612 ins_encode %{
4613 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
4614 %}
4615 ins_pipe(pipe_slow);
4616 %}
4617
4618 instruct vcvtDtoF(vReg dst, vReg src, vReg tmp)
4619 %{
4620 predicate(UseSVE > 0 &&
4621 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4622 match(Set dst (VectorCastD2X src));
4623 effect(TEMP_DEF dst, TEMP tmp);
4624 ins_cost(3 * SVE_COST);
4625 format %{ "sve_vectorcast_d2f $dst, S, $dst\t# convert D to F vector" %}
4626 ins_encode %{
4627 __ sve_fcvt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
4628 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
4629 as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
4630 %}
4631 ins_pipe(pipe_slow);
4632 %}
4633
4634 // ------------------------------ Vector extract ---------------------------------
4635
4636 instruct extractB(iRegINoSp dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
4637 %{
4638 predicate(UseSVE > 0);
4639 match(Set dst (ExtractB src idx));
4640 effect(TEMP pgtmp, KILL cr);
4641 ins_cost(2 * SVE_COST);
4642 format %{ "sve_extract $dst, B, $pgtmp, $src, $idx\n\t"
4643 "sbfmw $dst, $dst, 0U, 7U\t# extract from vector(B)" %}
4644 ins_encode %{
4645 __ sve_extract(as_Register($dst$$reg), __ B, as_PRegister($pgtmp$$reg),
4646 as_FloatRegister($src$$reg), (int)($idx$$constant));
4647 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 7U);
4648 %}
4649 ins_pipe(pipe_slow);
4650 %}
4651
4652 instruct extractS(iRegINoSp dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
4653 %{
4654 predicate(UseSVE > 0);
4655 match(Set dst (ExtractS src idx));
4656 effect(TEMP pgtmp, KILL cr);
4657 ins_cost(2 * SVE_COST);
4658 format %{ "sve_extract $dst, H, $pgtmp, $src, $idx\n\t"
4659 "sbfmw $dst, $dst, 0U, 15U\t# extract from vector(S)" %}
4660 ins_encode %{
4661 __ sve_extract(as_Register($dst$$reg), __ H, as_PRegister($pgtmp$$reg),
4662 as_FloatRegister($src$$reg), (int)($idx$$constant));
4663 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
4664 %}
4665 ins_pipe(pipe_slow);
4666 %}
4667
4668
4669 instruct extractI(iRegINoSp dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
4670 %{
4671 predicate(UseSVE > 0);
4672 match(Set dst (ExtractI src idx));
4673 effect(TEMP pgtmp, KILL cr);
4674 ins_cost(2 * SVE_COST);
4675 format %{ "sve_extract $dst, S, $pgtmp, $src, $idx\t# extract from vector(I)" %}
4676 ins_encode %{
4677 __ sve_extract(as_Register($dst$$reg), __ S, as_PRegister($pgtmp$$reg),
4678 as_FloatRegister($src$$reg), (int)($idx$$constant));
4679 %}
4680 ins_pipe(pipe_slow);
4681 %}
4682
4683 instruct extractL(iRegLNoSp dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
4684 %{
4685 predicate(UseSVE > 0);
4686 match(Set dst (ExtractL src idx));
4687 effect(TEMP pgtmp, KILL cr);
4688 ins_cost(2 * SVE_COST);
4689 format %{ "sve_extract $dst, D, $pgtmp, $src, $idx\t# extract from vector(L)" %}
4690 ins_encode %{
4691 __ sve_extract(as_Register($dst$$reg), __ D, as_PRegister($pgtmp$$reg),
4692 as_FloatRegister($src$$reg), (int)($idx$$constant));
4693 %}
4694 ins_pipe(pipe_slow);
4695 %}
4696
4697 instruct extractF(vRegF dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
4698 %{
4699 predicate(UseSVE > 0);
4700 match(Set dst (ExtractF src idx));
4701 effect(TEMP pgtmp, KILL cr);
4702 ins_cost(2 * SVE_COST);
4703 format %{ "sve_extract $dst, S, $pgtmp, $src, $idx\t# extract from vector(F)" %}
4704 ins_encode %{
4705 __ sve_extract(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg),
4706 as_FloatRegister($src$$reg), (int)($idx$$constant));
4707 %}
4708 ins_pipe(pipe_slow);
4709 %}
4710
4711 instruct extractD(vRegD dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
4712 %{
4713 predicate(UseSVE > 0);
4714 match(Set dst (ExtractD src idx));
4715 effect(TEMP pgtmp, KILL cr);
4716 ins_cost(2 * SVE_COST);
4717 format %{ "sve_extract $dst, D, $pgtmp, $src, $idx\t# extract from vector(D)" %}
4718 ins_encode %{
4719 __ sve_extract(as_FloatRegister($dst$$reg), __ D, as_PRegister($pgtmp$$reg),
4720 as_FloatRegister($src$$reg), (int)($idx$$constant));
4721 %}
4722 ins_pipe(pipe_slow);
4723 %}
4724
4725 // ------------------------------- VectorTest ----------------------------------
4726
4727 instruct vtest_alltrue(iRegINoSp dst, pRegGov src1, pRegGov src2, pReg ptmp, rFlagsReg cr)
4728 %{
4729 predicate(UseSVE > 0 &&
4730 n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
4731 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
4732 match(Set dst (VectorTest src1 src2));
4733 effect(TEMP ptmp, KILL cr);
4734 ins_cost(SVE_COST);
4735 format %{ "sve_eors $ptmp, $src1, $src2\t# $src2 is all true mask\n"
4736 "csetw $dst, EQ\t# VectorTest (sve) - alltrue" %}
4737 ins_encode %{
4738 __ sve_eors(as_PRegister($ptmp$$reg), ptrue,
4739 as_PRegister($src1$$reg), as_PRegister($src2$$reg));
4740 __ csetw(as_Register($dst$$reg), Assembler::EQ);
4741 %}
4742 ins_pipe(pipe_slow);
4743 %}
4744
4745 instruct vtest_anytrue(iRegINoSp dst, pRegGov src1, pRegGov src2, rFlagsReg cr)
4746 %{
4747 predicate(UseSVE > 0 &&
4748 n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
4749 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
4750 match(Set dst (VectorTest src1 src2));
4751 effect(KILL cr);
4752 ins_cost(SVE_COST);
4753 format %{ "sve_ptest $src1\n\t"
4754 "csetw $dst, NE\t# VectorTest (sve) - anytrue" %}
4755 ins_encode %{
4756 // "src2" is not used for sve.
4757 __ sve_ptest(ptrue, as_PRegister($src1$$reg));
4758 __ csetw(as_Register($dst$$reg), Assembler::NE);
4759 %}
4760 ins_pipe(pipe_slow);
4761 %}
4762
4763 instruct vtest_alltrue_partial(iRegINoSp dst, pRegGov src1, pRegGov src2, pRegGov ptmp, rFlagsReg cr)
4764 %{
4765 predicate(UseSVE > 0 &&
4766 n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
4767 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
4768 match(Set dst (VectorTest src1 src2));
4769 effect(TEMP ptmp, KILL cr);
4770 ins_cost(SVE_COST);
4771 format %{ "vtest_alltrue_partial $dst, $src1, $src2\t# VectorTest partial (sve) - alltrue" %}
4772 ins_encode %{
4773 BasicType bt = Matcher::vector_element_basic_type(this, $src1);
4774 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
4775 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size,
4776 Matcher::vector_length(this, $src1));
4777 __ sve_eors(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
4778 as_PRegister($src1$$reg), as_PRegister($src2$$reg));
4779 __ csetw(as_Register($dst$$reg), Assembler::EQ);
4780 %}
4781 ins_pipe(pipe_slow);
4782 %}
4783
4784 instruct vtest_anytrue_partial(iRegINoSp dst, pRegGov src1, pRegGov src2, pRegGov ptmp, rFlagsReg cr)
4785 %{
4786 predicate(UseSVE > 0 &&
4787 n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
4788 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
4789 match(Set dst (VectorTest src1 src2));
4790 effect(TEMP ptmp, KILL cr);
4791 ins_cost(SVE_COST);
4792 format %{ "vtest_anytrue_partial $dst, $src1, $src2\t# VectorTest partial (sve) - anytrue" %}
4793 ins_encode %{
4794 BasicType bt = Matcher::vector_element_basic_type(this, $src1);
4795 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
4796 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size,
4797 Matcher::vector_length(this, $src1));
4798 __ sve_ands(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
4799 as_PRegister($src1$$reg), as_PRegister($src2$$reg));
4800 __ csetw(as_Register($dst$$reg), Assembler::NE);
4801 %}
4802 ins_pipe(pipe_slow);
4803 %}
4804
4805 // ------------------------------ Vector insert ---------------------------------
4806
4807 instruct insertI_small(vReg dst, vReg src, iRegIorL2I val, immI idx, pRegGov pgtmp, rFlagsReg cr)
4808 %{
4809 predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
4810 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4811 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
4812 n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4813 match(Set dst (VectorInsert (Binary src val) idx));
4814 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
4815 ins_cost(4 * SVE_COST);
4816 format %{ "sve_index $dst, -16, 1\t# (B/H/S)\n\t"
4817 "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
4818 "sve_orr $dst, $src, $src\n\t"
4819 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
4820 ins_encode %{
4821 BasicType bt = Matcher::vector_element_basic_type(this, $src);
4822 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
4823 __ sve_index(as_FloatRegister($dst$$reg), size, -16, 1);
4824 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
4825 as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
4826 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4827 __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
4828 %}
4829 ins_pipe(pipe_slow);
4830 %}
4831
4832 instruct insertF_small(vReg dst, vReg src, vRegF val, immI idx, pRegGov pgtmp, rFlagsReg cr)
4833 %{
4834 predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
4835 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4836 match(Set dst (VectorInsert (Binary src val) idx));
4837 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
4838 ins_cost(4 * SVE_COST);
4839 format %{ "sve_index $dst, S, -16, 1\n\t"
4840 "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
4841 "sve_orr $dst, $src, $src\n\t"
4842 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
4843 ins_encode %{
4844 __ sve_index(as_FloatRegister($dst$$reg), __ S, -16, 1);
4845 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
4846 as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
4847 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4848 __ sve_cpy(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
4849 %}
4850 ins_pipe(pipe_slow);
4851 %}
4852
4853 instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr)
4854 %{
4855 predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
4856 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4857 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
4858 n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4859 match(Set dst (VectorInsert (Binary src val) idx));
4860 effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
4861 ins_cost(5 * SVE_COST);
4862 format %{ "sve_index $tmp1, 0, 1\t# (B/H/S)\n\t"
4863 "sve_dup $dst, $idx\t# (B/H/S)\n\t"
4864 "sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
4865 "sve_orr $dst, $src, $src\n\t"
4866 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
4867 ins_encode %{
4868 BasicType bt = Matcher::vector_element_basic_type(this, $src);
4869 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
4870 __ sve_index(as_FloatRegister($tmp1$$reg), size, 0, 1);
4871 __ sve_dup(as_FloatRegister($dst$$reg), size, (int)($idx$$constant));
4872 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
4873 as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
4874 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4875 __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
4876 %}
4877 ins_pipe(pipe_slow);
4878 %}
4879
4880 instruct insertL(vReg dst, vReg src, iRegL val, immI idx, pRegGov pgtmp, rFlagsReg cr)
4881 %{
4882 predicate(UseSVE > 0 &&
4883 n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
4884 match(Set dst (VectorInsert (Binary src val) idx));
4885 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
4886 ins_cost(4 * SVE_COST);
4887 format %{ "sve_index $dst, D, -16, 1\n\t"
4888 "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
4889 "sve_orr $dst, $src, $src\n\t"
4890 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (L)" %}
4891 ins_encode %{
4892 __ sve_index(as_FloatRegister($dst$$reg), __ D, -16, 1);
4893 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
4894 as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
4895 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4896 __ sve_cpy(as_FloatRegister($dst$$reg), __ D, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
4897 %}
4898 ins_pipe(pipe_slow);
4899 %}
4900
4901 instruct insertD(vReg dst, vReg src, vRegD val, immI idx, pRegGov pgtmp, rFlagsReg cr)
4902 %{
4903 predicate(UseSVE > 0 &&
4904 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
4905 match(Set dst (VectorInsert (Binary src val) idx));
4906 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
4907 ins_cost(4 * SVE_COST);
4908 format %{ "sve_index $dst, D, -16, 1\n\t"
4909 "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
4910 "sve_orr $dst, $src, $src\n\t"
4911 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (D)" %}
4912 ins_encode %{
4913 __ sve_index(as_FloatRegister($dst$$reg), __ D, -16, 1);
4914 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
4915 as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
4916 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4917 __ sve_cpy(as_FloatRegister($dst$$reg), __ D, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
4918 %}
4919 ins_pipe(pipe_slow);
4920 %}
4921
4922 instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr)
4923 %{
4924 predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
4925 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4926 match(Set dst (VectorInsert (Binary src val) idx));
4927 effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
4928 ins_cost(5 * SVE_COST);
4929 format %{ "sve_index $tmp1, S, 0, 1\n\t"
4930 "sve_dup $dst, S, $idx\n\t"
4931 "sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
4932 "sve_orr $dst, $src, $src\n\t"
4933 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
4934 ins_encode %{
4935 __ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
4936 __ sve_dup(as_FloatRegister($dst$$reg), __ S, (int)($idx$$constant));
4937 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
4938 as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
4939 __ sve_orr(as_FloatRegister($dst$$reg),
4940 as_FloatRegister($src$$reg),
4941 as_FloatRegister($src$$reg));
4942 __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
4943 as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
4944 %}
4945 ins_pipe(pipe_slow);
4946 %}
4947
4948 // ------------------------------ Vector shuffle -------------------------------
4949
4950 instruct loadshuffle(vReg dst, vReg src) %{
4951 predicate(UseSVE > 0);
4952 match(Set dst (VectorLoadShuffle src));
4953 ins_cost(SVE_COST);
4954 format %{ "sve_loadshuffle $dst, $src\t# vector load shuffle (B/H/S/D)" %}
4955 ins_encode %{
4956 BasicType bt = Matcher::vector_element_basic_type(this);
4957 if (bt == T_BYTE) {
4958 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
4959 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
4960 as_FloatRegister($src$$reg));
4961 }
4962 } else {
4963 __ sve_vector_extend(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
4964 as_FloatRegister($src$$reg), __ B);
4965 }
4966 %}
4967 ins_pipe(pipe_slow);
4968 %}
4969
4970 // ------------------------------ Vector rearrange -------------------------------
4971
4972 instruct rearrange(vReg dst, vReg src, vReg shuffle)
4973 %{
4974 predicate(UseSVE > 0);
4975 match(Set dst (VectorRearrange src shuffle));
4976 ins_cost(SVE_COST);
4977 format %{ "sve_tbl $dst, $src, $shuffle\t# vector rearrange" %}
4978 ins_encode %{
4979 BasicType bt = Matcher::vector_element_basic_type(this, $src);
4980 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
4981 __ sve_tbl(as_FloatRegister($dst$$reg), size,
4982 as_FloatRegister($src$$reg), as_FloatRegister($shuffle$$reg));
4983 %}
4984 ins_pipe(pipe_slow);
4985 %}
4986
4987 // ------------------------------ Vector Load Gather ---------------------------------
4988
4989 instruct gatherI(vReg dst, indirect mem, vReg idx) %{
4990 predicate(UseSVE > 0 &&
4991 n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
4992 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
4993 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
4994 match(Set dst (LoadVectorGather mem idx));
4995 ins_cost(SVE_COST);
4996 format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (S)" %}
4997 ins_encode %{
4998 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), ptrue,
4999 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5000 %}
5001 ins_pipe(pipe_slow);
5002 %}
5003
5004 instruct gatherL(vReg dst, indirect mem, vReg idx) %{
5005 predicate(UseSVE > 0 &&
5006 n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
5007 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5008 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5009 match(Set dst (LoadVectorGather mem idx));
5010 ins_cost(2 * SVE_COST);
5011 format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (D)" %}
5012 ins_encode %{
5013 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5014 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), ptrue, as_Register($mem$$base),
5015 as_FloatRegister($idx$$reg));
5016 %}
5017 ins_pipe(pipe_slow);
5018 %}
5019
5020 // ------------------------------ Vector Load Gather Partial-------------------------------
5021
5022 instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5023 predicate(UseSVE > 0 &&
5024 n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
5025 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5026 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5027 match(Set dst (LoadVectorGather mem idx));
5028 effect(TEMP ptmp, KILL cr);
5029 ins_cost(2 * SVE_COST + INSN_COST);
5030 format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (S)" %}
5031 ins_encode %{
5032 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this));
5033 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
5034 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5035 %}
5036 ins_pipe(pipe_slow);
5037 %}
5038
5039 instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5040 predicate(UseSVE > 0 &&
5041 n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
5042 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5043 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5044 match(Set dst (LoadVectorGather mem idx));
5045 effect(TEMP ptmp, KILL cr);
5046 ins_cost(3 * SVE_COST + INSN_COST);
5047 format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (D)" %}
5048 ins_encode %{
5049 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
5050 Matcher::vector_length(this));
5051 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5052 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
5053 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5054 %}
5055 ins_pipe(pipe_slow);
5056 %}
5057
5058 // ------------------------------ Vector Load Gather Predicated -------------------------------
5059
5060 instruct gatherI_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
5061 predicate(UseSVE > 0 &&
5062 n->as_LoadVector()->memory_size() == MaxVectorSize &&
5063 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5064 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5065 match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
5066 ins_cost(SVE_COST);
5067 format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (S)" %}
5068 ins_encode %{
5069 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
5070 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5071 %}
5072 ins_pipe(pipe_slow);
5073 %}
5074
5075 instruct gatherL_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
5076 predicate(UseSVE > 0 &&
5077 n->as_LoadVector()->memory_size() == MaxVectorSize &&
5078 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5079 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5080 match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
5081 ins_cost(2 * SVE_COST);
5082 format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (D)" %}
5083 ins_encode %{
5084 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5085 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
5086 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5087 %}
5088 ins_pipe(pipe_slow);
5089 %}
5090
5091 // ------------------------------ Vector Load Gather Predicated Partial -------------------------------
5092
5093 instruct gatherI_masked_partial(vReg dst, indirect mem, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
5094 predicate(UseSVE > 0 &&
5095 n->as_LoadVector()->memory_size() < MaxVectorSize &&
5096 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5097 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5098 match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
5099 effect(TEMP ptmp, KILL cr);
5100 ins_cost(3 * SVE_COST);
5101 format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated partial (S)" %}
5102 ins_encode %{
5103 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
5104 Matcher::vector_length(this));
5105 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
5106 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
5107 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
5108 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5109 %}
5110 ins_pipe(pipe_slow);
5111 %}
5112
5113 instruct gatherL_masked_partial(vReg dst, indirect mem, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
5114 predicate(UseSVE > 0 &&
5115 n->as_LoadVector()->memory_size() < MaxVectorSize &&
5116 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5117 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5118 match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
5119 effect(TEMP ptmp, KILL cr);
5120 ins_cost(4 * SVE_COST);
5121 format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated partial (D)" %}
5122 ins_encode %{
5123 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this));
5124 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
5125 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
5126 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5127 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
5128 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5129 %}
5130 ins_pipe(pipe_slow);
5131 %}
5132
5133 // ------------------------------ Vector Store Scatter -------------------------------
5134
5135 instruct scatterI(indirect mem, vReg src, vReg idx) %{
5136 predicate(UseSVE > 0 &&
5137 n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
5138 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5139 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5140 match(Set mem (StoreVectorScatter mem (Binary src idx)));
5141 ins_cost(SVE_COST);
5142 format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (S)" %}
5143 ins_encode %{
5144 __ sve_st1w_scatter(as_FloatRegister($src$$reg), ptrue,
5145 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5146 %}
5147 ins_pipe(pipe_slow);
5148 %}
5149
5150 instruct scatterL(indirect mem, vReg src, vReg idx) %{
5151 predicate(UseSVE > 0 &&
5152 n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
5153 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5154 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5155 match(Set mem (StoreVectorScatter mem (Binary src idx)));
5156 ins_cost(2 * SVE_COST);
5157 format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (D)" %}
5158 ins_encode %{
5159 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5160 __ sve_st1d_scatter(as_FloatRegister($src$$reg), ptrue,
5161 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5162 %}
5163 ins_pipe(pipe_slow);
5164 %}
5165
5166 // ------------------------------ Vector Store Scatter Partial -------------------------------
5167
5168 instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5169 predicate(UseSVE > 0 &&
5170 n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
5171 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5172 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5173 match(Set mem (StoreVectorScatter mem (Binary src idx)));
5174 effect(TEMP ptmp, KILL cr);
5175 ins_cost(2 * SVE_COST + INSN_COST);
5176 format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (S)" %}
5177 ins_encode %{
5178 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
5179 Matcher::vector_length(this, $src));
5180 __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
5181 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5182 %}
5183 ins_pipe(pipe_slow);
5184 %}
5185
5186 instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5187 predicate(UseSVE > 0 &&
5188 n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
5189 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5190 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5191 match(Set mem (StoreVectorScatter mem (Binary src idx)));
5192 effect(TEMP ptmp, KILL cr);
5193 ins_cost(3 * SVE_COST + INSN_COST);
5194 format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (D)" %}
5195 ins_encode %{
5196 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
5197 Matcher::vector_length(this, $src));
5198 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5199 __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
5200 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5201 %}
5202 ins_pipe(pipe_slow);
5203 %}
5204
5205 // ------------------------------ Vector Store Scatter Predicated -------------------------------
5206
5207 instruct scatterI_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
5208 predicate(UseSVE > 0 &&
5209 n->as_StoreVector()->memory_size() == MaxVectorSize &&
5210 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5211 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5212 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
5213 ins_cost(SVE_COST);
5214 format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicate (S)" %}
5215 ins_encode %{
5216 __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
5217 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5218 %}
5219 ins_pipe(pipe_slow);
5220 %}
5221
5222 instruct scatterL_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
5223 predicate(UseSVE > 0 &&
5224 n->as_StoreVector()->memory_size() == MaxVectorSize &&
5225 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5226 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5227 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
5228 ins_cost(2 * SVE_COST);
5229 format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated (D)" %}
5230 ins_encode %{
5231 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5232 __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
5233 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5234 %}
5235 ins_pipe(pipe_slow);
5236 %}
5237
5238 // ------------------------------ Vector Store Scatter Predicated Partial -------------------------------
5239
5240 instruct scatterI_masked_partial(indirect mem, vReg src, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
5241 predicate(UseSVE > 0 &&
5242 n->as_StoreVector()->memory_size() < MaxVectorSize &&
5243 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5244 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5245 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
5246 effect(TEMP ptmp, KILL cr);
5247 ins_cost(3 * SVE_COST);
5248 format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated partial (S)" %}
5249 ins_encode %{
5250 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
5251 Matcher::vector_length(this, $src));
5252 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
5253 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
5254 __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
5255 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5256 %}
5257 ins_pipe(pipe_slow);
5258 %}
5259
5260 instruct scatterL_masked_partial(indirect mem, vReg src, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
5261 predicate(UseSVE > 0 &&
5262 n->as_StoreVector()->memory_size() < MaxVectorSize &&
5263 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5264 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5265 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
5266 effect(TEMP ptmp, KILL cr);
5267 ins_cost(4 * SVE_COST);
5268 format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated partial (D)" %}
5269 ins_encode %{
5270 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
5271 Matcher::vector_length(this, $src));
5272 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
5273 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
5274 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5275 __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
5276 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5277 %}
5278 ins_pipe(pipe_slow);
5279 %}
5280
5281 // ------------------------------ Vector Load Const -------------------------------
5282
5283 instruct loadconB(vReg dst, immI0 src) %{
5284 predicate(UseSVE > 0 &&
5285 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
5286 match(Set dst (VectorLoadConst src));
5287 ins_cost(SVE_COST);
5288 format %{ "sve_index $dst, 0, 1\t# generate iota indices" %}
5289 ins_encode %{
5290 __ sve_index(as_FloatRegister($dst$$reg), __ B, 0, 1);
5291 %}
5292 ins_pipe(pipe_slow);
5293 %}
5294
5295 // Intrisics for String.indexOf(char)
5296
5297
5298 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
5299 iRegI_R0 result, vReg ztmp1, vReg ztmp2,
5300 pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
5301 %{
5302 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
5303 predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
5304 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
5305
5306 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
5307
5308 ins_encode %{
5309 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
5310 as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
5311 as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), true /* isL */);
5312 %}
5313 ins_pipe(pipe_class_memory);
5314 %}
5315
5316 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
5317 iRegI_R0 result, vReg ztmp1, vReg ztmp2,
5318 pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
5319 %{
5320 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
5321 predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
5322 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
5323
5324 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
5325
5326 ins_encode %{
5327 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
5328 as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
5329 as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), false /* isL */);
5330 %}
5331 ins_pipe(pipe_class_memory);
5332 %}
5333
5334 // ---------------------------- Vector mask reductions ---------------------------
5335 instruct vmask_truecount(iRegINoSp dst, pReg src) %{
5336 predicate(UseSVE > 0 &&
5337 n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
5338 match(Set dst (VectorMaskTrueCount src));
5339 ins_cost(SVE_COST);
5340 format %{ "vmask_truecount $dst, $src\t# vector mask truecount (sve)" %}
5341 ins_encode %{
5342 BasicType bt = Matcher::vector_element_basic_type(this, $src);
5343 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5344 __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($src$$reg));
5345 %}
5346 ins_pipe(pipe_slow);
5347 %}
5348
5349 instruct vmask_firsttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
5350 predicate(UseSVE > 0 &&
5351 n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
5352 match(Set dst (VectorMaskFirstTrue src));
5353 effect(TEMP ptmp);
5354 ins_cost(2 * SVE_COST);
5355 format %{ "vmask_firsttrue $dst, $src\t# vector mask firsttrue (sve)" %}
5356 ins_encode %{
5357 BasicType bt = Matcher::vector_element_basic_type(this, $src);
5358 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5359 __ sve_brkb(as_PRegister($ptmp$$reg), ptrue, as_PRegister($src$$reg), false);
5360 __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($ptmp$$reg));
5361 %}
5362 ins_pipe(pipe_slow);
5363 %}
5364
5365 instruct vmask_lasttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
5366 predicate(UseSVE > 0 &&
5367 n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
5368 match(Set dst (VectorMaskLastTrue src));
5369 effect(TEMP ptmp);
5370 ins_cost(3 * SVE_COST);
5371 format %{ "vmask_lasttrue $dst, $src\t# vector mask lasttrue (sve)" %}
5372 ins_encode %{
5373 BasicType bt = Matcher::vector_element_basic_type(this, $src);
5374 __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($src$$reg), as_PRegister($ptmp$$reg));
5375 %}
5376 ins_pipe(pipe_slow);
5377 %}
5378
5379 instruct vmask_truecount_partial(iRegINoSp dst, pReg src, pRegGov pgtmp, rFlagsReg cr) %{
5380 predicate(UseSVE > 0 &&
5381 n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
5382 match(Set dst (VectorMaskTrueCount src));
5383 effect(TEMP pgtmp, KILL cr);
5384 ins_cost(2 * SVE_COST);
5385 format %{ "vmask_truecount_partial $dst, $src\t# vector mask truecount partial (sve)" %}
5386 ins_encode %{
5387 BasicType bt = Matcher::vector_element_basic_type(this, $src);
5388 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5389 __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), size, Matcher::vector_length(this, $src));
5390 __ sve_cntp($dst$$Register, size, as_PRegister($pgtmp$$reg), as_PRegister($src$$reg));
5391 %}
5392 ins_pipe(pipe_slow);
5393 %}
5394
5395 instruct vmask_firsttrue_partial(iRegINoSp dst, pReg src, pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
5396 predicate(UseSVE > 0 &&
5397 n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
5398 match(Set dst (VectorMaskFirstTrue src));
5399 effect(TEMP pgtmp, TEMP ptmp, KILL cr);
5400 ins_cost(3 * SVE_COST);
5401 format %{ "vmask_firsttrue_partial $dst, $src\t# vector mask firsttrue partial (sve)" %}
5402 ins_encode %{
5403 BasicType bt = Matcher::vector_element_basic_type(this, $src);
5404 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5405 __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), size,
5406 Matcher::vector_length(this, $src));
5407 __ sve_brkb(as_PRegister($ptmp$$reg), as_PRegister($pgtmp$$reg), as_PRegister($src$$reg), false);
5408 __ sve_cntp($dst$$Register, size, as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg));
5409 %}
5410 ins_pipe(pipe_slow);
5411 %}
5412
5413 instruct vmask_lasttrue_partial(iRegINoSp dst, pReg src, pReg ptmp, rFlagsReg cr) %{
5414 predicate(UseSVE > 0 &&
5415 n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
5416 match(Set dst (VectorMaskLastTrue src));
5417 effect(TEMP ptmp, KILL cr);
5418 ins_cost(5 * SVE_COST);
5419 format %{ "vmask_lasttrue_partial $dst, $src\t# vector mask lasttrue partial (sve)" %}
5420 ins_encode %{
5421 BasicType bt = Matcher::vector_element_basic_type(this, $src);
5422 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5423 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
5424 __ sve_and(as_PRegister($ptmp$$reg), ptrue, as_PRegister($ptmp$$reg), as_PRegister($src$$reg));
5425 __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg));
5426 %}
5427 ins_pipe(pipe_slow);
5428 %}