1 //
2 // Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2020, 2022, Arm Limited. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // This file is automatically generated by running "m4 aarch64_sve_ad.m4". Do not edit ----
27
28 // AArch64 SVE Architecture Description File
29
30
31 // 4 bit signed offset -- for predicated load/store
32
33 operand vmemA_immIOffset4()
34 %{
35 // (esize / msize) = 1
36 predicate(Address::offset_ok_for_sve_immed(n->get_int(), 4,
37 Matcher::scalable_vector_reg_size(T_BYTE)));
38 match(ConI);
39
40 op_cost(0);
41 format %{ %}
42 interface(CONST_INTER);
43 %}
44
45 operand vmemA_immLOffset4()
46 %{
47 // (esize / msize) = 1
48 predicate(Address::offset_ok_for_sve_immed(n->get_long(), 4,
49 Matcher::scalable_vector_reg_size(T_BYTE)));
50 match(ConL);
51
52 op_cost(0);
53 format %{ %}
54 interface(CONST_INTER);
55 %}
56
57 operand vmemA_indOffI4(iRegP reg, vmemA_immIOffset4 off)
58 %{
59 constraint(ALLOC_IN_RC(ptr_reg));
60 match(AddP reg off);
61 op_cost(0);
62 format %{ "[$reg, $off]" %}
63 interface(MEMORY_INTER) %{
64 base($reg);
65 index(0xffffffff);
66 scale(0x0);
67 disp($off);
68 %}
69 %}
70
71 operand vmemA_indOffL4(iRegP reg, vmemA_immLOffset4 off)
72 %{
73 constraint(ALLOC_IN_RC(ptr_reg));
74 match(AddP reg off);
75 op_cost(0);
76 format %{ "[$reg, $off]" %}
77 interface(MEMORY_INTER) %{
78 base($reg);
79 index(0xffffffff);
80 scale(0x0);
81 disp($off);
82 %}
83 %}
84
85 // The indOff of vmemA is valid only when the vector element (load to/store from)
86 // size equals to memory element (load from/store to) size.
87 opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);
88
89 source_hpp %{
90 bool op_sve_supported(int opcode, int vlen, BasicType bt);
91 bool masked_op_sve_supported(int opcode, int vlen, BasicType bt);
92 %}
93
94 source %{
95
96 typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
97 PRegister Pg, const Address &adr);
98
99 // Predicated load/store, with optional ptrue to all elements of given predicate register.
100 static void loadStoreA_predicated(C2_MacroAssembler masm, bool is_store, FloatRegister reg,
101 PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
102 int opcode, Register base, int index, int size, int disp) {
103 sve_mem_insn_predicate insn;
104 int mesize = type2aelembytes(mem_elem_bt);
105 if (index == -1) {
106 assert(size == 0, "unsupported address mode: scale size = %d", size);
107 switch(mesize) {
108 case 1:
109 insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
110 break;
111 case 2:
112 insn = is_store ? &C2_MacroAssembler::sve_st1h : &C2_MacroAssembler::sve_ld1h;
113 break;
114 case 4:
115 insn = is_store ? &C2_MacroAssembler::sve_st1w : &C2_MacroAssembler::sve_ld1w;
116 break;
117 case 8:
118 insn = is_store ? &C2_MacroAssembler::sve_st1d : &C2_MacroAssembler::sve_ld1d;
119 break;
120 default:
121 assert(false, "unsupported");
122 ShouldNotReachHere();
123 }
124 int imm4 = disp / mesize / Matcher::scalable_vector_reg_size(vector_elem_bt);
125 (masm.*insn)(reg, Assembler::elemType_to_regVariant(vector_elem_bt), pg, Address(base, imm4));
126 } else {
127 assert(false, "unimplemented");
128 ShouldNotReachHere();
129 }
130 }
131
132 bool op_sve_supported(int opcode, int vlen, BasicType bt) {
133 int length_in_bytes = vlen * type2aelembytes(bt);
134 switch (opcode) {
135 case Op_MulAddVS2VI:
136 // No multiply reduction instructions
137 case Op_MulReductionVD:
138 case Op_MulReductionVF:
139 case Op_MulReductionVI:
140 case Op_MulReductionVL:
141 // Others
142 case Op_ExtractC:
143 case Op_ExtractUB:
144 return false;
145 // Vector API specific
146 case Op_VectorLoadShuffle:
147 case Op_VectorRearrange:
148 return vlen >= 4 && length_in_bytes <= MaxVectorSize;
149 case Op_LoadVector:
150 case Op_StoreVector:
151 return Matcher::vector_size_supported(bt, vlen);
152 case Op_ExpandV:
153 if (UseSVE < 2 || is_subword_type(bt)) return false;
154 default:
155 break;
156 }
157 // By default, we only support vector operations with no less than 8 bytes and 2 elements.
158 return 8 <= length_in_bytes && length_in_bytes <= MaxVectorSize && vlen >= 2;
159 }
160
161 bool masked_op_sve_supported(int opcode, int vlen, BasicType bt) {
162 if (opcode == Op_VectorRearrange) {
163 return false;
164 }
165 return op_sve_supported(opcode, vlen, bt);
166 }
167 %}
168
169 definitions %{
170 int_def SVE_COST (200, 200);
171 %}
172
173
174 // All SVE instructions
175
176 // vector load/store
177
178 // Unpredicated vector load/store
179 instruct loadV(vReg dst, vmemA mem) %{
180 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() >= 16 &&
181 n->as_LoadVector()->memory_size() == MaxVectorSize);
182 match(Set dst (LoadVector mem));
183 ins_cost(4 * SVE_COST);
184 format %{ "sve_ldr $dst, $mem\t# vector (sve)" %}
185 ins_encode %{
186 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
187 BasicType bt = Matcher::vector_element_basic_type(this);
188 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg, ptrue,
189 bt, bt, $mem->opcode(),
190 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
191 %}
192 ins_pipe(pipe_slow);
193 %}
194
195 instruct storeV(vReg src, vmemA mem) %{
196 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() >= 16 &&
197 n->as_StoreVector()->memory_size() == MaxVectorSize);
198 match(Set mem (StoreVector mem src));
199 ins_cost(4 * SVE_COST);
200 format %{ "sve_str $mem, $src\t# vector (sve)" %}
201 ins_encode %{
202 FloatRegister src_reg = as_FloatRegister($src$$reg);
203 BasicType bt = Matcher::vector_element_basic_type(this, $src);
204 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg, ptrue,
205 bt, bt, $mem->opcode(),
206 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
207 %}
208 ins_pipe(pipe_slow);
209 %}
210
211 // Load Vector (16 bits)
212 instruct loadV2_vreg(vReg dst, vmem2 mem)
213 %{
214 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 2);
215 match(Set dst (LoadVector mem));
216 ins_cost(4 * INSN_COST);
217 format %{ "ldrh $dst,$mem\t# vector (16 bits)" %}
218 ins_encode( aarch64_enc_ldrvH(dst, mem) );
219 ins_pipe(vload_reg_mem64);
220 %}
221
222 // Store Vector (16 bits)
223 instruct storeV2_vreg(vReg src, vmem2 mem)
224 %{
225 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 2);
226 match(Set mem (StoreVector mem src));
227 ins_cost(4 * INSN_COST);
228 format %{ "strh $mem,$src\t# vector (16 bits)" %}
229 ins_encode( aarch64_enc_strvH(src, mem) );
230 ins_pipe(vstore_reg_mem64);
231 %}
232
233 // Load Vector (32 bits)
234 instruct loadV4_vreg(vReg dst, vmem4 mem)
235 %{
236 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 4);
237 match(Set dst (LoadVector mem));
238 ins_cost(4 * INSN_COST);
239 format %{ "ldrs $dst,$mem\t# vector (32 bits)" %}
240 ins_encode( aarch64_enc_ldrvS(dst, mem) );
241 ins_pipe(vload_reg_mem64);
242 %}
243
244 // Store Vector (32 bits)
245 instruct storeV4_vreg(vReg src, vmem4 mem)
246 %{
247 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 4);
248 match(Set mem (StoreVector mem src));
249 ins_cost(4 * INSN_COST);
250 format %{ "strs $mem,$src\t# vector (32 bits)" %}
251 ins_encode( aarch64_enc_strvS(src, mem) );
252 ins_pipe(vstore_reg_mem64);
253 %}
254
255 // Load Vector (64 bits)
256 instruct loadV8_vreg(vReg dst, vmem8 mem)
257 %{
258 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 8);
259 match(Set dst (LoadVector mem));
260 ins_cost(4 * INSN_COST);
261 format %{ "ldrd $dst,$mem\t# vector (64 bits)" %}
262 ins_encode( aarch64_enc_ldrvD(dst, mem) );
263 ins_pipe(vload_reg_mem64);
264 %}
265
266 // Store Vector (64 bits)
267 instruct storeV8_vreg(vReg src, vmem8 mem)
268 %{
269 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 8);
270 match(Set mem (StoreVector mem src));
271 ins_cost(4 * INSN_COST);
272 format %{ "strd $mem,$src\t# vector (64 bits)" %}
273 ins_encode( aarch64_enc_strvD(src, mem) );
274 ins_pipe(vstore_reg_mem64);
275 %}
276
277 // Load Vector (128 bits)
278 instruct loadV16_vreg(vReg dst, vmem16 mem)
279 %{
280 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 16);
281 match(Set dst (LoadVector mem));
282 ins_cost(4 * INSN_COST);
283 format %{ "ldrq $dst,$mem\t# vector (128 bits)" %}
284 ins_encode( aarch64_enc_ldrvQ(dst, mem) );
285 ins_pipe(vload_reg_mem128);
286 %}
287
288 // Store Vector (128 bits)
289 instruct storeV16_vreg(vReg src, vmem16 mem)
290 %{
291 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 16);
292 match(Set mem (StoreVector mem src));
293 ins_cost(4 * INSN_COST);
294 format %{ "strq $mem,$src\t# vector (128 bits)" %}
295 ins_encode( aarch64_enc_strvQ(src, mem) );
296 ins_pipe(vstore_reg_mem128);
297 %}
298
299 // Predicated vector load/store, based on the vector length of the node.
300 // Only load/store values in the range of the memory_size. This is needed
301 // when the memory_size is lower than the hardware supported max vector size.
302 // And this might happen for Vector API mask vector load/store.
303 instruct loadV_partial(vReg dst, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
304 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() > 16 &&
305 n->as_LoadVector()->memory_size() < MaxVectorSize);
306 match(Set dst (LoadVector mem));
307 effect(TEMP pgtmp, KILL cr);
308 ins_cost(6 * SVE_COST);
309 format %{ "sve_ptrue $pgtmp, vector_length\n\t"
310 "sve_ldr $dst, $pgtmp, $mem\t# load vector partial" %}
311 ins_encode %{
312 BasicType bt = Matcher::vector_element_basic_type(this);
313 __ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
314 Matcher::vector_length(this));
315 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
316 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg,
317 as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
318 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
319 %}
320 ins_pipe(pipe_slow);
321 %}
322
323 instruct storeV_partial(vReg src, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
324 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() > 16 &&
325 n->as_StoreVector()->memory_size() < MaxVectorSize);
326 match(Set mem (StoreVector mem src));
327 effect(TEMP pgtmp, KILL cr);
328 ins_cost(5 * SVE_COST);
329 format %{ "sve_ptrue $pgtmp, vector_length\n\t"
330 "sve_str $src, $pgtmp, $mem\t# store vector partial" %}
331 ins_encode %{
332 BasicType bt = Matcher::vector_element_basic_type(this, $src);
333 __ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
334 Matcher::vector_length(this, $src));
335 FloatRegister src_reg = as_FloatRegister($src$$reg);
336 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg,
337 as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
338 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
339 %}
340 ins_pipe(pipe_slow);
341 %}
342
343 // vector load/store - predicated
344
345 instruct loadV_masked(vReg dst, vmemA mem, pRegGov pg) %{
346 predicate(UseSVE > 0);
347 match(Set dst (LoadVectorMasked mem pg));
348 ins_cost(4 * SVE_COST);
349 format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated (sve)" %}
350 ins_encode %{
351 BasicType bt = Matcher::vector_element_basic_type(this);
352 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
353 as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
354 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
355 %}
356 ins_pipe(pipe_slow);
357 %}
358
359 instruct storeV_masked(vReg src, vmemA mem, pRegGov pg) %{
360 predicate(UseSVE > 0);
361 match(Set mem (StoreVectorMasked mem (Binary src pg)));
362 ins_cost(4 * SVE_COST);
363 format %{ "sve_str $mem, $pg, $src\t# store vector predicated (sve)" %}
364 ins_encode %{
365 BasicType bt = Matcher::vector_element_basic_type(this, $src);
366 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
367 as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
368 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
369 %}
370 ins_pipe(pipe_slow);
371 %}
372
373 // mask logical and/or/xor
374
375 instruct vmask_and(pRegGov pd, pRegGov pn, pRegGov pm) %{
376 predicate(UseSVE > 0);
377 match(Set pd (AndVMask pn pm));
378 ins_cost(SVE_COST);
379 format %{ "sve_and $pd, $pn, $pm\t# predicate (sve)" %}
380 ins_encode %{
381 __ sve_and(as_PRegister($pd$$reg), ptrue,
382 as_PRegister($pn$$reg), as_PRegister($pm$$reg));
383 %}
384 ins_pipe(pipe_slow);
385 %}
386
387 instruct vmask_or(pRegGov pd, pRegGov pn, pRegGov pm) %{
388 predicate(UseSVE > 0);
389 match(Set pd (OrVMask pn pm));
390 ins_cost(SVE_COST);
391 format %{ "sve_orr $pd, $pn, $pm\t# predicate (sve)" %}
392 ins_encode %{
393 __ sve_orr(as_PRegister($pd$$reg), ptrue,
394 as_PRegister($pn$$reg), as_PRegister($pm$$reg));
395 %}
396 ins_pipe(pipe_slow);
397 %}
398
399 instruct vmask_xor(pRegGov pd, pRegGov pn, pRegGov pm) %{
400 predicate(UseSVE > 0);
401 match(Set pd (XorVMask pn pm));
402 ins_cost(SVE_COST);
403 format %{ "sve_eor $pd, $pn, $pm\t# predicate (sve)" %}
404 ins_encode %{
405 __ sve_eor(as_PRegister($pd$$reg), ptrue,
406 as_PRegister($pn$$reg), as_PRegister($pm$$reg));
407 %}
408 ins_pipe(pipe_slow);
409 %}
410
411 // mask logical and_not
412
413 instruct vmask_and_notI(pRegGov pd, pRegGov pn, pRegGov pm, immI_M1 m1) %{
414 predicate(UseSVE > 0);
415 match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
416 ins_cost(SVE_COST);
417 format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) (B/H/S)" %}
418 ins_encode %{
419 __ sve_bic(as_PRegister($pd$$reg), ptrue,
420 as_PRegister($pn$$reg), as_PRegister($pm$$reg));
421 %}
422 ins_pipe(pipe_slow);
423 %}
424
425 instruct vmask_and_notL(pRegGov pd, pRegGov pn, pRegGov pm, immL_M1 m1) %{
426 predicate(UseSVE > 0);
427 match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
428 ins_cost(SVE_COST);
429 format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) (D)" %}
430 ins_encode %{
431 __ sve_bic(as_PRegister($pd$$reg), ptrue,
432 as_PRegister($pn$$reg), as_PRegister($pm$$reg));
433 %}
434 ins_pipe(pipe_slow);
435 %}
436
437 // vector reinterpret
438
439 instruct reinterpret(vReg dst) %{
440 predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() ==
441 n->in(1)->bottom_type()->is_vect()->length_in_bytes()); // src == dst
442 match(Set dst (VectorReinterpret dst));
443 ins_cost(0);
444 format %{ "# reinterpret $dst\t# do nothing" %}
445 ins_encode %{
446 // empty
447 %}
448 ins_pipe(pipe_class_empty);
449 %}
450
451 instruct reinterpretResize(vReg dst, vReg src, pRegGov pgtmp, rFlagsReg cr) %{
452 predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() !=
453 n->in(1)->bottom_type()->is_vect()->length_in_bytes()); // src != dst
454 match(Set dst (VectorReinterpret src));
455 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
456 ins_cost(3 * SVE_COST);
457 format %{ "reinterpretResize $dst, $src\t# vector (sve)" %}
458 ins_encode %{
459 uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
460 uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
461 uint length_in_bytes_resize = length_in_bytes_src < length_in_bytes_dst ?
462 length_in_bytes_src : length_in_bytes_dst;
463 assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
464 "invalid vector length");
465 __ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ B, length_in_bytes_resize);
466 __ sve_dup(as_FloatRegister($dst$$reg), __ B, 0);
467 __ sve_sel(as_FloatRegister($dst$$reg), __ B, as_PRegister($pgtmp$$reg),
468 as_FloatRegister($src$$reg), as_FloatRegister($dst$$reg));
469 %}
470 ins_pipe(pipe_slow);
471 %}
472
473 // vector mask reinterpret
474
475 instruct vmask_reinterpret_same_esize(pRegGov dst_src) %{
476 predicate(UseSVE > 0 &&
477 n->as_Vector()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
478 n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
479 match(Set dst_src (VectorReinterpret dst_src));
480 ins_cost(0);
481 format %{ "# vmask_reinterpret $dst_src\t# do nothing" %}
482 ins_encode %{
483 // empty
484 %}
485 ins_pipe(pipe_class_empty);
486 %}
487
488 instruct vmask_reinterpret_diff_esize(pRegGov dst, pRegGov src, vReg tmp, rFlagsReg cr) %{
489 predicate(UseSVE > 0 &&
490 n->as_Vector()->length() != n->in(1)->bottom_type()->is_vect()->length() &&
491 n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
492 match(Set dst (VectorReinterpret src));
493 effect(TEMP tmp, KILL cr);
494 ins_cost(2 * SVE_COST);
495 format %{ "# vmask_reinterpret $dst, $src\t# vector (sve)" %}
496 ins_encode %{
497 BasicType from_bt = Matcher::vector_element_basic_type(this, $src);
498 Assembler::SIMD_RegVariant from_size = __ elemType_to_regVariant(from_bt);
499 BasicType to_bt = Matcher::vector_element_basic_type(this);
500 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
501 __ sve_cpy(as_FloatRegister($tmp$$reg), from_size, as_PRegister($src$$reg), -1, false);
502 __ sve_cmp(Assembler::EQ, as_PRegister($dst$$reg), to_size, ptrue, as_FloatRegister($tmp$$reg), -1);
503 %}
504 ins_pipe(pipe_slow);
505 %}
506
507 // vector abs
508
509 instruct vabsB(vReg dst, vReg src) %{
510 predicate(UseSVE > 0 &&
511 !n->as_Vector()->is_predicated_vector());
512 match(Set dst (AbsVB src));
513 ins_cost(SVE_COST);
514 format %{ "sve_abs $dst, $src\t# vector (sve) (B)" %}
515 ins_encode %{
516 __ sve_abs(as_FloatRegister($dst$$reg), __ B,
517 ptrue, as_FloatRegister($src$$reg));
518 %}
519 ins_pipe(pipe_slow);
520 %}
521
522 instruct vabsS(vReg dst, vReg src) %{
523 predicate(UseSVE > 0 &&
524 !n->as_Vector()->is_predicated_vector());
525 match(Set dst (AbsVS src));
526 ins_cost(SVE_COST);
527 format %{ "sve_abs $dst, $src\t# vector (sve) (H)" %}
528 ins_encode %{
529 __ sve_abs(as_FloatRegister($dst$$reg), __ H,
530 ptrue, as_FloatRegister($src$$reg));
531 %}
532 ins_pipe(pipe_slow);
533 %}
534
535 instruct vabsI(vReg dst, vReg src) %{
536 predicate(UseSVE > 0 &&
537 !n->as_Vector()->is_predicated_vector());
538 match(Set dst (AbsVI src));
539 ins_cost(SVE_COST);
540 format %{ "sve_abs $dst, $src\t# vector (sve) (S)" %}
541 ins_encode %{
542 __ sve_abs(as_FloatRegister($dst$$reg), __ S,
543 ptrue, as_FloatRegister($src$$reg));
544 %}
545 ins_pipe(pipe_slow);
546 %}
547
548 instruct vabsL(vReg dst, vReg src) %{
549 predicate(UseSVE > 0 &&
550 !n->as_Vector()->is_predicated_vector());
551 match(Set dst (AbsVL src));
552 ins_cost(SVE_COST);
553 format %{ "sve_abs $dst, $src\t# vector (sve) (D)" %}
554 ins_encode %{
555 __ sve_abs(as_FloatRegister($dst$$reg), __ D,
556 ptrue, as_FloatRegister($src$$reg));
557 %}
558 ins_pipe(pipe_slow);
559 %}
560
561 instruct vabsF(vReg dst, vReg src) %{
562 predicate(UseSVE > 0 &&
563 !n->as_Vector()->is_predicated_vector());
564 match(Set dst (AbsVF src));
565 ins_cost(SVE_COST);
566 format %{ "sve_fabs $dst, $src\t# vector (sve) (S)" %}
567 ins_encode %{
568 __ sve_fabs(as_FloatRegister($dst$$reg), __ S,
569 ptrue, as_FloatRegister($src$$reg));
570 %}
571 ins_pipe(pipe_slow);
572 %}
573
574 instruct vabsD(vReg dst, vReg src) %{
575 predicate(UseSVE > 0 &&
576 !n->as_Vector()->is_predicated_vector());
577 match(Set dst (AbsVD src));
578 ins_cost(SVE_COST);
579 format %{ "sve_fabs $dst, $src\t# vector (sve) (D)" %}
580 ins_encode %{
581 __ sve_fabs(as_FloatRegister($dst$$reg), __ D,
582 ptrue, as_FloatRegister($src$$reg));
583 %}
584 ins_pipe(pipe_slow);
585 %}
586
587 // vector abs - predicated
588
589 instruct vabsB_masked(vReg dst_src, pRegGov pg) %{
590 predicate(UseSVE > 0);
591 match(Set dst_src (AbsVB dst_src pg));
592 ins_cost(SVE_COST);
593 format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (B)" %}
594 ins_encode %{
595 __ sve_abs(as_FloatRegister($dst_src$$reg), __ B,
596 as_PRegister($pg$$reg),
597 as_FloatRegister($dst_src$$reg));
598 %}
599 ins_pipe(pipe_slow);
600 %}
601
602 instruct vabsS_masked(vReg dst_src, pRegGov pg) %{
603 predicate(UseSVE > 0);
604 match(Set dst_src (AbsVS dst_src pg));
605 ins_cost(SVE_COST);
606 format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (H)" %}
607 ins_encode %{
608 __ sve_abs(as_FloatRegister($dst_src$$reg), __ H,
609 as_PRegister($pg$$reg),
610 as_FloatRegister($dst_src$$reg));
611 %}
612 ins_pipe(pipe_slow);
613 %}
614
615 instruct vabsI_masked(vReg dst_src, pRegGov pg) %{
616 predicate(UseSVE > 0);
617 match(Set dst_src (AbsVI dst_src pg));
618 ins_cost(SVE_COST);
619 format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
620 ins_encode %{
621 __ sve_abs(as_FloatRegister($dst_src$$reg), __ S,
622 as_PRegister($pg$$reg),
623 as_FloatRegister($dst_src$$reg));
624 %}
625 ins_pipe(pipe_slow);
626 %}
627
628 instruct vabsL_masked(vReg dst_src, pRegGov pg) %{
629 predicate(UseSVE > 0);
630 match(Set dst_src (AbsVL dst_src pg));
631 ins_cost(SVE_COST);
632 format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
633 ins_encode %{
634 __ sve_abs(as_FloatRegister($dst_src$$reg), __ D,
635 as_PRegister($pg$$reg),
636 as_FloatRegister($dst_src$$reg));
637 %}
638 ins_pipe(pipe_slow);
639 %}
640
641 instruct vabsF_masked(vReg dst_src, pRegGov pg) %{
642 predicate(UseSVE > 0);
643 match(Set dst_src (AbsVF dst_src pg));
644 ins_cost(SVE_COST);
645 format %{ "sve_fabs $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
646 ins_encode %{
647 __ sve_fabs(as_FloatRegister($dst_src$$reg), __ S,
648 as_PRegister($pg$$reg),
649 as_FloatRegister($dst_src$$reg));
650 %}
651 ins_pipe(pipe_slow);
652 %}
653
654 instruct vabsD_masked(vReg dst_src, pRegGov pg) %{
655 predicate(UseSVE > 0);
656 match(Set dst_src (AbsVD dst_src pg));
657 ins_cost(SVE_COST);
658 format %{ "sve_fabs $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
659 ins_encode %{
660 __ sve_fabs(as_FloatRegister($dst_src$$reg), __ D,
661 as_PRegister($pg$$reg),
662 as_FloatRegister($dst_src$$reg));
663 %}
664 ins_pipe(pipe_slow);
665 %}
666
667 // vector add
668
669 instruct vaddB(vReg dst, vReg src1, vReg src2) %{
670 predicate(UseSVE > 0);
671 match(Set dst (AddVB src1 src2));
672 ins_cost(SVE_COST);
673 format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (B)" %}
674 ins_encode %{
675 __ sve_add(as_FloatRegister($dst$$reg), __ B,
676 as_FloatRegister($src1$$reg),
677 as_FloatRegister($src2$$reg));
678 %}
679 ins_pipe(pipe_slow);
680 %}
681
682 instruct vaddS(vReg dst, vReg src1, vReg src2) %{
683 predicate(UseSVE > 0);
684 match(Set dst (AddVS src1 src2));
685 ins_cost(SVE_COST);
686 format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (H)" %}
687 ins_encode %{
688 __ sve_add(as_FloatRegister($dst$$reg), __ H,
689 as_FloatRegister($src1$$reg),
690 as_FloatRegister($src2$$reg));
691 %}
692 ins_pipe(pipe_slow);
693 %}
694
695 instruct vaddI(vReg dst, vReg src1, vReg src2) %{
696 predicate(UseSVE > 0);
697 match(Set dst (AddVI src1 src2));
698 ins_cost(SVE_COST);
699 format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (S)" %}
700 ins_encode %{
701 __ sve_add(as_FloatRegister($dst$$reg), __ S,
702 as_FloatRegister($src1$$reg),
703 as_FloatRegister($src2$$reg));
704 %}
705 ins_pipe(pipe_slow);
706 %}
707
708 instruct vaddL(vReg dst, vReg src1, vReg src2) %{
709 predicate(UseSVE > 0);
710 match(Set dst (AddVL src1 src2));
711 ins_cost(SVE_COST);
712 format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (D)" %}
713 ins_encode %{
714 __ sve_add(as_FloatRegister($dst$$reg), __ D,
715 as_FloatRegister($src1$$reg),
716 as_FloatRegister($src2$$reg));
717 %}
718 ins_pipe(pipe_slow);
719 %}
720
721 instruct vaddF(vReg dst, vReg src1, vReg src2) %{
722 predicate(UseSVE > 0);
723 match(Set dst (AddVF src1 src2));
724 ins_cost(SVE_COST);
725 format %{ "sve_fadd $dst, $src1, $src2\t # vector (sve) (S)" %}
726 ins_encode %{
727 __ sve_fadd(as_FloatRegister($dst$$reg), __ S,
728 as_FloatRegister($src1$$reg),
729 as_FloatRegister($src2$$reg));
730 %}
731 ins_pipe(pipe_slow);
732 %}
733
734 instruct vaddD(vReg dst, vReg src1, vReg src2) %{
735 predicate(UseSVE > 0);
736 match(Set dst (AddVD src1 src2));
737 ins_cost(SVE_COST);
738 format %{ "sve_fadd $dst, $src1, $src2\t # vector (sve) (D)" %}
739 ins_encode %{
740 __ sve_fadd(as_FloatRegister($dst$$reg), __ D,
741 as_FloatRegister($src1$$reg),
742 as_FloatRegister($src2$$reg));
743 %}
744 ins_pipe(pipe_slow);
745 %}
746
747 // vector add - predicated
748
749 instruct vaddB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
750 predicate(UseSVE > 0);
751 match(Set dst_src1 (AddVB (Binary dst_src1 src2) pg));
752 ins_cost(SVE_COST);
753 format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
754 ins_encode %{
755 __ sve_add(as_FloatRegister($dst_src1$$reg), __ B,
756 as_PRegister($pg$$reg),
757 as_FloatRegister($src2$$reg));
758 %}
759 ins_pipe(pipe_slow);
760 %}
761
762 instruct vaddS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
763 predicate(UseSVE > 0);
764 match(Set dst_src1 (AddVS (Binary dst_src1 src2) pg));
765 ins_cost(SVE_COST);
766 format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
767 ins_encode %{
768 __ sve_add(as_FloatRegister($dst_src1$$reg), __ H,
769 as_PRegister($pg$$reg),
770 as_FloatRegister($src2$$reg));
771 %}
772 ins_pipe(pipe_slow);
773 %}
774
775 instruct vaddI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
776 predicate(UseSVE > 0);
777 match(Set dst_src1 (AddVI (Binary dst_src1 src2) pg));
778 ins_cost(SVE_COST);
779 format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
780 ins_encode %{
781 __ sve_add(as_FloatRegister($dst_src1$$reg), __ S,
782 as_PRegister($pg$$reg),
783 as_FloatRegister($src2$$reg));
784 %}
785 ins_pipe(pipe_slow);
786 %}
787
788 instruct vaddL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
789 predicate(UseSVE > 0);
790 match(Set dst_src1 (AddVL (Binary dst_src1 src2) pg));
791 ins_cost(SVE_COST);
792 format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
793 ins_encode %{
794 __ sve_add(as_FloatRegister($dst_src1$$reg), __ D,
795 as_PRegister($pg$$reg),
796 as_FloatRegister($src2$$reg));
797 %}
798 ins_pipe(pipe_slow);
799 %}
800
801 instruct vaddF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
802 predicate(UseSVE > 0);
803 match(Set dst_src1 (AddVF (Binary dst_src1 src2) pg));
804 ins_cost(SVE_COST);
805 format %{ "sve_fadd $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
806 ins_encode %{
807 __ sve_fadd(as_FloatRegister($dst_src1$$reg), __ S,
808 as_PRegister($pg$$reg),
809 as_FloatRegister($src2$$reg));
810 %}
811 ins_pipe(pipe_slow);
812 %}
813
814 instruct vaddD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
815 predicate(UseSVE > 0);
816 match(Set dst_src1 (AddVD (Binary dst_src1 src2) pg));
817 ins_cost(SVE_COST);
818 format %{ "sve_fadd $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
819 ins_encode %{
820 __ sve_fadd(as_FloatRegister($dst_src1$$reg), __ D,
821 as_PRegister($pg$$reg),
822 as_FloatRegister($src2$$reg));
823 %}
824 ins_pipe(pipe_slow);
825 %}
826
827 // vector add reg imm (unpredicated)
828
829 instruct vaddImmB(vReg dst_src, immBAddSubV con) %{
830 predicate(UseSVE > 0);
831 match(Set dst_src (AddVB dst_src (ReplicateB con)));
832 ins_cost(SVE_COST);
833 format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) (B)" %}
834 ins_encode %{
835 int32_t val = $con$$constant;
836 if (val > 0){
837 __ sve_add(as_FloatRegister($dst_src$$reg), __ B, val);
838 } else if (val < 0){
839 __ sve_sub(as_FloatRegister($dst_src$$reg), __ B, -val);
840 }
841 %}
842 ins_pipe(pipe_slow);
843 %}
844
845 instruct vaddImmS(vReg dst_src, immIAddSubV con) %{
846 predicate(UseSVE > 0);
847 match(Set dst_src (AddVS dst_src (ReplicateS con)));
848 ins_cost(SVE_COST);
849 format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) (H)" %}
850 ins_encode %{
851 int32_t val = $con$$constant;
852 if (val > 0){
853 __ sve_add(as_FloatRegister($dst_src$$reg), __ H, val);
854 } else if (val < 0){
855 __ sve_sub(as_FloatRegister($dst_src$$reg), __ H, -val);
856 }
857 %}
858 ins_pipe(pipe_slow);
859 %}
860
861 instruct vaddImmI(vReg dst_src, immIAddSubV con) %{
862 predicate(UseSVE > 0);
863 match(Set dst_src (AddVI dst_src (ReplicateI con)));
864 ins_cost(SVE_COST);
865 format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) (S)" %}
866 ins_encode %{
867 int32_t val = $con$$constant;
868 if (val > 0){
869 __ sve_add(as_FloatRegister($dst_src$$reg), __ S, val);
870 } else if (val < 0){
871 __ sve_sub(as_FloatRegister($dst_src$$reg), __ S, -val);
872 }
873 %}
874 ins_pipe(pipe_slow);
875 %}
876
877 instruct vaddImmL(vReg dst_src, immLAddSubV con) %{
878 predicate(UseSVE > 0);
879 match(Set dst_src (AddVL dst_src (ReplicateL con)));
880 ins_cost(SVE_COST);
881 format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) (D)" %}
882 ins_encode %{
883 int32_t val = $con$$constant;
884 if (val > 0){
885 __ sve_add(as_FloatRegister($dst_src$$reg), __ D, val);
886 } else if (val < 0){
887 __ sve_sub(as_FloatRegister($dst_src$$reg), __ D, -val);
888 }
889 %}
890 ins_pipe(pipe_slow);
891 %}
892
893 // vector binary op reg imm (unpredicated)
894
895 instruct vandB(vReg dst_src, immBLog con) %{
896 predicate(UseSVE > 0);
897 match(Set dst_src (AndV dst_src (ReplicateB con)));
898 ins_cost(SVE_COST);
899 format %{ "sve_and $dst_src, $dst_src, $con\t # vector (sve) (B)" %}
900 ins_encode %{
901 __ sve_and(as_FloatRegister($dst_src$$reg), __ B,
902 (uint64_t)($con$$constant));
903 %}
904 ins_pipe(pipe_slow);
905 %}
906
907 instruct vandH(vReg dst_src, immSLog con) %{
908 predicate(UseSVE > 0);
909 match(Set dst_src (AndV dst_src (ReplicateS con)));
910 ins_cost(SVE_COST);
911 format %{ "sve_and $dst_src, $dst_src, $con\t # vector (sve) (H)" %}
912 ins_encode %{
913 __ sve_and(as_FloatRegister($dst_src$$reg), __ H,
914 (uint64_t)($con$$constant));
915 %}
916 ins_pipe(pipe_slow);
917 %}
918
919 instruct vandS(vReg dst_src, immILog con) %{
920 predicate(UseSVE > 0);
921 match(Set dst_src (AndV dst_src (ReplicateI con)));
922 ins_cost(SVE_COST);
923 format %{ "sve_and $dst_src, $dst_src, $con\t # vector (sve) (S)" %}
924 ins_encode %{
925 __ sve_and(as_FloatRegister($dst_src$$reg), __ S,
926 (uint64_t)($con$$constant));
927 %}
928 ins_pipe(pipe_slow);
929 %}
930
931 instruct vandD(vReg dst_src, immLLog con) %{
932 predicate(UseSVE > 0);
933 match(Set dst_src (AndV dst_src (ReplicateL con)));
934 ins_cost(SVE_COST);
935 format %{ "sve_and $dst_src, $dst_src, $con\t # vector (sve) (D)" %}
936 ins_encode %{
937 __ sve_and(as_FloatRegister($dst_src$$reg), __ D,
938 (uint64_t)($con$$constant));
939 %}
940 ins_pipe(pipe_slow);
941 %}
942
943 instruct vorB(vReg dst_src, immBLog con) %{
944 predicate(UseSVE > 0);
945 match(Set dst_src (OrV dst_src (ReplicateB con)));
946 ins_cost(SVE_COST);
947 format %{ "sve_orr $dst_src, $dst_src, $con\t # vector (sve) (B)" %}
948 ins_encode %{
949 __ sve_orr(as_FloatRegister($dst_src$$reg), __ B,
950 (uint64_t)($con$$constant));
951 %}
952 ins_pipe(pipe_slow);
953 %}
954
955 instruct vorH(vReg dst_src, immSLog con) %{
956 predicate(UseSVE > 0);
957 match(Set dst_src (OrV dst_src (ReplicateS con)));
958 ins_cost(SVE_COST);
959 format %{ "sve_orr $dst_src, $dst_src, $con\t # vector (sve) (H)" %}
960 ins_encode %{
961 __ sve_orr(as_FloatRegister($dst_src$$reg), __ H,
962 (uint64_t)($con$$constant));
963 %}
964 ins_pipe(pipe_slow);
965 %}
966
967 instruct vorS(vReg dst_src, immILog con) %{
968 predicate(UseSVE > 0);
969 match(Set dst_src (OrV dst_src (ReplicateI con)));
970 ins_cost(SVE_COST);
971 format %{ "sve_orr $dst_src, $dst_src, $con\t # vector (sve) (S)" %}
972 ins_encode %{
973 __ sve_orr(as_FloatRegister($dst_src$$reg), __ S,
974 (uint64_t)($con$$constant));
975 %}
976 ins_pipe(pipe_slow);
977 %}
978
979 instruct vorD(vReg dst_src, immLLog con) %{
980 predicate(UseSVE > 0);
981 match(Set dst_src (OrV dst_src (ReplicateL con)));
982 ins_cost(SVE_COST);
983 format %{ "sve_orr $dst_src, $dst_src, $con\t # vector (sve) (D)" %}
984 ins_encode %{
985 __ sve_orr(as_FloatRegister($dst_src$$reg), __ D,
986 (uint64_t)($con$$constant));
987 %}
988 ins_pipe(pipe_slow);
989 %}
990
991 instruct vxorB(vReg dst_src, immBLog con) %{
992 predicate(UseSVE > 0);
993 match(Set dst_src (XorV dst_src (ReplicateB con)));
994 ins_cost(SVE_COST);
995 format %{ "sve_eor $dst_src, $dst_src, $con\t # vector (sve) (B)" %}
996 ins_encode %{
997 __ sve_eor(as_FloatRegister($dst_src$$reg), __ B,
998 (uint64_t)($con$$constant));
999 %}
1000 ins_pipe(pipe_slow);
1001 %}
1002
1003 instruct vxorH(vReg dst_src, immSLog con) %{
1004 predicate(UseSVE > 0);
1005 match(Set dst_src (XorV dst_src (ReplicateS con)));
1006 ins_cost(SVE_COST);
1007 format %{ "sve_eor $dst_src, $dst_src, $con\t # vector (sve) (H)" %}
1008 ins_encode %{
1009 __ sve_eor(as_FloatRegister($dst_src$$reg), __ H,
1010 (uint64_t)($con$$constant));
1011 %}
1012 ins_pipe(pipe_slow);
1013 %}
1014
1015 instruct vxorS(vReg dst_src, immILog con) %{
1016 predicate(UseSVE > 0);
1017 match(Set dst_src (XorV dst_src (ReplicateI con)));
1018 ins_cost(SVE_COST);
1019 format %{ "sve_eor $dst_src, $dst_src, $con\t # vector (sve) (S)" %}
1020 ins_encode %{
1021 __ sve_eor(as_FloatRegister($dst_src$$reg), __ S,
1022 (uint64_t)($con$$constant));
1023 %}
1024 ins_pipe(pipe_slow);
1025 %}
1026
1027 instruct vxorD(vReg dst_src, immLLog con) %{
1028 predicate(UseSVE > 0);
1029 match(Set dst_src (XorV dst_src (ReplicateL con)));
1030 ins_cost(SVE_COST);
1031 format %{ "sve_eor $dst_src, $dst_src, $con\t # vector (sve) (D)" %}
1032 ins_encode %{
1033 __ sve_eor(as_FloatRegister($dst_src$$reg), __ D,
1034 (uint64_t)($con$$constant));
1035 %}
1036 ins_pipe(pipe_slow);
1037 %}
1038 // vector and
1039
1040 instruct vand(vReg dst, vReg src1, vReg src2) %{
1041 predicate(UseSVE > 0);
1042 match(Set dst (AndV src1 src2));
1043 ins_cost(SVE_COST);
1044 format %{ "sve_and $dst, $src1, $src2\t# vector (sve)" %}
1045 ins_encode %{
1046 __ sve_and(as_FloatRegister($dst$$reg),
1047 as_FloatRegister($src1$$reg),
1048 as_FloatRegister($src2$$reg));
1049 %}
1050 ins_pipe(pipe_slow);
1051 %}
1052
1053 // vector or
1054
1055 instruct vor(vReg dst, vReg src1, vReg src2) %{
1056 predicate(UseSVE > 0);
1057 match(Set dst (OrV src1 src2));
1058 ins_cost(SVE_COST);
1059 format %{ "sve_orr $dst, $src1, $src2\t# vector (sve)" %}
1060 ins_encode %{
1061 __ sve_orr(as_FloatRegister($dst$$reg),
1062 as_FloatRegister($src1$$reg),
1063 as_FloatRegister($src2$$reg));
1064 %}
1065 ins_pipe(pipe_slow);
1066 %}
1067
1068 // vector xor
1069
1070 instruct vxor(vReg dst, vReg src1, vReg src2) %{
1071 predicate(UseSVE > 0);
1072 match(Set dst (XorV src1 src2));
1073 ins_cost(SVE_COST);
1074 format %{ "sve_eor $dst, $src1, $src2\t# vector (sve)" %}
1075 ins_encode %{
1076 __ sve_eor(as_FloatRegister($dst$$reg),
1077 as_FloatRegister($src1$$reg),
1078 as_FloatRegister($src2$$reg));
1079 %}
1080 ins_pipe(pipe_slow);
1081 %}
1082
1083 // vector and - predicated
1084
1085 instruct vand_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1086 predicate(UseSVE > 0);
1087 match(Set dst_src1 (AndV (Binary dst_src1 src2) pg));
1088 ins_cost(SVE_COST);
1089 format %{ "sve_and $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
1090 ins_encode %{
1091 BasicType bt = Matcher::vector_element_basic_type(this);
1092 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1093 __ sve_and(as_FloatRegister($dst_src1$$reg), size,
1094 as_PRegister($pg$$reg),
1095 as_FloatRegister($src2$$reg));
1096 %}
1097 ins_pipe(pipe_slow);
1098 %}
1099
1100 // vector or - predicated
1101
1102 instruct vor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1103 predicate(UseSVE > 0);
1104 match(Set dst_src1 (OrV (Binary dst_src1 src2) pg));
1105 ins_cost(SVE_COST);
1106 format %{ "sve_orr $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
1107 ins_encode %{
1108 BasicType bt = Matcher::vector_element_basic_type(this);
1109 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1110 __ sve_orr(as_FloatRegister($dst_src1$$reg), size,
1111 as_PRegister($pg$$reg),
1112 as_FloatRegister($src2$$reg));
1113 %}
1114 ins_pipe(pipe_slow);
1115 %}
1116
1117 // vector xor - predicated
1118
1119 instruct vxor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1120 predicate(UseSVE > 0);
1121 match(Set dst_src1 (XorV (Binary dst_src1 src2) pg));
1122 ins_cost(SVE_COST);
1123 format %{ "sve_eor $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
1124 ins_encode %{
1125 BasicType bt = Matcher::vector_element_basic_type(this);
1126 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1127 __ sve_eor(as_FloatRegister($dst_src1$$reg), size,
1128 as_PRegister($pg$$reg),
1129 as_FloatRegister($src2$$reg));
1130 %}
1131 ins_pipe(pipe_slow);
1132 %}
1133
1134 // vector not
1135
1136 instruct vnotI(vReg dst, vReg src, immI_M1 m1) %{
1137 predicate(UseSVE > 0);
1138 match(Set dst (XorV src (ReplicateB m1)));
1139 match(Set dst (XorV src (ReplicateS m1)));
1140 match(Set dst (XorV src (ReplicateI m1)));
1141 ins_cost(SVE_COST);
1142 format %{ "sve_not $dst, $src\t# vector (sve) B/H/S" %}
1143 ins_encode %{
1144 __ sve_not(as_FloatRegister($dst$$reg), __ D,
1145 ptrue, as_FloatRegister($src$$reg));
1146 %}
1147 ins_pipe(pipe_slow);
1148 %}
1149
1150 instruct vnotL(vReg dst, vReg src, immL_M1 m1) %{
1151 predicate(UseSVE > 0);
1152 match(Set dst (XorV src (ReplicateL m1)));
1153 ins_cost(SVE_COST);
1154 format %{ "sve_not $dst, $src\t# vector (sve) D" %}
1155 ins_encode %{
1156 __ sve_not(as_FloatRegister($dst$$reg), __ D,
1157 ptrue, as_FloatRegister($src$$reg));
1158 %}
1159 ins_pipe(pipe_slow);
1160 %}
1161
1162 // vector not - predicated
1163
1164 instruct vnotI_masked(vReg dst, vReg src, immI_M1 m1, pRegGov pg) %{
1165 predicate(UseSVE > 0);
1166 match(Set dst (XorV (Binary src (ReplicateB m1)) pg));
1167 match(Set dst (XorV (Binary src (ReplicateS m1)) pg));
1168 match(Set dst (XorV (Binary src (ReplicateI m1)) pg));
1169 ins_cost(SVE_COST);
1170 format %{ "sve_not $dst, $pg, $src\t# vector (sve) B/H/S" %}
1171 ins_encode %{
1172 BasicType bt = Matcher::vector_element_basic_type(this);
1173 __ sve_not(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
1174 as_PRegister($pg$$reg), as_FloatRegister($src$$reg));
1175 %}
1176 ins_pipe(pipe_slow);
1177 %}
1178
1179 instruct vnotL_masked(vReg dst, vReg src, immL_M1 m1, pRegGov pg) %{
1180 predicate(UseSVE > 0);
1181 match(Set dst (XorV (Binary src (ReplicateL m1)) pg));
1182 ins_cost(SVE_COST);
1183 format %{ "sve_not $dst, $pg, $src\t# vector (sve) D" %}
1184 ins_encode %{
1185 BasicType bt = Matcher::vector_element_basic_type(this);
1186 __ sve_not(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
1187 as_PRegister($pg$$reg), as_FloatRegister($src$$reg));
1188 %}
1189 ins_pipe(pipe_slow);
1190 %}
1191
1192 // vector and_not
1193
1194 instruct vand_notI(vReg dst, vReg src1, vReg src2, immI_M1 m1) %{
1195 predicate(UseSVE > 0);
1196 match(Set dst (AndV src1 (XorV src2 (ReplicateB m1))));
1197 match(Set dst (AndV src1 (XorV src2 (ReplicateS m1))));
1198 match(Set dst (AndV src1 (XorV src2 (ReplicateI m1))));
1199 ins_cost(SVE_COST);
1200 format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) B/H/S" %}
1201 ins_encode %{
1202 __ sve_bic(as_FloatRegister($dst$$reg),
1203 as_FloatRegister($src1$$reg),
1204 as_FloatRegister($src2$$reg));
1205 %}
1206 ins_pipe(pipe_slow);
1207 %}
1208
1209 instruct vand_notL(vReg dst, vReg src1, vReg src2, immL_M1 m1) %{
1210 predicate(UseSVE > 0);
1211 match(Set dst (AndV src1 (XorV src2 (ReplicateL m1))));
1212 ins_cost(SVE_COST);
1213 format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) D" %}
1214 ins_encode %{
1215 __ sve_bic(as_FloatRegister($dst$$reg),
1216 as_FloatRegister($src1$$reg),
1217 as_FloatRegister($src2$$reg));
1218 %}
1219 ins_pipe(pipe_slow);
1220 %}
1221
1222 // vector and_not - predicated
1223
1224 instruct vand_notI_masked(vReg dst_src1, vReg src2, immI_M1 m1, pRegGov pg) %{
1225 predicate(UseSVE > 0);
1226 match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateB m1))) pg));
1227 match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateS m1))) pg));
1228 match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateI m1))) pg));
1229 ins_cost(SVE_COST);
1230 format %{ "sve_bic $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) B/H/S" %}
1231 ins_encode %{
1232 BasicType bt = Matcher::vector_element_basic_type(this);
1233 __ sve_bic(as_FloatRegister($dst_src1$$reg), __ elemType_to_regVariant(bt),
1234 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1235 %}
1236 ins_pipe(pipe_slow);
1237 %}
1238
1239 instruct vand_notL_masked(vReg dst_src1, vReg src2, immL_M1 m1, pRegGov pg) %{
1240 predicate(UseSVE > 0);
1241 match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateL m1))) pg));
1242 ins_cost(SVE_COST);
1243 format %{ "sve_bic $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) D" %}
1244 ins_encode %{
1245 BasicType bt = Matcher::vector_element_basic_type(this);
1246 __ sve_bic(as_FloatRegister($dst_src1$$reg), __ elemType_to_regVariant(bt),
1247 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1248 %}
1249 ins_pipe(pipe_slow);
1250 %}
1251
1252 // vector float div
1253
1254 instruct vdivF(vReg dst_src1, vReg src2) %{
1255 predicate(UseSVE > 0);
1256 match(Set dst_src1 (DivVF dst_src1 src2));
1257 ins_cost(SVE_COST);
1258 format %{ "sve_fdiv $dst_src1, $dst_src1, $src2\t# vector (sve) (S)" %}
1259 ins_encode %{
1260 __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ S,
1261 ptrue, as_FloatRegister($src2$$reg));
1262 %}
1263 ins_pipe(pipe_slow);
1264 %}
1265
1266 instruct vdivD(vReg dst_src1, vReg src2) %{
1267 predicate(UseSVE > 0);
1268 match(Set dst_src1 (DivVD dst_src1 src2));
1269 ins_cost(SVE_COST);
1270 format %{ "sve_fdiv $dst_src1, $dst_src1, $src2\t# vector (sve) (D)" %}
1271 ins_encode %{
1272 __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ D,
1273 ptrue, as_FloatRegister($src2$$reg));
1274 %}
1275 ins_pipe(pipe_slow);
1276 %}
1277
1278 // vector float div - predicated
1279
1280 instruct vfdivF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1281 predicate(UseSVE > 0);
1282 match(Set dst_src1 (DivVF (Binary dst_src1 src2) pg));
1283 ins_cost(SVE_COST);
1284 format %{ "sve_fdiv $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
1285 ins_encode %{
1286 __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ S,
1287 as_PRegister($pg$$reg),
1288 as_FloatRegister($src2$$reg));
1289 %}
1290 ins_pipe(pipe_slow);
1291 %}
1292
1293 instruct vfdivD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1294 predicate(UseSVE > 0);
1295 match(Set dst_src1 (DivVD (Binary dst_src1 src2) pg));
1296 ins_cost(SVE_COST);
1297 format %{ "sve_fdiv $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
1298 ins_encode %{
1299 __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ D,
1300 as_PRegister($pg$$reg),
1301 as_FloatRegister($src2$$reg));
1302 %}
1303 ins_pipe(pipe_slow);
1304 %}
1305
1306 // vector min/max
1307
1308 instruct vmin(vReg dst_src1, vReg src2) %{
1309 predicate(UseSVE > 0);
1310 match(Set dst_src1 (MinV dst_src1 src2));
1311 ins_cost(SVE_COST);
1312 format %{ "sve_min $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
1313 ins_encode %{
1314 BasicType bt = Matcher::vector_element_basic_type(this);
1315 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1316 if (is_floating_point_type(bt)) {
1317 __ sve_fmin(as_FloatRegister($dst_src1$$reg), size,
1318 ptrue, as_FloatRegister($src2$$reg));
1319 } else {
1320 assert(is_integral_type(bt), "unsupported type");
1321 __ sve_smin(as_FloatRegister($dst_src1$$reg), size,
1322 ptrue, as_FloatRegister($src2$$reg));
1323 }
1324 %}
1325 ins_pipe(pipe_slow);
1326 %}
1327
1328 instruct vmax(vReg dst_src1, vReg src2) %{
1329 predicate(UseSVE > 0);
1330 match(Set dst_src1 (MaxV dst_src1 src2));
1331 ins_cost(SVE_COST);
1332 format %{ "sve_max $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
1333 ins_encode %{
1334 BasicType bt = Matcher::vector_element_basic_type(this);
1335 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1336 if (is_floating_point_type(bt)) {
1337 __ sve_fmax(as_FloatRegister($dst_src1$$reg), size,
1338 ptrue, as_FloatRegister($src2$$reg));
1339 } else {
1340 assert(is_integral_type(bt), "unsupported type");
1341 __ sve_smax(as_FloatRegister($dst_src1$$reg), size,
1342 ptrue, as_FloatRegister($src2$$reg));
1343 }
1344 %}
1345 ins_pipe(pipe_slow);
1346 %}
1347
1348 // vector min/max - predicated
1349
1350 instruct vmin_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1351 predicate(UseSVE > 0);
1352 match(Set dst_src1 (MinV (Binary dst_src1 src2) pg));
1353 ins_cost(SVE_COST);
1354 format %{ "sve_min $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %}
1355 ins_encode %{
1356 BasicType bt = Matcher::vector_element_basic_type(this);
1357 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1358 if (is_floating_point_type(bt)) {
1359 __ sve_fmin(as_FloatRegister($dst_src1$$reg), size,
1360 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1361 } else {
1362 assert(is_integral_type(bt), "unsupported type");
1363 __ sve_smin(as_FloatRegister($dst_src1$$reg), size,
1364 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1365 }
1366 %}
1367 ins_pipe(pipe_slow);
1368 %}
1369
1370 instruct vmax_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
1371 predicate(UseSVE > 0);
1372 match(Set dst_src1 (MaxV (Binary dst_src1 src2) pg));
1373 ins_cost(SVE_COST);
1374 format %{ "sve_max $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %}
1375 ins_encode %{
1376 BasicType bt = Matcher::vector_element_basic_type(this);
1377 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1378 if (is_floating_point_type(bt)) {
1379 __ sve_fmax(as_FloatRegister($dst_src1$$reg), size,
1380 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1381 } else {
1382 assert(is_integral_type(bt), "unsupported type");
1383 __ sve_smax(as_FloatRegister($dst_src1$$reg), size,
1384 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1385 }
1386 %}
1387 ins_pipe(pipe_slow);
1388 %}
1389
1390 // vector fmla
1391
1392 // dst_src1 = dst_src1 + src2 * src3
1393 instruct vfmlaF(vReg dst_src1, vReg src2, vReg src3) %{
1394 predicate(UseFMA && UseSVE > 0);
1395 match(Set dst_src1 (FmaVF dst_src1 (Binary src2 src3)));
1396 ins_cost(SVE_COST);
1397 format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1398 ins_encode %{
1399 __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ S,
1400 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1401 %}
1402 ins_pipe(pipe_slow);
1403 %}
1404
1405 // dst_src1 = dst_src1 + src2 * src3
1406 instruct vfmlaD(vReg dst_src1, vReg src2, vReg src3) %{
1407 predicate(UseFMA && UseSVE > 0);
1408 match(Set dst_src1 (FmaVD dst_src1 (Binary src2 src3)));
1409 ins_cost(SVE_COST);
1410 format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1411 ins_encode %{
1412 __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ D,
1413 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1414 %}
1415 ins_pipe(pipe_slow);
1416 %}
1417
1418 // vector fmad - predicated
1419
1420 // dst_src1 = dst_src1 * src2 + src3
1421 instruct vfmadF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1422 predicate(UseFMA && UseSVE > 0);
1423 match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary src3 pg)));
1424 ins_cost(SVE_COST);
1425 format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) (S)" %}
1426 ins_encode %{
1427 __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
1428 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1429 %}
1430 ins_pipe(pipe_slow);
1431 %}
1432
1433 // dst_src1 = dst_src1 * src2 + src3
1434 instruct vfmadD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1435 predicate(UseFMA && UseSVE > 0);
1436 match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary src3 pg)));
1437 ins_cost(SVE_COST);
1438 format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) (D)" %}
1439 ins_encode %{
1440 __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
1441 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1442 %}
1443 ins_pipe(pipe_slow);
1444 %}
1445
1446 // vector fmls
1447
1448 // dst_src1 = dst_src1 + -src2 * src3
1449 // The NegVF must not be predicated.
1450 instruct vfmlsF1(vReg dst_src1, vReg src2, vReg src3) %{
1451 predicate(UseFMA && UseSVE > 0 &&
1452 !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1453 match(Set dst_src1 (FmaVF dst_src1 (Binary (NegVF src2) src3)));
1454 ins_cost(SVE_COST);
1455 format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1456 ins_encode %{
1457 __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ S,
1458 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1459 %}
1460 ins_pipe(pipe_slow);
1461 %}
1462
1463 // dst_src1 = dst_src1 + src2 * -src3
1464 // The NegVF must not be predicated.
1465 instruct vfmlsF2(vReg dst_src1, vReg src2, vReg src3) %{
1466 predicate(UseFMA && UseSVE > 0 &&
1467 !n->in(2)->in(2)->as_Vector()->is_predicated_vector());
1468 match(Set dst_src1 (FmaVF dst_src1 (Binary src2 (NegVF src3))));
1469 ins_cost(SVE_COST);
1470 format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1471 ins_encode %{
1472 __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ S,
1473 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1474 %}
1475 ins_pipe(pipe_slow);
1476 %}
1477
1478 // dst_src1 = dst_src1 + -src2 * src3
1479 // The NegVD must not be predicated.
1480 instruct vfmlsD1(vReg dst_src1, vReg src2, vReg src3) %{
1481 predicate(UseFMA && UseSVE > 0 &&
1482 !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1483 match(Set dst_src1 (FmaVD dst_src1 (Binary (NegVD src2) src3)));
1484 ins_cost(SVE_COST);
1485 format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1486 ins_encode %{
1487 __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ D,
1488 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1489 %}
1490 ins_pipe(pipe_slow);
1491 %}
1492
1493 // dst_src1 = dst_src1 + src2 * -src3
1494 // The NegVD must not be predicated.
1495 instruct vfmlsD2(vReg dst_src1, vReg src2, vReg src3) %{
1496 predicate(UseFMA && UseSVE > 0 &&
1497 !n->in(2)->in(2)->as_Vector()->is_predicated_vector());
1498 match(Set dst_src1 (FmaVD dst_src1 (Binary src2 (NegVD src3))));
1499 ins_cost(SVE_COST);
1500 format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1501 ins_encode %{
1502 __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ D,
1503 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1504 %}
1505 ins_pipe(pipe_slow);
1506 %}
1507
1508 // vector fmsb - predicated
1509
1510 // dst_src1 = dst_src1 * -src2 + src3
1511 // The NegVF must not be predicated.
1512 instruct vfmsbF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1513 predicate(UseFMA && UseSVE > 0 &&
1514 !n->in(1)->in(2)->as_Vector()->is_predicated_vector());
1515 match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary src3 pg)));
1516 ins_cost(SVE_COST);
1517 format %{ "sve_fmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) (S)" %}
1518 ins_encode %{
1519 __ sve_fmsb(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
1520 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1521 %}
1522 ins_pipe(pipe_slow);
1523 %}
1524
1525 // dst_src1 = dst_src1 * -src2 + src3
1526 // The NegVD must not be predicated.
1527 instruct vfmsbD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1528 predicate(UseFMA && UseSVE > 0 &&
1529 !n->in(1)->in(2)->as_Vector()->is_predicated_vector());
1530 match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary src3 pg)));
1531 ins_cost(SVE_COST);
1532 format %{ "sve_fmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) (D)" %}
1533 ins_encode %{
1534 __ sve_fmsb(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
1535 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1536 %}
1537 ins_pipe(pipe_slow);
1538 %}
1539
1540 // vector fnmla
1541
1542 // dst_src1 = -dst_src1 + -src2 * src3
1543 // The NegVF must not be predicated.
1544 instruct vfnmlaF1(vReg dst_src1, vReg src2, vReg src3) %{
1545 predicate(UseFMA && UseSVE > 0 &&
1546 !n->in(1)->as_Vector()->is_predicated_vector() &&
1547 !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1548 match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary (NegVF src2) src3)));
1549 ins_cost(SVE_COST);
1550 format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1551 ins_encode %{
1552 __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ S,
1553 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1554 %}
1555 ins_pipe(pipe_slow);
1556 %}
1557
1558 // dst_src1 = -dst_src1 + src2 * -src3
1559 // The NegVF must not be predicated.
1560 instruct vfnmlaF2(vReg dst_src1, vReg src2, vReg src3) %{
1561 predicate(UseFMA && UseSVE > 0 &&
1562 !n->in(1)->as_Vector()->is_predicated_vector() &&
1563 !n->in(2)->in(2)->as_Vector()->is_predicated_vector());
1564 match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 (NegVF src3))));
1565 ins_cost(SVE_COST);
1566 format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1567 ins_encode %{
1568 __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ S,
1569 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1570 %}
1571 ins_pipe(pipe_slow);
1572 %}
1573
1574 // dst_src1 = -dst_src1 + -src2 * src3
1575 // The NegVD must not be predicated.
1576 instruct vfnmlaD1(vReg dst_src1, vReg src2, vReg src3) %{
1577 predicate(UseFMA && UseSVE > 0 &&
1578 !n->in(1)->as_Vector()->is_predicated_vector() &&
1579 !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1580 match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary (NegVD src2) src3)));
1581 ins_cost(SVE_COST);
1582 format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1583 ins_encode %{
1584 __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ D,
1585 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1586 %}
1587 ins_pipe(pipe_slow);
1588 %}
1589
1590 // dst_src1 = -dst_src1 + src2 * -src3
1591 // The NegVD must not be predicated.
1592 instruct vfnmlaD2(vReg dst_src1, vReg src2, vReg src3) %{
1593 predicate(UseFMA && UseSVE > 0 &&
1594 !n->in(1)->as_Vector()->is_predicated_vector() &&
1595 !n->in(2)->in(2)->as_Vector()->is_predicated_vector());
1596 match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 (NegVD src3))));
1597 ins_cost(SVE_COST);
1598 format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1599 ins_encode %{
1600 __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ D,
1601 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1602 %}
1603 ins_pipe(pipe_slow);
1604 %}
1605
1606 // vector fnmad - predicated
1607
1608 // dst_src1 = -src3 + dst_src1 * -src2
1609 // The NegVF must not be predicated.
1610 instruct vfnmadF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1611 predicate(UseFMA && UseSVE > 0 &&
1612 !n->in(1)->in(2)->as_Vector()->is_predicated_vector() &&
1613 !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1614 match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary (NegVF src3) pg)));
1615 ins_cost(SVE_COST);
1616 format %{ "sve_fnmad $dst_src1, $pg, $src2, $src3\t # vector (sve) (S)" %}
1617 ins_encode %{
1618 __ sve_fnmad(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
1619 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1620 %}
1621 ins_pipe(pipe_slow);
1622 %}
1623
1624 // dst_src1 = -src3 + dst_src1 * -src2
1625 // The NegVD must not be predicated.
1626 instruct vfnmadD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1627 predicate(UseFMA && UseSVE > 0 &&
1628 !n->in(1)->in(2)->as_Vector()->is_predicated_vector() &&
1629 !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1630 match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary (NegVD src3) pg)));
1631 ins_cost(SVE_COST);
1632 format %{ "sve_fnmad $dst_src1, $pg, $src2, $src3\t # vector (sve) (D)" %}
1633 ins_encode %{
1634 __ sve_fnmad(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
1635 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1636 %}
1637 ins_pipe(pipe_slow);
1638 %}
1639
1640 // vector fnmls
1641
1642 // dst_src1 = -dst_src1 + src2 * src3
1643 // The NegVF must not be predicated.
1644 instruct vfnmlsF(vReg dst_src1, vReg src2, vReg src3) %{
1645 predicate(UseFMA && UseSVE > 0 &&
1646 !n->in(1)->as_Vector()->is_predicated_vector());
1647 match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 src3)));
1648 ins_cost(SVE_COST);
1649 format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
1650 ins_encode %{
1651 __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ S,
1652 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1653 %}
1654 ins_pipe(pipe_slow);
1655 %}
1656
1657 // dst_src1 = -dst_src1 + src2 * src3
1658 // The NegVD must not be predicated.
1659 instruct vfnmlsD(vReg dst_src1, vReg src2, vReg src3) %{
1660 predicate(UseFMA && UseSVE > 0 &&
1661 !n->in(1)->as_Vector()->is_predicated_vector());
1662 match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 src3)));
1663 ins_cost(SVE_COST);
1664 format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
1665 ins_encode %{
1666 __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ D,
1667 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1668 %}
1669 ins_pipe(pipe_slow);
1670 %}
1671
1672 // vector fnmsb - predicated
1673
1674 // dst_src1 = -src3 + dst_src1 * src2
1675 // The NegVF must not be predicated.
1676 instruct vfnmsbF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1677 predicate(UseFMA && UseSVE > 0 &&
1678 !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1679 match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary (NegVF src3) pg)));
1680 ins_cost(SVE_COST);
1681 format %{ "sve_fnmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) (S)" %}
1682 ins_encode %{
1683 __ sve_fnmsb(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
1684 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1685 %}
1686 ins_pipe(pipe_slow);
1687 %}
1688
1689 // dst_src1 = -src3 + dst_src1 * src2
1690 // The NegVD must not be predicated.
1691 instruct vfnmsbD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1692 predicate(UseFMA && UseSVE > 0 &&
1693 !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1694 match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary (NegVD src3) pg)));
1695 ins_cost(SVE_COST);
1696 format %{ "sve_fnmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) (D)" %}
1697 ins_encode %{
1698 __ sve_fnmsb(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
1699 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1700 %}
1701 ins_pipe(pipe_slow);
1702 %}
1703
1704 // vector mla
1705
1706 // dst_src1 = dst_src1 + src2 * src3
1707 instruct vmlaB(vReg dst_src1, vReg src2, vReg src3)
1708 %{
1709 predicate(UseSVE > 0);
1710 match(Set dst_src1 (AddVB dst_src1 (MulVB src2 src3)));
1711 ins_cost(SVE_COST);
1712 format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (B)" %}
1713 ins_encode %{
1714 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ B,
1715 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1716 %}
1717 ins_pipe(pipe_slow);
1718 %}
1719
1720 // dst_src1 = dst_src1 + src2 * src3
1721 instruct vmlaS(vReg dst_src1, vReg src2, vReg src3)
1722 %{
1723 predicate(UseSVE > 0);
1724 match(Set dst_src1 (AddVS dst_src1 (MulVS src2 src3)));
1725 ins_cost(SVE_COST);
1726 format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (H)" %}
1727 ins_encode %{
1728 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ H,
1729 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1730 %}
1731 ins_pipe(pipe_slow);
1732 %}
1733
1734 // dst_src1 = dst_src1 + src2 * src3
1735 instruct vmlaI(vReg dst_src1, vReg src2, vReg src3)
1736 %{
1737 predicate(UseSVE > 0);
1738 match(Set dst_src1 (AddVI dst_src1 (MulVI src2 src3)));
1739 ins_cost(SVE_COST);
1740 format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (S)" %}
1741 ins_encode %{
1742 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ S,
1743 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1744 %}
1745 ins_pipe(pipe_slow);
1746 %}
1747
1748 // dst_src1 = dst_src1 + src2 * src3
1749 instruct vmlaL(vReg dst_src1, vReg src2, vReg src3)
1750 %{
1751 predicate(UseSVE > 0);
1752 match(Set dst_src1 (AddVL dst_src1 (MulVL src2 src3)));
1753 ins_cost(SVE_COST);
1754 format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (D)" %}
1755 ins_encode %{
1756 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ D,
1757 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1758 %}
1759 ins_pipe(pipe_slow);
1760 %}
1761
1762 // vector mla - predicated
1763
1764 // dst_src1 = dst_src1 + src2 * src3
1765 instruct vmlaB_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1766 %{
1767 predicate(UseSVE > 0);
1768 match(Set dst_src1 (AddVB (Binary dst_src1 (MulVB src2 src3)) pg));
1769 ins_cost(SVE_COST);
1770 format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) (B)" %}
1771 ins_encode %{
1772 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ B, as_PRegister($pg$$reg),
1773 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1774 %}
1775 ins_pipe(pipe_slow);
1776 %}
1777
1778 // dst_src1 = dst_src1 + src2 * src3
1779 instruct vmlaS_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1780 %{
1781 predicate(UseSVE > 0);
1782 match(Set dst_src1 (AddVS (Binary dst_src1 (MulVS src2 src3)) pg));
1783 ins_cost(SVE_COST);
1784 format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) (H)" %}
1785 ins_encode %{
1786 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ H, as_PRegister($pg$$reg),
1787 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1788 %}
1789 ins_pipe(pipe_slow);
1790 %}
1791
1792 // dst_src1 = dst_src1 + src2 * src3
1793 instruct vmlaI_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1794 %{
1795 predicate(UseSVE > 0);
1796 match(Set dst_src1 (AddVI (Binary dst_src1 (MulVI src2 src3)) pg));
1797 ins_cost(SVE_COST);
1798 format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) (S)" %}
1799 ins_encode %{
1800 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
1801 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1802 %}
1803 ins_pipe(pipe_slow);
1804 %}
1805
1806 // dst_src1 = dst_src1 + src2 * src3
1807 instruct vmlaL_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1808 %{
1809 predicate(UseSVE > 0);
1810 match(Set dst_src1 (AddVL (Binary dst_src1 (MulVL src2 src3)) pg));
1811 ins_cost(SVE_COST);
1812 format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) (D)" %}
1813 ins_encode %{
1814 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
1815 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1816 %}
1817 ins_pipe(pipe_slow);
1818 %}
1819
1820 // vector mls
1821
1822 // dst_src1 = dst_src1 - src2 * src3
1823 instruct vmlsB(vReg dst_src1, vReg src2, vReg src3)
1824 %{
1825 predicate(UseSVE > 0);
1826 match(Set dst_src1 (SubVB dst_src1 (MulVB src2 src3)));
1827 ins_cost(SVE_COST);
1828 format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (B)" %}
1829 ins_encode %{
1830 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ B,
1831 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1832 %}
1833 ins_pipe(pipe_slow);
1834 %}
1835
1836 // dst_src1 = dst_src1 - src2 * src3
1837 instruct vmlsS(vReg dst_src1, vReg src2, vReg src3)
1838 %{
1839 predicate(UseSVE > 0);
1840 match(Set dst_src1 (SubVS dst_src1 (MulVS src2 src3)));
1841 ins_cost(SVE_COST);
1842 format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (H)" %}
1843 ins_encode %{
1844 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ H,
1845 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1846 %}
1847 ins_pipe(pipe_slow);
1848 %}
1849
1850 // dst_src1 = dst_src1 - src2 * src3
1851 instruct vmlsI(vReg dst_src1, vReg src2, vReg src3)
1852 %{
1853 predicate(UseSVE > 0);
1854 match(Set dst_src1 (SubVI dst_src1 (MulVI src2 src3)));
1855 ins_cost(SVE_COST);
1856 format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (S)" %}
1857 ins_encode %{
1858 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ S,
1859 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1860 %}
1861 ins_pipe(pipe_slow);
1862 %}
1863
1864 // dst_src1 = dst_src1 - src2 * src3
1865 instruct vmlsL(vReg dst_src1, vReg src2, vReg src3)
1866 %{
1867 predicate(UseSVE > 0);
1868 match(Set dst_src1 (SubVL dst_src1 (MulVL src2 src3)));
1869 ins_cost(SVE_COST);
1870 format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (D)" %}
1871 ins_encode %{
1872 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ D,
1873 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1874 %}
1875 ins_pipe(pipe_slow);
1876 %}
1877
1878 // vector mls - predicated
1879
1880 // dst_src1 = dst_src1 - src2 * src3
1881 instruct vmlsB_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1882 %{
1883 predicate(UseSVE > 0);
1884 match(Set dst_src1 (SubVB (Binary dst_src1 (MulVB src2 src3)) pg));
1885 ins_cost(SVE_COST);
1886 format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) (B)" %}
1887 ins_encode %{
1888 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ B, as_PRegister($pg$$reg),
1889 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1890 %}
1891 ins_pipe(pipe_slow);
1892 %}
1893
1894 // dst_src1 = dst_src1 - src2 * src3
1895 instruct vmlsS_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1896 %{
1897 predicate(UseSVE > 0);
1898 match(Set dst_src1 (SubVS (Binary dst_src1 (MulVS src2 src3)) pg));
1899 ins_cost(SVE_COST);
1900 format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) (H)" %}
1901 ins_encode %{
1902 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ H, as_PRegister($pg$$reg),
1903 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1904 %}
1905 ins_pipe(pipe_slow);
1906 %}
1907
1908 // dst_src1 = dst_src1 - src2 * src3
1909 instruct vmlsI_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1910 %{
1911 predicate(UseSVE > 0);
1912 match(Set dst_src1 (SubVI (Binary dst_src1 (MulVI src2 src3)) pg));
1913 ins_cost(SVE_COST);
1914 format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) (S)" %}
1915 ins_encode %{
1916 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
1917 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1918 %}
1919 ins_pipe(pipe_slow);
1920 %}
1921
1922 // dst_src1 = dst_src1 - src2 * src3
1923 instruct vmlsL_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1924 %{
1925 predicate(UseSVE > 0);
1926 match(Set dst_src1 (SubVL (Binary dst_src1 (MulVL src2 src3)) pg));
1927 ins_cost(SVE_COST);
1928 format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) (D)" %}
1929 ins_encode %{
1930 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
1931 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1932 %}
1933 ins_pipe(pipe_slow);
1934 %}
1935
1936 // vector mul
1937
1938 instruct vmulB(vReg dst_src1, vReg src2) %{
1939 predicate(UseSVE > 0);
1940 match(Set dst_src1 (MulVB dst_src1 src2));
1941 ins_cost(SVE_COST);
1942 format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (B)" %}
1943 ins_encode %{
1944 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ B,
1945 ptrue, as_FloatRegister($src2$$reg));
1946 %}
1947 ins_pipe(pipe_slow);
1948 %}
1949
1950 instruct vmulS(vReg dst_src1, vReg src2) %{
1951 predicate(UseSVE > 0);
1952 match(Set dst_src1 (MulVS dst_src1 src2));
1953 ins_cost(SVE_COST);
1954 format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (H)" %}
1955 ins_encode %{
1956 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ H,
1957 ptrue, as_FloatRegister($src2$$reg));
1958 %}
1959 ins_pipe(pipe_slow);
1960 %}
1961
1962 instruct vmulI(vReg dst_src1, vReg src2) %{
1963 predicate(UseSVE > 0);
1964 match(Set dst_src1 (MulVI dst_src1 src2));
1965 ins_cost(SVE_COST);
1966 format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (S)" %}
1967 ins_encode %{
1968 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ S,
1969 ptrue, as_FloatRegister($src2$$reg));
1970 %}
1971 ins_pipe(pipe_slow);
1972 %}
1973
1974 instruct vmulL(vReg dst_src1, vReg src2) %{
1975 predicate(UseSVE > 0);
1976 match(Set dst_src1 (MulVL dst_src1 src2));
1977 ins_cost(SVE_COST);
1978 format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (D)" %}
1979 ins_encode %{
1980 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ D,
1981 ptrue, as_FloatRegister($src2$$reg));
1982 %}
1983 ins_pipe(pipe_slow);
1984 %}
1985
1986 instruct vmulF(vReg dst, vReg src1, vReg src2) %{
1987 predicate(UseSVE > 0);
1988 match(Set dst (MulVF src1 src2));
1989 ins_cost(SVE_COST);
1990 format %{ "sve_fmul $dst, $src1, $src2\t # vector (sve) (S)" %}
1991 ins_encode %{
1992 __ sve_fmul(as_FloatRegister($dst$$reg), __ S,
1993 as_FloatRegister($src1$$reg),
1994 as_FloatRegister($src2$$reg));
1995 %}
1996 ins_pipe(pipe_slow);
1997 %}
1998
1999 instruct vmulD(vReg dst, vReg src1, vReg src2) %{
2000 predicate(UseSVE > 0);
2001 match(Set dst (MulVD src1 src2));
2002 ins_cost(SVE_COST);
2003 format %{ "sve_fmul $dst, $src1, $src2\t # vector (sve) (D)" %}
2004 ins_encode %{
2005 __ sve_fmul(as_FloatRegister($dst$$reg), __ D,
2006 as_FloatRegister($src1$$reg),
2007 as_FloatRegister($src2$$reg));
2008 %}
2009 ins_pipe(pipe_slow);
2010 %}
2011
2012 // vector mul - predicated
2013
2014 instruct vmulB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
2015 predicate(UseSVE > 0);
2016 match(Set dst_src1 (MulVB (Binary dst_src1 src2) pg));
2017 ins_cost(SVE_COST);
2018 format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
2019 ins_encode %{
2020 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ B,
2021 as_PRegister($pg$$reg),
2022 as_FloatRegister($src2$$reg));
2023 %}
2024 ins_pipe(pipe_slow);
2025 %}
2026
2027 instruct vmulS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
2028 predicate(UseSVE > 0);
2029 match(Set dst_src1 (MulVS (Binary dst_src1 src2) pg));
2030 ins_cost(SVE_COST);
2031 format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
2032 ins_encode %{
2033 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ H,
2034 as_PRegister($pg$$reg),
2035 as_FloatRegister($src2$$reg));
2036 %}
2037 ins_pipe(pipe_slow);
2038 %}
2039
2040 instruct vmulI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
2041 predicate(UseSVE > 0);
2042 match(Set dst_src1 (MulVI (Binary dst_src1 src2) pg));
2043 ins_cost(SVE_COST);
2044 format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
2045 ins_encode %{
2046 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ S,
2047 as_PRegister($pg$$reg),
2048 as_FloatRegister($src2$$reg));
2049 %}
2050 ins_pipe(pipe_slow);
2051 %}
2052
2053 instruct vmulL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
2054 predicate(UseSVE > 0);
2055 match(Set dst_src1 (MulVL (Binary dst_src1 src2) pg));
2056 ins_cost(SVE_COST);
2057 format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
2058 ins_encode %{
2059 __ sve_mul(as_FloatRegister($dst_src1$$reg), __ D,
2060 as_PRegister($pg$$reg),
2061 as_FloatRegister($src2$$reg));
2062 %}
2063 ins_pipe(pipe_slow);
2064 %}
2065
2066 instruct vmulF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
2067 predicate(UseSVE > 0);
2068 match(Set dst_src1 (MulVF (Binary dst_src1 src2) pg));
2069 ins_cost(SVE_COST);
2070 format %{ "sve_fmul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
2071 ins_encode %{
2072 __ sve_fmul(as_FloatRegister($dst_src1$$reg), __ S,
2073 as_PRegister($pg$$reg),
2074 as_FloatRegister($src2$$reg));
2075 %}
2076 ins_pipe(pipe_slow);
2077 %}
2078
2079 instruct vmulD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
2080 predicate(UseSVE > 0);
2081 match(Set dst_src1 (MulVD (Binary dst_src1 src2) pg));
2082 ins_cost(SVE_COST);
2083 format %{ "sve_fmul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
2084 ins_encode %{
2085 __ sve_fmul(as_FloatRegister($dst_src1$$reg), __ D,
2086 as_PRegister($pg$$reg),
2087 as_FloatRegister($src2$$reg));
2088 %}
2089 ins_pipe(pipe_slow);
2090 %}
2091
2092 // vector neg
2093
2094 instruct vnegI(vReg dst, vReg src) %{
2095 predicate(UseSVE > 0 &&
2096 !n->as_Vector()->is_predicated_vector());
2097 match(Set dst (NegVI src));
2098 ins_cost(SVE_COST);
2099 format %{ "sve_neg $dst, $src\t# vector (sve) (B/H/S)" %}
2100 ins_encode %{
2101 BasicType bt = Matcher::vector_element_basic_type(this);
2102 __ sve_neg(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
2103 ptrue, as_FloatRegister($src$$reg));
2104 %}
2105 ins_pipe(pipe_slow);
2106 %}
2107
2108 instruct vnegL(vReg dst, vReg src) %{
2109 predicate(UseSVE > 0 &&
2110 !n->as_Vector()->is_predicated_vector());
2111 match(Set dst (NegVL src));
2112 ins_cost(SVE_COST);
2113 format %{ "sve_neg $dst, $src\t# vector (sve) (D)" %}
2114 ins_encode %{
2115 __ sve_neg(as_FloatRegister($dst$$reg), __ D,
2116 ptrue, as_FloatRegister($src$$reg));
2117 %}
2118 ins_pipe(pipe_slow);
2119 %}
2120
2121 instruct vnegF(vReg dst, vReg src) %{
2122 predicate(UseSVE > 0 &&
2123 !n->as_Vector()->is_predicated_vector());
2124 match(Set dst (NegVF src));
2125 ins_cost(SVE_COST);
2126 format %{ "sve_fneg $dst, $src\t# vector (sve) (S)" %}
2127 ins_encode %{
2128 __ sve_fneg(as_FloatRegister($dst$$reg), __ S,
2129 ptrue, as_FloatRegister($src$$reg));
2130 %}
2131 ins_pipe(pipe_slow);
2132 %}
2133
2134 instruct vnegD(vReg dst, vReg src) %{
2135 predicate(UseSVE > 0 &&
2136 !n->as_Vector()->is_predicated_vector());
2137 match(Set dst (NegVD src));
2138 ins_cost(SVE_COST);
2139 format %{ "sve_fneg $dst, $src\t# vector (sve) (D)" %}
2140 ins_encode %{
2141 __ sve_fneg(as_FloatRegister($dst$$reg), __ D,
2142 ptrue, as_FloatRegister($src$$reg));
2143 %}
2144 ins_pipe(pipe_slow);
2145 %}
2146
2147 // vector neg - predicated
2148
2149 instruct vnegI_masked(vReg dst_src, pRegGov pg) %{
2150 predicate(UseSVE > 0);
2151 match(Set dst_src (NegVI dst_src pg));
2152 ins_cost(SVE_COST);
2153 format %{ "sve_neg $dst_src, $pg, $dst_src\t# vector (sve) (B/H/S)" %}
2154 ins_encode %{
2155 BasicType bt = Matcher::vector_element_basic_type(this);
2156 __ sve_neg(as_FloatRegister($dst_src$$reg), __ elemType_to_regVariant(bt),
2157 as_PRegister($pg$$reg),
2158 as_FloatRegister($dst_src$$reg));
2159 %}
2160 ins_pipe(pipe_slow);
2161 %}
2162
2163 instruct vnegL_masked(vReg dst_src, pRegGov pg) %{
2164 predicate(UseSVE > 0);
2165 match(Set dst_src (NegVL dst_src pg));
2166 ins_cost(SVE_COST);
2167 format %{ "sve_neg $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
2168 ins_encode %{
2169 __ sve_neg(as_FloatRegister($dst_src$$reg), __ D,
2170 as_PRegister($pg$$reg),
2171 as_FloatRegister($dst_src$$reg));
2172 %}
2173 ins_pipe(pipe_slow);
2174 %}
2175
2176 instruct vnegF_masked(vReg dst_src, pRegGov pg) %{
2177 predicate(UseSVE > 0);
2178 match(Set dst_src (NegVF dst_src pg));
2179 ins_cost(SVE_COST);
2180 format %{ "sve_fneg $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
2181 ins_encode %{
2182 __ sve_fneg(as_FloatRegister($dst_src$$reg), __ S,
2183 as_PRegister($pg$$reg),
2184 as_FloatRegister($dst_src$$reg));
2185 %}
2186 ins_pipe(pipe_slow);
2187 %}
2188
2189 instruct vnegD_masked(vReg dst_src, pRegGov pg) %{
2190 predicate(UseSVE > 0);
2191 match(Set dst_src (NegVD dst_src pg));
2192 ins_cost(SVE_COST);
2193 format %{ "sve_fneg $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
2194 ins_encode %{
2195 __ sve_fneg(as_FloatRegister($dst_src$$reg), __ D,
2196 as_PRegister($pg$$reg),
2197 as_FloatRegister($dst_src$$reg));
2198 %}
2199 ins_pipe(pipe_slow);
2200 %}
2201
2202 // vector popcount
2203
2204 instruct vpopcountI(vReg dst, vReg src) %{
2205 predicate(UseSVE > 0 &&
2206 !n->as_Vector()->is_predicated_vector());
2207 match(Set dst (PopCountVI src));
2208 ins_cost(SVE_COST);
2209 format %{ "sve_cnt $dst, $src\t# vector (sve) (B/H/S)" %}
2210 ins_encode %{
2211 assert(UsePopCountInstruction, "unsupported");
2212 BasicType bt = Matcher::vector_element_basic_type(this);
2213 __ sve_cnt(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
2214 ptrue, as_FloatRegister($src$$reg));
2215 %}
2216 ins_pipe(pipe_slow);
2217 %}
2218
2219 instruct vpopcountL(vReg dst, vReg src) %{
2220 predicate(UseSVE > 0 &&
2221 !n->as_Vector()->is_predicated_vector() &&
2222 n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2223 match(Set dst (PopCountVL src));
2224 ins_cost(SVE_COST);
2225 format %{ "sve_cnt $dst, $src\t# vector (sve) (D)" %}
2226 ins_encode %{
2227 assert(UsePopCountInstruction, "unsupported");
2228 __ sve_cnt(as_FloatRegister($dst$$reg), __ D,
2229 ptrue, as_FloatRegister($src$$reg));
2230 %}
2231 ins_pipe(pipe_slow);
2232 %}
2233
2234 // If the PopCountVL is generated by auto-vectorization, the dst basic
2235 // type is T_INT. And once we have unified the type definition for
2236 // Vector API and auto-vectorization, this rule can be merged with
2237 // "vpopcountL" rule.
2238 instruct vpopcountLI(vReg dst, vReg src, vReg vtmp) %{
2239 predicate(UseSVE > 0 &&
2240 !n->as_Vector()->is_predicated_vector() &&
2241 n->bottom_type()->is_vect()->element_basic_type() == T_INT);
2242 match(Set dst (PopCountVL src));
2243 effect(TEMP_DEF dst, TEMP vtmp);
2244 ins_cost(3 * SVE_COST);
2245 format %{ "sve_cnt $dst, $src\n\t"
2246 "sve_dup $vtmp, #0\n\t"
2247 "sve_uzp1 $dst, $dst, $vtmp\t# vector (sve) (S)" %}
2248 ins_encode %{
2249 assert(UsePopCountInstruction, "unsupported");
2250 __ sve_cnt(as_FloatRegister($dst$$reg), __ D,
2251 ptrue, as_FloatRegister($src$$reg));
2252 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
2253 as_FloatRegister($dst$$reg), __ D, as_FloatRegister($vtmp$$reg));
2254 %}
2255 ins_pipe(pipe_slow);
2256 %}
2257
2258 // vector popcount - predicated
2259
2260 instruct vpopcountI_masked(vReg dst_src, pRegGov pg) %{
2261 predicate(UseSVE > 0);
2262 match(Set dst_src (PopCountVI dst_src pg));
2263 ins_cost(SVE_COST);
2264 format %{ "sve_cnt $dst_src, $pg, $dst_src\t# vector (sve) (B/H/S)" %}
2265 ins_encode %{
2266 assert(UsePopCountInstruction, "unsupported");
2267 BasicType bt = Matcher::vector_element_basic_type(this);
2268 __ sve_cnt(as_FloatRegister($dst_src$$reg), __ elemType_to_regVariant(bt),
2269 as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
2270 %}
2271 ins_pipe(pipe_slow);
2272 %}
2273
2274 instruct vpopcountL_masked(vReg dst_src, pRegGov pg) %{
2275 predicate(UseSVE > 0 &&
2276 n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2277 match(Set dst_src (PopCountVL dst_src pg));
2278 ins_cost(SVE_COST);
2279 format %{ "sve_cnt $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
2280 ins_encode %{
2281 assert(UsePopCountInstruction, "unsupported");
2282 __ sve_cnt(as_FloatRegister($dst_src$$reg), __ D,
2283 as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
2284 %}
2285 ins_pipe(pipe_slow);
2286 %}
2287
2288 // vector blend
2289
2290 instruct vblend(vReg dst, vReg src1, vReg src2, pRegGov pg) %{
2291 predicate(UseSVE > 0);
2292 match(Set dst (VectorBlend (Binary src1 src2) pg));
2293 ins_cost(SVE_COST);
2294 format %{ "sve_sel $dst, $pg, $src2, $src1\t# vector blend (sve)" %}
2295 ins_encode %{
2296 Assembler::SIMD_RegVariant size =
2297 __ elemType_to_regVariant(Matcher::vector_element_basic_type(this));
2298 __ sve_sel(as_FloatRegister($dst$$reg), size, as_PRegister($pg$$reg),
2299 as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
2300 %}
2301 ins_pipe(pipe_slow);
2302 %}
2303
2304 // vector store mask
2305
2306 instruct vstoremaskB(vReg dst, pRegGov src, immI_1 size) %{
2307 predicate(UseSVE > 0);
2308 match(Set dst (VectorStoreMask src size));
2309 ins_cost(SVE_COST);
2310 format %{ "vstoremask $dst, $src\t# vector store mask (sve) (B)" %}
2311 ins_encode %{
2312 __ sve_cpy(as_FloatRegister($dst$$reg), __ B, as_PRegister($src$$reg), 1, false);
2313 %}
2314 ins_pipe(pipe_slow);
2315 %}
2316
2317 instruct vstoremask_narrow(vReg dst, pRegGov src, vReg tmp, immI_gt_1 size) %{
2318 predicate(UseSVE > 0);
2319 match(Set dst (VectorStoreMask src size));
2320 effect(TEMP_DEF dst, TEMP tmp);
2321 ins_cost(3 * SVE_COST);
2322 format %{ "vstoremask $dst, $src\t# vector store mask (sve) (H/S/D)" %}
2323 ins_encode %{
2324 Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant((int)$size$$constant);
2325 __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($src$$reg), 1, false);
2326 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
2327 as_FloatRegister($dst$$reg), size, as_FloatRegister($tmp$$reg));
2328 %}
2329 ins_pipe(pipe_slow);
2330 %}
2331
2332 // Combine LoadVector+VectorLoadMask when the vector element type is not T_BYTE
2333
2334 instruct vloadmask_loadV(pRegGov dst, indirect mem, vReg tmp, rFlagsReg cr) %{
2335 predicate(UseSVE > 0 &&
2336 n->as_Vector()->length_in_bytes() == MaxVectorSize &&
2337 type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
2338 match(Set dst (VectorLoadMask (LoadVector mem)));
2339 effect(TEMP tmp, KILL cr);
2340 ins_cost(3 * SVE_COST);
2341 format %{ "sve_ld1b $tmp, $mem\n\t"
2342 "sve_cmpne $dst, $tmp, 0\t# load vector mask (sve) (H/S/D)" %}
2343 ins_encode %{
2344 // Load mask values which are boolean type, and extend them to the
2345 // expected vector element type. Convert the vector to predicate.
2346 BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
2347 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($tmp$$reg),
2348 ptrue, T_BOOLEAN, to_vect_bt, $mem->opcode(),
2349 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2350 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ elemType_to_regVariant(to_vect_bt),
2351 ptrue, as_FloatRegister($tmp$$reg), 0);
2352 %}
2353 ins_pipe(pipe_slow);
2354 %}
2355
2356 instruct vloadmask_loadV_partial(pRegGov dst, indirect mem, vReg vtmp, pRegGov ptmp, rFlagsReg cr) %{
2357 predicate(UseSVE > 0 &&
2358 n->as_Vector()->length_in_bytes() > 16 &&
2359 n->as_Vector()->length_in_bytes() < MaxVectorSize &&
2360 type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
2361 match(Set dst (VectorLoadMask (LoadVector mem)));
2362 effect(TEMP vtmp, TEMP ptmp, KILL cr);
2363 ins_cost(6 * SVE_COST);
2364 format %{ "vloadmask_loadV $dst, $mem\t# load vector mask partial (sve) (H/S/D)" %}
2365 ins_encode %{
2366 // Load valid mask values which are boolean type, and extend them to the
2367 // expected vector element type. Convert the vector to predicate.
2368 BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
2369 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(to_vect_bt);
2370 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this));
2371 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($vtmp$$reg),
2372 as_PRegister($ptmp$$reg), T_BOOLEAN, to_vect_bt, $mem->opcode(),
2373 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2374 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($vtmp$$reg), 0);
2375 %}
2376 ins_pipe(pipe_slow);
2377 %}
2378
2379 // Combine VectorStoreMask+StoreVector when the vector element type is not T_BYTE
2380
2381 instruct storeV_vstoremask(indirect mem, pRegGov src, vReg tmp, immI_gt_1 esize) %{
2382 predicate(UseSVE > 0 &&
2383 Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
2384 match(Set mem (StoreVector mem (VectorStoreMask src esize)));
2385 effect(TEMP tmp);
2386 ins_cost(3 * SVE_COST);
2387 format %{ "sve_cpy $tmp, $src, 1\n\t"
2388 "sve_st1b $tmp, $mem\t# store vector mask (sve) (H/S/D)" %}
2389 ins_encode %{
2390 BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
2391 assert(type2aelembytes(from_vect_bt) == (int)$esize$$constant, "unsupported type.");
2392 Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
2393 __ sve_cpy(as_FloatRegister($tmp$$reg), size, as_PRegister($src$$reg), 1, false);
2394 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($tmp$$reg),
2395 ptrue, T_BOOLEAN, from_vect_bt, $mem->opcode(),
2396 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2397 %}
2398 ins_pipe(pipe_slow);
2399 %}
2400
2401 instruct storeV_vstoremask_partial(indirect mem, pRegGov src, vReg vtmp,
2402 immI_gt_1 esize, pRegGov ptmp, rFlagsReg cr) %{
2403 predicate(UseSVE > 0 &&
2404 n->as_StoreVector()->memory_size() > 16 &&
2405 type2aelembytes(n->as_StoreVector()->vect_type()->element_basic_type()) > 1 &&
2406 Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
2407 match(Set mem (StoreVector mem (VectorStoreMask src esize)));
2408 effect(TEMP vtmp, TEMP ptmp, KILL cr);
2409 format %{ "storeV_vstoremask $src, $mem\t# store vector mask partial (sve) (H/S/D)" %}
2410 ins_cost(6 * SVE_COST);
2411 ins_encode %{
2412 // Convert the valid src predicate to vector, and store the vector
2413 // elements as boolean values.
2414 BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
2415 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(from_vect_bt);
2416 __ sve_cpy(as_FloatRegister($vtmp$$reg), size, as_PRegister($src$$reg), 1, false);
2417 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
2418 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($vtmp$$reg),
2419 as_PRegister($ptmp$$reg), T_BOOLEAN, from_vect_bt, $mem->opcode(),
2420 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2421 %}
2422 ins_pipe(pipe_slow);
2423 %}
2424
2425 // vector add reduction
2426
2427 instruct reduce_addI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
2428 predicate(UseSVE > 0 &&
2429 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2430 match(Set dst (AddReductionVI src1 src2));
2431 effect(TEMP_DEF dst, TEMP tmp);
2432 ins_cost(SVE_COST);
2433 format %{ "sve_reduce_addI $dst, $src1, $src2\t# addI reduction (sve) (may extend)" %}
2434 ins_encode %{
2435 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2436 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2437 $src1$$Register, as_FloatRegister($src2$$reg),
2438 ptrue, as_FloatRegister($tmp$$reg));
2439 %}
2440 ins_pipe(pipe_slow);
2441 %}
2442
2443 instruct reduce_addL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
2444 predicate(UseSVE > 0 &&
2445 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2446 match(Set dst (AddReductionVL src1 src2));
2447 effect(TEMP_DEF dst, TEMP tmp);
2448 ins_cost(SVE_COST);
2449 format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction (sve)" %}
2450 ins_encode %{
2451 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2452 $src1$$Register, as_FloatRegister($src2$$reg),
2453 ptrue, as_FloatRegister($tmp$$reg));
2454 %}
2455 ins_pipe(pipe_slow);
2456 %}
2457
2458 instruct reduce_addF(vRegF src1_dst, vReg src2) %{
2459 predicate(UseSVE > 0 &&
2460 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2461 match(Set src1_dst (AddReductionVF src1_dst src2));
2462 ins_cost(SVE_COST);
2463 format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (S)" %}
2464 ins_encode %{
2465 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
2466 ptrue, as_FloatRegister($src2$$reg));
2467 %}
2468 ins_pipe(pipe_slow);
2469 %}
2470
2471 instruct reduce_addD(vRegD src1_dst, vReg src2) %{
2472 predicate(UseSVE > 0 &&
2473 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2474 match(Set src1_dst (AddReductionVD src1_dst src2));
2475 ins_cost(SVE_COST);
2476 format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (D)" %}
2477 ins_encode %{
2478 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
2479 ptrue, as_FloatRegister($src2$$reg));
2480 %}
2481 ins_pipe(pipe_slow);
2482 %}
2483
2484 instruct reduce_addI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2485 pRegGov ptmp, rFlagsReg cr) %{
2486 predicate(UseSVE > 0 &&
2487 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2488 match(Set dst (AddReductionVI src1 src2));
2489 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2490 ins_cost(2 * SVE_COST);
2491 format %{ "sve_reduce_addI $dst, $src1, $src2\t# addI reduction partial (sve) (may extend)" %}
2492 ins_encode %{
2493 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2494 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2495 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
2496 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2497 $src1$$Register, as_FloatRegister($src2$$reg),
2498 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2499 %}
2500 ins_pipe(pipe_slow);
2501 %}
2502
2503 instruct reduce_addL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2504 pRegGov ptmp, rFlagsReg cr) %{
2505 predicate(UseSVE > 0 &&
2506 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2507 match(Set dst (AddReductionVL src1 src2));
2508 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2509 ins_cost(2 * SVE_COST);
2510 format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction partial (sve)" %}
2511 ins_encode %{
2512 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
2513 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2514 $src1$$Register, as_FloatRegister($src2$$reg),
2515 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2516 %}
2517 ins_pipe(pipe_slow);
2518 %}
2519
2520 instruct reduce_addF_partial(vRegF src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
2521 predicate(UseSVE > 0 &&
2522 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2523 match(Set src1_dst (AddReductionVF src1_dst src2));
2524 ins_cost(SVE_COST);
2525 effect(TEMP ptmp, KILL cr);
2526 format %{ "sve_reduce_addF $src1_dst, $src1_dst, $src2\t# addF reduction partial (sve) (S)" %}
2527 ins_encode %{
2528 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src2));
2529 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
2530 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2531 %}
2532 ins_pipe(pipe_slow);
2533 %}
2534
2535 instruct reduce_addD_partial(vRegD src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
2536 predicate(UseSVE > 0 &&
2537 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2538 match(Set src1_dst (AddReductionVD src1_dst src2));
2539 ins_cost(SVE_COST);
2540 effect(TEMP ptmp, KILL cr);
2541 format %{ "sve_reduce_addD $src1_dst, $src1_dst, $src2\t# addD reduction partial (sve) (D)" %}
2542 ins_encode %{
2543 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
2544 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
2545 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
2546 %}
2547 ins_pipe(pipe_slow);
2548 %}
2549
2550 // vector add reduction - predicated
2551
2552 instruct reduce_addI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2553 predicate(UseSVE > 0);
2554 match(Set dst (AddReductionVI (Binary src1 src2) pg));
2555 effect(TEMP_DEF dst, TEMP tmp);
2556 ins_cost(SVE_COST);
2557 format %{ "sve_reduce_addI $dst, $src1, $pg, $src2\t# addI reduction predicated (sve) (may extend)" %}
2558 ins_encode %{
2559 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2560 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2561 $src1$$Register, as_FloatRegister($src2$$reg),
2562 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2563 %}
2564 ins_pipe(pipe_slow);
2565 %}
2566
2567 instruct reduce_addL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2568 predicate(UseSVE > 0);
2569 match(Set dst (AddReductionVL (Binary src1 src2) pg));
2570 effect(TEMP_DEF dst, TEMP tmp);
2571 ins_cost(SVE_COST);
2572 format %{ "sve_reduce_addL $dst, $src1, $pg, $src2\t# addL reduction predicated (sve)" %}
2573 ins_encode %{
2574 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2575 $src1$$Register, as_FloatRegister($src2$$reg),
2576 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2577 %}
2578 ins_pipe(pipe_slow);
2579 %}
2580
2581 instruct reduce_addF_masked(vRegF src1_dst, vReg src2, pRegGov pg) %{
2582 predicate(UseSVE > 0);
2583 match(Set src1_dst (AddReductionVF (Binary src1_dst src2) pg));
2584 ins_cost(SVE_COST);
2585 format %{ "sve_reduce_addF $src1_dst, $pg, $src2\t# addF reduction predicated (sve)" %}
2586 ins_encode %{
2587 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
2588 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
2589 %}
2590 ins_pipe(pipe_slow);
2591 %}
2592
2593 instruct reduce_addD_masked(vRegD src1_dst, vReg src2, pRegGov pg) %{
2594 predicate(UseSVE > 0);
2595 match(Set src1_dst (AddReductionVD (Binary src1_dst src2) pg));
2596 ins_cost(SVE_COST);
2597 format %{ "sve_reduce_addD $src1_dst, $pg, $src2\t# addD reduction predicated (sve)" %}
2598 ins_encode %{
2599 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
2600 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
2601 %}
2602 ins_pipe(pipe_slow);
2603 %}
2604
2605 // vector and reduction
2606
2607 instruct reduce_andI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
2608 predicate(UseSVE > 0 &&
2609 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2610 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2611 match(Set dst (AndReductionV src1 src2));
2612 effect(TEMP_DEF dst, TEMP tmp);
2613 ins_cost(SVE_COST);
2614 format %{ "sve_reduce_andI $dst, $src1, $src2\t# andI reduction (sve) (may extend)" %}
2615 ins_encode %{
2616 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2617 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2618 $src1$$Register, as_FloatRegister($src2$$reg),
2619 ptrue, as_FloatRegister($tmp$$reg));
2620 %}
2621 ins_pipe(pipe_slow);
2622 %}
2623
2624 instruct reduce_andL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
2625 predicate(UseSVE > 0 &&
2626 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2627 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2628 match(Set dst (AndReductionV src1 src2));
2629 effect(TEMP_DEF dst, TEMP tmp);
2630 ins_cost(SVE_COST);
2631 format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction (sve)" %}
2632 ins_encode %{
2633 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2634 $src1$$Register, as_FloatRegister($src2$$reg),
2635 ptrue, as_FloatRegister($tmp$$reg));
2636 %}
2637 ins_pipe(pipe_slow);
2638 %}
2639
2640 instruct reduce_andI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2641 pRegGov ptmp, rFlagsReg cr) %{
2642 predicate(UseSVE > 0 &&
2643 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2644 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2645 match(Set dst (AndReductionV src1 src2));
2646 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2647 ins_cost(2 * SVE_COST);
2648 format %{ "sve_reduce_andI $dst, $src1, $src2\t# andI reduction partial (sve) (may extend)" %}
2649 ins_encode %{
2650 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2651 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2652 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
2653 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2654 $src1$$Register, as_FloatRegister($src2$$reg),
2655 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2656 %}
2657 ins_pipe(pipe_slow);
2658 %}
2659
2660 instruct reduce_andL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2661 pRegGov ptmp, rFlagsReg cr) %{
2662 predicate(UseSVE > 0 &&
2663 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2664 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2665 match(Set dst (AndReductionV src1 src2));
2666 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2667 ins_cost(2 * SVE_COST);
2668 format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction partial (sve)" %}
2669 ins_encode %{
2670 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
2671 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2672 $src1$$Register, as_FloatRegister($src2$$reg),
2673 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2674 %}
2675 ins_pipe(pipe_slow);
2676 %}
2677
2678 // vector and reduction - predicated
2679
2680 instruct reduce_andI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2681 predicate(UseSVE > 0 &&
2682 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG);
2683 match(Set dst (AndReductionV (Binary src1 src2) pg));
2684 effect(TEMP_DEF dst, TEMP tmp);
2685 ins_cost(SVE_COST);
2686 format %{ "sve_reduce_andI $dst, $src1, $pg, $src2\t# andI reduction predicated (sve) (may extend)" %}
2687 ins_encode %{
2688 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2689 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2690 $src1$$Register, as_FloatRegister($src2$$reg),
2691 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2692 %}
2693 ins_pipe(pipe_slow);
2694 %}
2695
2696 instruct reduce_andL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2697 predicate(UseSVE > 0 &&
2698 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2699 match(Set dst (AndReductionV (Binary src1 src2) pg));
2700 effect(TEMP_DEF dst, TEMP tmp);
2701 ins_cost(SVE_COST);
2702 format %{ "sve_reduce_andL $dst, $src1, $pg, $src2\t# andL reduction predicated (sve)" %}
2703 ins_encode %{
2704 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2705 $src1$$Register, as_FloatRegister($src2$$reg),
2706 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2707 %}
2708 ins_pipe(pipe_slow);
2709 %}
2710
2711 // vector or reduction
2712
2713 instruct reduce_orI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
2714 predicate(UseSVE > 0 &&
2715 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2716 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2717 match(Set dst (OrReductionV src1 src2));
2718 effect(TEMP_DEF dst, TEMP tmp);
2719 ins_cost(SVE_COST);
2720 format %{ "sve_reduce_orI $dst, $src1, $src2\t# orI reduction (sve) (may extend)" %}
2721 ins_encode %{
2722 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2723 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2724 $src1$$Register, as_FloatRegister($src2$$reg),
2725 ptrue, as_FloatRegister($tmp$$reg));
2726 %}
2727 ins_pipe(pipe_slow);
2728 %}
2729
2730 instruct reduce_orL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
2731 predicate(UseSVE > 0 &&
2732 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2733 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2734 match(Set dst (OrReductionV src1 src2));
2735 effect(TEMP_DEF dst, TEMP tmp);
2736 ins_cost(SVE_COST);
2737 format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction (sve)" %}
2738 ins_encode %{
2739 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2740 $src1$$Register, as_FloatRegister($src2$$reg),
2741 ptrue, as_FloatRegister($tmp$$reg));
2742 %}
2743 ins_pipe(pipe_slow);
2744 %}
2745
2746 instruct reduce_orI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2747 pRegGov ptmp, rFlagsReg cr) %{
2748 predicate(UseSVE > 0 &&
2749 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2750 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2751 match(Set dst (OrReductionV src1 src2));
2752 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2753 ins_cost(2 * SVE_COST);
2754 format %{ "sve_reduce_orI $dst, $src1, $src2\t# orI reduction partial (sve) (may extend)" %}
2755 ins_encode %{
2756 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2757 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2758 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
2759 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2760 $src1$$Register, as_FloatRegister($src2$$reg),
2761 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2762 %}
2763 ins_pipe(pipe_slow);
2764 %}
2765
2766 instruct reduce_orL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2767 pRegGov ptmp, rFlagsReg cr) %{
2768 predicate(UseSVE > 0 &&
2769 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2770 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2771 match(Set dst (OrReductionV src1 src2));
2772 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2773 ins_cost(2 * SVE_COST);
2774 format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction partial (sve)" %}
2775 ins_encode %{
2776 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
2777 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2778 $src1$$Register, as_FloatRegister($src2$$reg),
2779 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2780 %}
2781 ins_pipe(pipe_slow);
2782 %}
2783
2784 // vector or reduction - predicated
2785
2786 instruct reduce_orI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2787 predicate(UseSVE > 0 &&
2788 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG);
2789 match(Set dst (OrReductionV (Binary src1 src2) pg));
2790 effect(TEMP_DEF dst, TEMP tmp);
2791 ins_cost(SVE_COST);
2792 format %{ "sve_reduce_orI $dst, $src1, $pg, $src2\t# orI reduction predicated (sve) (may extend)" %}
2793 ins_encode %{
2794 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2795 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2796 $src1$$Register, as_FloatRegister($src2$$reg),
2797 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2798 %}
2799 ins_pipe(pipe_slow);
2800 %}
2801
2802 instruct reduce_orL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2803 predicate(UseSVE > 0 &&
2804 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2805 match(Set dst (OrReductionV (Binary src1 src2) pg));
2806 effect(TEMP_DEF dst, TEMP tmp);
2807 ins_cost(SVE_COST);
2808 format %{ "sve_reduce_orL $dst, $src1, $pg, $src2\t# orL reduction predicated (sve)" %}
2809 ins_encode %{
2810 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2811 $src1$$Register, as_FloatRegister($src2$$reg),
2812 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2813 %}
2814 ins_pipe(pipe_slow);
2815 %}
2816
2817 // vector xor reduction
2818
2819 instruct reduce_eorI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
2820 predicate(UseSVE > 0 &&
2821 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2822 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2823 match(Set dst (XorReductionV src1 src2));
2824 effect(TEMP_DEF dst, TEMP tmp);
2825 ins_cost(SVE_COST);
2826 format %{ "sve_reduce_eorI $dst, $src1, $src2\t# eorI reduction (sve) (may extend)" %}
2827 ins_encode %{
2828 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2829 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2830 $src1$$Register, as_FloatRegister($src2$$reg),
2831 ptrue, as_FloatRegister($tmp$$reg));
2832 %}
2833 ins_pipe(pipe_slow);
2834 %}
2835
2836 instruct reduce_eorL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
2837 predicate(UseSVE > 0 &&
2838 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2839 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2840 match(Set dst (XorReductionV src1 src2));
2841 effect(TEMP_DEF dst, TEMP tmp);
2842 ins_cost(SVE_COST);
2843 format %{ "sve_reduce_eorL $dst, $src1, $src2\t# eorL reduction (sve)" %}
2844 ins_encode %{
2845 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2846 $src1$$Register, as_FloatRegister($src2$$reg),
2847 ptrue, as_FloatRegister($tmp$$reg));
2848 %}
2849 ins_pipe(pipe_slow);
2850 %}
2851
2852 instruct reduce_eorI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2853 pRegGov ptmp, rFlagsReg cr) %{
2854 predicate(UseSVE > 0 &&
2855 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2856 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2857 match(Set dst (XorReductionV src1 src2));
2858 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2859 ins_cost(2 * SVE_COST);
2860 format %{ "sve_reduce_eorI $dst, $src1, $src2\t# eorI reduction partial (sve) (may extend)" %}
2861 ins_encode %{
2862 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2863 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2864 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
2865 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2866 $src1$$Register, as_FloatRegister($src2$$reg),
2867 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2868 %}
2869 ins_pipe(pipe_slow);
2870 %}
2871
2872 instruct reduce_eorL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2873 pRegGov ptmp, rFlagsReg cr) %{
2874 predicate(UseSVE > 0 &&
2875 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
2876 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2877 match(Set dst (XorReductionV src1 src2));
2878 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2879 ins_cost(2 * SVE_COST);
2880 format %{ "sve_reduce_eorL $dst, $src1, $src2\t# eorL reduction partial (sve)" %}
2881 ins_encode %{
2882 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
2883 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2884 $src1$$Register, as_FloatRegister($src2$$reg),
2885 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2886 %}
2887 ins_pipe(pipe_slow);
2888 %}
2889
2890 // vector xor reduction - predicated
2891
2892 instruct reduce_eorI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
2893 predicate(UseSVE > 0 &&
2894 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG);
2895 match(Set dst (XorReductionV (Binary src1 src2) pg));
2896 effect(TEMP_DEF dst, TEMP tmp);
2897 ins_cost(SVE_COST);
2898 format %{ "sve_reduce_eorI $dst, $src1, $pg, $src2\t# eorI reduction predicated (sve) (may extend)" %}
2899 ins_encode %{
2900 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2901 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2902 $src1$$Register, as_FloatRegister($src2$$reg),
2903 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2904 %}
2905 ins_pipe(pipe_slow);
2906 %}
2907
2908 instruct reduce_eorL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
2909 predicate(UseSVE > 0 &&
2910 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2911 match(Set dst (XorReductionV (Binary src1 src2) pg));
2912 effect(TEMP_DEF dst, TEMP tmp);
2913 ins_cost(SVE_COST);
2914 format %{ "sve_reduce_eorL $dst, $src1, $pg, $src2\t# eorL reduction predicated (sve)" %}
2915 ins_encode %{
2916 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2917 $src1$$Register, as_FloatRegister($src2$$reg),
2918 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
2919 %}
2920 ins_pipe(pipe_slow);
2921 %}
2922
2923 // vector max reduction
2924
2925 instruct reduce_maxI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
2926 predicate(UseSVE > 0 &&
2927 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2928 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2929 is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
2930 match(Set dst (MaxReductionV src1 src2));
2931 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2932 ins_cost(SVE_COST);
2933 format %{ "sve_reduce_maxI $dst, $src1, $src2\t# maxI reduction (sve)" %}
2934 ins_encode %{
2935 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2936 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2937 $src1$$Register, as_FloatRegister($src2$$reg),
2938 ptrue, as_FloatRegister($tmp$$reg));
2939 %}
2940 ins_pipe(pipe_slow);
2941 %}
2942
2943 instruct reduce_maxL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
2944 predicate(UseSVE > 0 &&
2945 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2946 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2947 match(Set dst (MaxReductionV src1 src2));
2948 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
2949 ins_cost(SVE_COST);
2950 format %{ "sve_reduce_maxL $dst, $src1, $src2\t# maxL reduction (sve)" %}
2951 ins_encode %{
2952 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2953 $src1$$Register, as_FloatRegister($src2$$reg),
2954 ptrue, as_FloatRegister($tmp$$reg));
2955 %}
2956 ins_pipe(pipe_slow);
2957 %}
2958
2959 instruct reduce_maxI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
2960 pRegGov ptmp, rFlagsReg cr) %{
2961 predicate(UseSVE > 0 &&
2962 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2963 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
2964 is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
2965 match(Set dst (MaxReductionV src1 src2));
2966 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2967 ins_cost(2 * SVE_COST);
2968 format %{ "sve_reduce_maxI $dst, $src1, $src2\t# maxI reduction partial (sve)" %}
2969 ins_encode %{
2970 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2971 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
2972 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
2973 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
2974 $src1$$Register, as_FloatRegister($src2$$reg),
2975 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2976 %}
2977 ins_pipe(pipe_slow);
2978 %}
2979
2980 instruct reduce_maxL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
2981 pRegGov ptmp, rFlagsReg cr) %{
2982 predicate(UseSVE > 0 &&
2983 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2984 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2985 match(Set dst (MaxReductionV src1 src2));
2986 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
2987 ins_cost(2 * SVE_COST);
2988 format %{ "sve_reduce_maxL $dst, $src1, $src2\t# maxL reduction partial (sve)" %}
2989 ins_encode %{
2990 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
2991 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
2992 $src1$$Register, as_FloatRegister($src2$$reg),
2993 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
2994 %}
2995 ins_pipe(pipe_slow);
2996 %}
2997
2998 instruct reduce_maxF(vRegF dst, vRegF src1, vReg src2) %{
2999 predicate(UseSVE > 0 &&
3000 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3001 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3002 match(Set dst (MaxReductionV src1 src2));
3003 ins_cost(INSN_COST);
3004 effect(TEMP_DEF dst);
3005 format %{ "sve_reduce_maxF $dst, $src1, $src2\t# maxF reduction (sve)" %}
3006 ins_encode %{
3007 __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src2$$reg));
3008 __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3009 %}
3010 ins_pipe(pipe_slow);
3011 %}
3012
3013 instruct reduce_maxF_partial(vRegF dst, vRegF src1, vReg src2,
3014 pRegGov ptmp, rFlagsReg cr) %{
3015 predicate(UseSVE > 0 &&
3016 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3017 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3018 match(Set dst (MaxReductionV src1 src2));
3019 ins_cost(INSN_COST);
3020 effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3021 format %{ "sve_reduce_maxF $dst, $src1, $src2\t# maxF reduction partial (sve)" %}
3022 ins_encode %{
3023 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src2));
3024 __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
3025 __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3026 %}
3027 ins_pipe(pipe_slow);
3028 %}
3029
3030 instruct reduce_maxD(vRegD dst, vRegD src1, vReg src2) %{
3031 predicate(UseSVE > 0 &&
3032 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3033 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3034 match(Set dst (MaxReductionV src1 src2));
3035 ins_cost(INSN_COST);
3036 effect(TEMP_DEF dst);
3037 format %{ "sve_reduce_maxD $dst, $src1, $src2\t# maxD reduction (sve)" %}
3038 ins_encode %{
3039 __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
3040 __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3041 %}
3042 ins_pipe(pipe_slow);
3043 %}
3044
3045 instruct reduce_maxD_partial(vRegD dst, vRegD src1, vReg src2,
3046 pRegGov ptmp, rFlagsReg cr) %{
3047 predicate(UseSVE > 0 &&
3048 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3049 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3050 match(Set dst (MaxReductionV src1 src2));
3051 ins_cost(INSN_COST);
3052 effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3053 format %{ "sve_reduce_maxD $dst, $src1, $src2\t# maxD reduction partial (sve)" %}
3054 ins_encode %{
3055 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
3056 __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
3057 __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3058 %}
3059 ins_pipe(pipe_slow);
3060 %}
3061
3062 // vector max reduction - predicated
3063
3064 instruct reduce_maxI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp,
3065 pRegGov pg, rFlagsReg cr) %{
3066 predicate(UseSVE > 0 &&
3067 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
3068 is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
3069 match(Set dst (MaxReductionV (Binary src1 src2) pg));
3070 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3071 ins_cost(SVE_COST);
3072 format %{ "sve_reduce_maxI $dst, $src1, $pg, $src2\t# maxI reduction predicated (sve)" %}
3073 ins_encode %{
3074 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
3075 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
3076 $src1$$Register, as_FloatRegister($src2$$reg),
3077 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
3078 %}
3079 ins_pipe(pipe_slow);
3080 %}
3081
3082 instruct reduce_maxL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp,
3083 pRegGov pg, rFlagsReg cr) %{
3084 predicate(UseSVE > 0 &&
3085 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3086 match(Set dst (MaxReductionV (Binary src1 src2) pg));
3087 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3088 ins_cost(SVE_COST);
3089 format %{ "sve_reduce_maxL $dst, $src1, $pg, $src2\t# maxL reduction predicated (sve)" %}
3090 ins_encode %{
3091 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
3092 $src1$$Register, as_FloatRegister($src2$$reg),
3093 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
3094 %}
3095 ins_pipe(pipe_slow);
3096 %}
3097
3098 instruct reduce_maxF_masked(vRegF dst, vRegF src1, vReg src2, pRegGov pg) %{
3099 predicate(UseSVE > 0 &&
3100 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
3101 match(Set dst (MaxReductionV (Binary src1 src2) pg));
3102 ins_cost(SVE_COST);
3103 effect(TEMP_DEF dst);
3104 format %{ "sve_reduce_maxF $dst, $src1, $pg, $src2\t# maxF reduction predicated (sve)" %}
3105 ins_encode %{
3106 __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
3107 __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3108 %}
3109 ins_pipe(pipe_slow);
3110 %}
3111
3112 instruct reduce_maxD_masked(vRegD dst, vRegD src1, vReg src2, pRegGov pg) %{
3113 predicate(UseSVE > 0 &&
3114 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
3115 match(Set dst (MaxReductionV (Binary src1 src2) pg));
3116 ins_cost(SVE_COST);
3117 effect(TEMP_DEF dst);
3118 format %{ "sve_reduce_maxD $dst, $src1, $pg, $src2\t# maxD reduction predicated (sve)" %}
3119 ins_encode %{
3120 __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
3121 __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3122 %}
3123 ins_pipe(pipe_slow);
3124 %}
3125
3126 // vector min reduction
3127
3128 instruct reduce_minI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
3129 predicate(UseSVE > 0 &&
3130 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
3131 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
3132 is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
3133 match(Set dst (MinReductionV src1 src2));
3134 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3135 ins_cost(SVE_COST);
3136 format %{ "sve_reduce_minI $dst, $src1, $src2\t# minI reduction (sve)" %}
3137 ins_encode %{
3138 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
3139 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
3140 $src1$$Register, as_FloatRegister($src2$$reg),
3141 ptrue, as_FloatRegister($tmp$$reg));
3142 %}
3143 ins_pipe(pipe_slow);
3144 %}
3145
3146 instruct reduce_minL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
3147 predicate(UseSVE > 0 &&
3148 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
3149 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3150 match(Set dst (MinReductionV src1 src2));
3151 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3152 ins_cost(SVE_COST);
3153 format %{ "sve_reduce_minL $dst, $src1, $src2\t# minL reduction (sve)" %}
3154 ins_encode %{
3155 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
3156 $src1$$Register, as_FloatRegister($src2$$reg),
3157 ptrue, as_FloatRegister($tmp$$reg));
3158 %}
3159 ins_pipe(pipe_slow);
3160 %}
3161
3162 instruct reduce_minI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
3163 pRegGov ptmp, rFlagsReg cr) %{
3164 predicate(UseSVE > 0 &&
3165 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
3166 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
3167 is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
3168 match(Set dst (MinReductionV src1 src2));
3169 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
3170 ins_cost(2 * SVE_COST);
3171 format %{ "sve_reduce_minI $dst, $src1, $src2\t# minI reduction partial (sve)" %}
3172 ins_encode %{
3173 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
3174 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
3175 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
3176 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
3177 $src1$$Register, as_FloatRegister($src2$$reg),
3178 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
3179 %}
3180 ins_pipe(pipe_slow);
3181 %}
3182
3183 instruct reduce_minL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
3184 pRegGov ptmp, rFlagsReg cr) %{
3185 predicate(UseSVE > 0 &&
3186 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
3187 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3188 match(Set dst (MinReductionV src1 src2));
3189 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
3190 ins_cost(2 * SVE_COST);
3191 format %{ "sve_reduce_minL $dst, $src1, $src2\t# minL reduction partial (sve)" %}
3192 ins_encode %{
3193 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
3194 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
3195 $src1$$Register, as_FloatRegister($src2$$reg),
3196 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
3197 %}
3198 ins_pipe(pipe_slow);
3199 %}
3200
3201 instruct reduce_minF(vRegF dst, vRegF src1, vReg src2) %{
3202 predicate(UseSVE > 0 &&
3203 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3204 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3205 match(Set dst (MinReductionV src1 src2));
3206 ins_cost(INSN_COST);
3207 effect(TEMP_DEF dst);
3208 format %{ "sve_reduce_minF $dst, $src1, $src2\t# minF reduction (sve)" %}
3209 ins_encode %{
3210 __ sve_fminv(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src2$$reg));
3211 __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3212 %}
3213 ins_pipe(pipe_slow);
3214 %}
3215
3216 instruct reduce_minF_partial(vRegF dst, vRegF src1, vReg src2,
3217 pRegGov ptmp, rFlagsReg cr) %{
3218 predicate(UseSVE > 0 &&
3219 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
3220 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3221 match(Set dst (MinReductionV src1 src2));
3222 ins_cost(INSN_COST);
3223 effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3224 format %{ "sve_reduce_minF $dst, $src1, $src2\t# minF reduction partial (sve)" %}
3225 ins_encode %{
3226 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src2));
3227 __ sve_fminv(as_FloatRegister($dst$$reg), __ S, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
3228 __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3229 %}
3230 ins_pipe(pipe_slow);
3231 %}
3232
3233 instruct reduce_minD(vRegD dst, vRegD src1, vReg src2) %{
3234 predicate(UseSVE > 0 &&
3235 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3236 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3237 match(Set dst (MinReductionV src1 src2));
3238 ins_cost(INSN_COST);
3239 effect(TEMP_DEF dst);
3240 format %{ "sve_reduce_minD $dst, $src1, $src2\t# minD reduction (sve)" %}
3241 ins_encode %{
3242 __ sve_fminv(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
3243 __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3244 %}
3245 ins_pipe(pipe_slow);
3246 %}
3247
3248 instruct reduce_minD_partial(vRegD dst, vRegD src1, vReg src2,
3249 pRegGov ptmp, rFlagsReg cr) %{
3250 predicate(UseSVE > 0 &&
3251 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
3252 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3253 match(Set dst (MinReductionV src1 src2));
3254 ins_cost(INSN_COST);
3255 effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
3256 format %{ "sve_reduce_minD $dst, $src1, $src2\t# minD reduction partial (sve)" %}
3257 ins_encode %{
3258 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
3259 __ sve_fminv(as_FloatRegister($dst$$reg), __ D, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
3260 __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3261 %}
3262 ins_pipe(pipe_slow);
3263 %}
3264
3265 // vector min reduction - predicated
3266
3267 instruct reduce_minI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp,
3268 pRegGov pg, rFlagsReg cr) %{
3269 predicate(UseSVE > 0 &&
3270 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
3271 is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
3272 match(Set dst (MinReductionV (Binary src1 src2) pg));
3273 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3274 ins_cost(SVE_COST);
3275 format %{ "sve_reduce_minI $dst, $src1, $pg, $src2\t# minI reduction predicated (sve)" %}
3276 ins_encode %{
3277 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
3278 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
3279 $src1$$Register, as_FloatRegister($src2$$reg),
3280 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
3281 %}
3282 ins_pipe(pipe_slow);
3283 %}
3284
3285 instruct reduce_minL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp,
3286 pRegGov pg, rFlagsReg cr) %{
3287 predicate(UseSVE > 0 &&
3288 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
3289 match(Set dst (MinReductionV (Binary src1 src2) pg));
3290 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
3291 ins_cost(SVE_COST);
3292 format %{ "sve_reduce_minL $dst, $src1, $pg, $src2\t# minL reduction predicated (sve)" %}
3293 ins_encode %{
3294 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
3295 $src1$$Register, as_FloatRegister($src2$$reg),
3296 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
3297 %}
3298 ins_pipe(pipe_slow);
3299 %}
3300
3301 instruct reduce_minF_masked(vRegF dst, vRegF src1, vReg src2, pRegGov pg) %{
3302 predicate(UseSVE > 0 &&
3303 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
3304 match(Set dst (MinReductionV (Binary src1 src2) pg));
3305 ins_cost(SVE_COST);
3306 effect(TEMP_DEF dst);
3307 format %{ "sve_reduce_minF $dst, $src1, $pg, $src2\t# minF reduction predicated (sve)" %}
3308 ins_encode %{
3309 __ sve_fminv(as_FloatRegister($dst$$reg), __ S, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
3310 __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3311 %}
3312 ins_pipe(pipe_slow);
3313 %}
3314
3315 instruct reduce_minD_masked(vRegD dst, vRegD src1, vReg src2, pRegGov pg) %{
3316 predicate(UseSVE > 0 &&
3317 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
3318 match(Set dst (MinReductionV (Binary src1 src2) pg));
3319 ins_cost(SVE_COST);
3320 effect(TEMP_DEF dst);
3321 format %{ "sve_reduce_minD $dst, $src1, $pg, $src2\t# minD reduction predicated (sve)" %}
3322 ins_encode %{
3323 __ sve_fminv(as_FloatRegister($dst$$reg), __ D, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
3324 __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
3325 %}
3326 ins_pipe(pipe_slow);
3327 %}
3328
3329 // vector Math.rint, floor, ceil
3330
3331 instruct vroundD(vReg dst, vReg src, immI rmode) %{
3332 predicate(UseSVE > 0 &&
3333 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
3334 match(Set dst (RoundDoubleModeV src rmode));
3335 format %{ "sve_frint $dst, $src, $rmode\t# vector (sve) (D)" %}
3336 ins_encode %{
3337 switch ($rmode$$constant) {
3338 case RoundDoubleModeNode::rmode_rint:
3339 __ sve_frintn(as_FloatRegister($dst$$reg), __ D,
3340 ptrue, as_FloatRegister($src$$reg));
3341 break;
3342 case RoundDoubleModeNode::rmode_floor:
3343 __ sve_frintm(as_FloatRegister($dst$$reg), __ D,
3344 ptrue, as_FloatRegister($src$$reg));
3345 break;
3346 case RoundDoubleModeNode::rmode_ceil:
3347 __ sve_frintp(as_FloatRegister($dst$$reg), __ D,
3348 ptrue, as_FloatRegister($src$$reg));
3349 break;
3350 }
3351 %}
3352 ins_pipe(pipe_slow);
3353 %}
3354
3355 instruct vroundFtoI(vReg dst, vReg src, vReg tmp1, vReg tmp2, vReg tmp3, pRegGov ptmp)
3356 %{
3357 predicate(UseSVE > 0);
3358 match(Set dst (RoundVF src));
3359 effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP ptmp);
3360 format %{ "sve_vround $dst, S, $src\t# round F to I vector" %}
3361 ins_encode %{
3362 BasicType bt = Matcher::vector_element_basic_type(this);
3363 int vlen = Matcher::vector_length_in_bytes(this);
3364 if (vlen > 16) {
3365 __ vector_round_sve(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3366 as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg),
3367 as_PRegister($ptmp$$reg), __ S);
3368 } else {
3369 __ vector_round_neon(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3370 as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg),
3371 as_FloatRegister($tmp3$$reg),
3372 __ esize2arrangement(type2aelembytes(bt),
3373 /*isQ*/ vlen == 16));
3374 }
3375 %}
3376 ins_pipe(pipe_class_default);
3377 %}
3378
3379 instruct vroundDtoL(vReg dst, vReg src, vReg tmp1, vReg tmp2, vReg tmp3, pRegGov ptmp)
3380 %{
3381 predicate(UseSVE > 0);
3382 match(Set dst (RoundVD src));
3383 effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP ptmp);
3384 format %{ "sve_vround $dst, D, $src\t# round D to L vector" %}
3385 ins_encode %{
3386 BasicType bt = Matcher::vector_element_basic_type(this);
3387 int vlen = Matcher::vector_length_in_bytes(this);
3388 if (vlen > 16) {
3389 __ vector_round_sve(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3390 as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg),
3391 as_PRegister($ptmp$$reg), __ D);
3392 } else {
3393 __ vector_round_neon(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3394 as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg),
3395 as_FloatRegister($tmp3$$reg),
3396 __ esize2arrangement(type2aelembytes(bt),
3397 /*isQ*/ vlen == 16));
3398 }
3399 %}
3400 ins_pipe(pipe_class_default);
3401 %}
3402
3403 // vector replicate
3404
3405 instruct replicateB(vReg dst, iRegIorL2I src) %{
3406 predicate(UseSVE > 0);
3407 match(Set dst (ReplicateB src));
3408 ins_cost(SVE_COST);
3409 format %{ "sve_dup $dst, $src\t# vector (sve) (B)" %}
3410 ins_encode %{
3411 __ sve_dup(as_FloatRegister($dst$$reg), __ B, as_Register($src$$reg));
3412 %}
3413 ins_pipe(pipe_slow);
3414 %}
3415
3416 instruct replicateS(vReg dst, iRegIorL2I src) %{
3417 predicate(UseSVE > 0);
3418 match(Set dst (ReplicateS src));
3419 ins_cost(SVE_COST);
3420 format %{ "sve_dup $dst, $src\t# vector (sve) (H)" %}
3421 ins_encode %{
3422 __ sve_dup(as_FloatRegister($dst$$reg), __ H, as_Register($src$$reg));
3423 %}
3424 ins_pipe(pipe_slow);
3425 %}
3426
3427 instruct replicateI(vReg dst, iRegIorL2I src) %{
3428 predicate(UseSVE > 0);
3429 match(Set dst (ReplicateI src));
3430 ins_cost(SVE_COST);
3431 format %{ "sve_dup $dst, $src\t# vector (sve) (S)" %}
3432 ins_encode %{
3433 __ sve_dup(as_FloatRegister($dst$$reg), __ S, as_Register($src$$reg));
3434 %}
3435 ins_pipe(pipe_slow);
3436 %}
3437
3438 instruct replicateL(vReg dst, iRegL src) %{
3439 predicate(UseSVE > 0);
3440 match(Set dst (ReplicateL src));
3441 ins_cost(SVE_COST);
3442 format %{ "sve_dup $dst, $src\t# vector (sve) (D)" %}
3443 ins_encode %{
3444 __ sve_dup(as_FloatRegister($dst$$reg), __ D, as_Register($src$$reg));
3445 %}
3446 ins_pipe(pipe_slow);
3447 %}
3448
3449 instruct replicateB_imm8(vReg dst, immI8 con) %{
3450 predicate(UseSVE > 0);
3451 match(Set dst (ReplicateB con));
3452 ins_cost(SVE_COST);
3453 format %{ "sve_dup $dst, $con\t# vector (sve) (B)" %}
3454 ins_encode %{
3455 __ sve_dup(as_FloatRegister($dst$$reg), __ B, $con$$constant);
3456 %}
3457 ins_pipe(pipe_slow);
3458 %}
3459
3460 instruct replicateS_imm8(vReg dst, immI8_shift8 con) %{
3461 predicate(UseSVE > 0);
3462 match(Set dst (ReplicateS con));
3463 ins_cost(SVE_COST);
3464 format %{ "sve_dup $dst, $con\t# vector (sve) (H)" %}
3465 ins_encode %{
3466 __ sve_dup(as_FloatRegister($dst$$reg), __ H, $con$$constant);
3467 %}
3468 ins_pipe(pipe_slow);
3469 %}
3470
3471 instruct replicateI_imm8(vReg dst, immI8_shift8 con) %{
3472 predicate(UseSVE > 0);
3473 match(Set dst (ReplicateI con));
3474 ins_cost(SVE_COST);
3475 format %{ "sve_dup $dst, $con\t# vector (sve) (S)" %}
3476 ins_encode %{
3477 __ sve_dup(as_FloatRegister($dst$$reg), __ S, $con$$constant);
3478 %}
3479 ins_pipe(pipe_slow);
3480 %}
3481
3482 instruct replicateL_imm8(vReg dst, immL8_shift8 con) %{
3483 predicate(UseSVE > 0);
3484 match(Set dst (ReplicateL con));
3485 ins_cost(SVE_COST);
3486 format %{ "sve_dup $dst, $con\t# vector (sve) (D)" %}
3487 ins_encode %{
3488 __ sve_dup(as_FloatRegister($dst$$reg), __ D, $con$$constant);
3489 %}
3490 ins_pipe(pipe_slow);
3491 %}
3492
3493 instruct replicateF(vReg dst, vRegF src) %{
3494 predicate(UseSVE > 0);
3495 match(Set dst (ReplicateF src));
3496 ins_cost(SVE_COST);
3497 format %{ "sve_cpy $dst, $src\t# vector (sve) (S)" %}
3498 ins_encode %{
3499 __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
3500 ptrue, as_FloatRegister($src$$reg));
3501 %}
3502 ins_pipe(pipe_slow);
3503 %}
3504
3505 instruct replicateD(vReg dst, vRegD src) %{
3506 predicate(UseSVE > 0);
3507 match(Set dst (ReplicateD src));
3508 ins_cost(SVE_COST);
3509 format %{ "sve_cpy $dst, $src\t# vector (sve) (D)" %}
3510 ins_encode %{
3511 __ sve_cpy(as_FloatRegister($dst$$reg), __ D,
3512 ptrue, as_FloatRegister($src$$reg));
3513 %}
3514 ins_pipe(pipe_slow);
3515 %}
3516
3517 // vector shift
3518
3519 instruct vasrB(vReg dst, vReg shift) %{
3520 predicate(UseSVE > 0);
3521 match(Set dst (RShiftVB dst shift));
3522 ins_cost(SVE_COST);
3523 format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (B)" %}
3524 ins_encode %{
3525 __ sve_asr(as_FloatRegister($dst$$reg), __ B,
3526 ptrue, as_FloatRegister($shift$$reg));
3527 %}
3528 ins_pipe(pipe_slow);
3529 %}
3530
3531 instruct vasrS(vReg dst, vReg shift) %{
3532 predicate(UseSVE > 0);
3533 match(Set dst (RShiftVS dst shift));
3534 ins_cost(SVE_COST);
3535 format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (H)" %}
3536 ins_encode %{
3537 __ sve_asr(as_FloatRegister($dst$$reg), __ H,
3538 ptrue, as_FloatRegister($shift$$reg));
3539 %}
3540 ins_pipe(pipe_slow);
3541 %}
3542
3543 instruct vasrI(vReg dst, vReg shift) %{
3544 predicate(UseSVE > 0);
3545 match(Set dst (RShiftVI dst shift));
3546 ins_cost(SVE_COST);
3547 format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (S)" %}
3548 ins_encode %{
3549 __ sve_asr(as_FloatRegister($dst$$reg), __ S,
3550 ptrue, as_FloatRegister($shift$$reg));
3551 %}
3552 ins_pipe(pipe_slow);
3553 %}
3554
3555 instruct vasrL(vReg dst, vReg shift) %{
3556 predicate(UseSVE > 0);
3557 match(Set dst (RShiftVL dst shift));
3558 ins_cost(SVE_COST);
3559 format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (D)" %}
3560 ins_encode %{
3561 __ sve_asr(as_FloatRegister($dst$$reg), __ D,
3562 ptrue, as_FloatRegister($shift$$reg));
3563 %}
3564 ins_pipe(pipe_slow);
3565 %}
3566
3567 instruct vlslB(vReg dst, vReg shift) %{
3568 predicate(UseSVE > 0);
3569 match(Set dst (LShiftVB dst shift));
3570 ins_cost(SVE_COST);
3571 format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (B)" %}
3572 ins_encode %{
3573 __ sve_lsl(as_FloatRegister($dst$$reg), __ B,
3574 ptrue, as_FloatRegister($shift$$reg));
3575 %}
3576 ins_pipe(pipe_slow);
3577 %}
3578
3579 instruct vlslS(vReg dst, vReg shift) %{
3580 predicate(UseSVE > 0);
3581 match(Set dst (LShiftVS dst shift));
3582 ins_cost(SVE_COST);
3583 format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (H)" %}
3584 ins_encode %{
3585 __ sve_lsl(as_FloatRegister($dst$$reg), __ H,
3586 ptrue, as_FloatRegister($shift$$reg));
3587 %}
3588 ins_pipe(pipe_slow);
3589 %}
3590
3591 instruct vlslI(vReg dst, vReg shift) %{
3592 predicate(UseSVE > 0);
3593 match(Set dst (LShiftVI dst shift));
3594 ins_cost(SVE_COST);
3595 format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (S)" %}
3596 ins_encode %{
3597 __ sve_lsl(as_FloatRegister($dst$$reg), __ S,
3598 ptrue, as_FloatRegister($shift$$reg));
3599 %}
3600 ins_pipe(pipe_slow);
3601 %}
3602
3603 instruct vlslL(vReg dst, vReg shift) %{
3604 predicate(UseSVE > 0);
3605 match(Set dst (LShiftVL dst shift));
3606 ins_cost(SVE_COST);
3607 format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (D)" %}
3608 ins_encode %{
3609 __ sve_lsl(as_FloatRegister($dst$$reg), __ D,
3610 ptrue, as_FloatRegister($shift$$reg));
3611 %}
3612 ins_pipe(pipe_slow);
3613 %}
3614
3615 instruct vlsrB(vReg dst, vReg shift) %{
3616 predicate(UseSVE > 0);
3617 match(Set dst (URShiftVB dst shift));
3618 ins_cost(SVE_COST);
3619 format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (B)" %}
3620 ins_encode %{
3621 __ sve_lsr(as_FloatRegister($dst$$reg), __ B,
3622 ptrue, as_FloatRegister($shift$$reg));
3623 %}
3624 ins_pipe(pipe_slow);
3625 %}
3626
3627 instruct vlsrS(vReg dst, vReg shift) %{
3628 predicate(UseSVE > 0);
3629 match(Set dst (URShiftVS dst shift));
3630 ins_cost(SVE_COST);
3631 format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (H)" %}
3632 ins_encode %{
3633 __ sve_lsr(as_FloatRegister($dst$$reg), __ H,
3634 ptrue, as_FloatRegister($shift$$reg));
3635 %}
3636 ins_pipe(pipe_slow);
3637 %}
3638
3639 instruct vlsrI(vReg dst, vReg shift) %{
3640 predicate(UseSVE > 0);
3641 match(Set dst (URShiftVI dst shift));
3642 ins_cost(SVE_COST);
3643 format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (S)" %}
3644 ins_encode %{
3645 __ sve_lsr(as_FloatRegister($dst$$reg), __ S,
3646 ptrue, as_FloatRegister($shift$$reg));
3647 %}
3648 ins_pipe(pipe_slow);
3649 %}
3650
3651 instruct vlsrL(vReg dst, vReg shift) %{
3652 predicate(UseSVE > 0);
3653 match(Set dst (URShiftVL dst shift));
3654 ins_cost(SVE_COST);
3655 format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (D)" %}
3656 ins_encode %{
3657 __ sve_lsr(as_FloatRegister($dst$$reg), __ D,
3658 ptrue, as_FloatRegister($shift$$reg));
3659 %}
3660 ins_pipe(pipe_slow);
3661 %}
3662
3663 instruct vasrB_imm(vReg dst, vReg src, immI shift) %{
3664 predicate(UseSVE > 0);
3665 match(Set dst (RShiftVB src (RShiftCntV shift)));
3666 ins_cost(SVE_COST);
3667 format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (B)" %}
3668 ins_encode %{
3669 int con = (int)$shift$$constant;
3670 if (con == 0) {
3671 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3672 as_FloatRegister($src$$reg));
3673 return;
3674 }
3675 if (con >= 8) con = 7;
3676 __ sve_asr(as_FloatRegister($dst$$reg), __ B,
3677 as_FloatRegister($src$$reg), con);
3678 %}
3679 ins_pipe(pipe_slow);
3680 %}
3681
3682 instruct vasrS_imm(vReg dst, vReg src, immI shift) %{
3683 predicate(UseSVE > 0);
3684 match(Set dst (RShiftVS src (RShiftCntV shift)));
3685 ins_cost(SVE_COST);
3686 format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (H)" %}
3687 ins_encode %{
3688 int con = (int)$shift$$constant;
3689 if (con == 0) {
3690 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3691 as_FloatRegister($src$$reg));
3692 return;
3693 }
3694 if (con >= 16) con = 15;
3695 __ sve_asr(as_FloatRegister($dst$$reg), __ H,
3696 as_FloatRegister($src$$reg), con);
3697 %}
3698 ins_pipe(pipe_slow);
3699 %}
3700
3701 instruct vasrI_imm(vReg dst, vReg src, immI shift) %{
3702 predicate(UseSVE > 0);
3703 match(Set dst (RShiftVI src (RShiftCntV shift)));
3704 ins_cost(SVE_COST);
3705 format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (S)" %}
3706 ins_encode %{
3707 int con = (int)$shift$$constant;
3708 if (con == 0) {
3709 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3710 as_FloatRegister($src$$reg));
3711 return;
3712 }
3713 __ sve_asr(as_FloatRegister($dst$$reg), __ S,
3714 as_FloatRegister($src$$reg), con);
3715 %}
3716 ins_pipe(pipe_slow);
3717 %}
3718
3719 instruct vasrL_imm(vReg dst, vReg src, immI shift) %{
3720 predicate(UseSVE > 0);
3721 match(Set dst (RShiftVL src (RShiftCntV shift)));
3722 ins_cost(SVE_COST);
3723 format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (D)" %}
3724 ins_encode %{
3725 int con = (int)$shift$$constant;
3726 if (con == 0) {
3727 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3728 as_FloatRegister($src$$reg));
3729 return;
3730 }
3731 __ sve_asr(as_FloatRegister($dst$$reg), __ D,
3732 as_FloatRegister($src$$reg), con);
3733 %}
3734 ins_pipe(pipe_slow);
3735 %}
3736
3737 instruct vlsrB_imm(vReg dst, vReg src, immI shift) %{
3738 predicate(UseSVE > 0);
3739 match(Set dst (URShiftVB src (RShiftCntV shift)));
3740 ins_cost(SVE_COST);
3741 format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (B)" %}
3742 ins_encode %{
3743 int con = (int)$shift$$constant;
3744 if (con == 0) {
3745 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3746 as_FloatRegister($src$$reg));
3747 return;
3748 }
3749 if (con >= 8) {
3750 __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3751 as_FloatRegister($src$$reg));
3752 return;
3753 }
3754 __ sve_lsr(as_FloatRegister($dst$$reg), __ B,
3755 as_FloatRegister($src$$reg), con);
3756 %}
3757 ins_pipe(pipe_slow);
3758 %}
3759
3760 instruct vlsrS_imm(vReg dst, vReg src, immI shift) %{
3761 predicate(UseSVE > 0);
3762 match(Set dst (URShiftVS src (RShiftCntV shift)));
3763 ins_cost(SVE_COST);
3764 format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (H)" %}
3765 ins_encode %{
3766 int con = (int)$shift$$constant;
3767 if (con == 0) {
3768 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3769 as_FloatRegister($src$$reg));
3770 return;
3771 }
3772 if (con >= 16) {
3773 __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3774 as_FloatRegister($src$$reg));
3775 return;
3776 }
3777 __ sve_lsr(as_FloatRegister($dst$$reg), __ H,
3778 as_FloatRegister($src$$reg), con);
3779 %}
3780 ins_pipe(pipe_slow);
3781 %}
3782
3783 instruct vlsrI_imm(vReg dst, vReg src, immI shift) %{
3784 predicate(UseSVE > 0);
3785 match(Set dst (URShiftVI src (RShiftCntV shift)));
3786 ins_cost(SVE_COST);
3787 format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (S)" %}
3788 ins_encode %{
3789 int con = (int)$shift$$constant;
3790 if (con == 0) {
3791 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3792 as_FloatRegister($src$$reg));
3793 return;
3794 }
3795 __ sve_lsr(as_FloatRegister($dst$$reg), __ S,
3796 as_FloatRegister($src$$reg), con);
3797 %}
3798 ins_pipe(pipe_slow);
3799 %}
3800
3801 instruct vlsrL_imm(vReg dst, vReg src, immI shift) %{
3802 predicate(UseSVE > 0);
3803 match(Set dst (URShiftVL src (RShiftCntV shift)));
3804 ins_cost(SVE_COST);
3805 format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (D)" %}
3806 ins_encode %{
3807 int con = (int)$shift$$constant;
3808 if (con == 0) {
3809 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3810 as_FloatRegister($src$$reg));
3811 return;
3812 }
3813 __ sve_lsr(as_FloatRegister($dst$$reg), __ D,
3814 as_FloatRegister($src$$reg), con);
3815 %}
3816 ins_pipe(pipe_slow);
3817 %}
3818
3819 instruct vlslB_imm(vReg dst, vReg src, immI shift) %{
3820 predicate(UseSVE > 0);
3821 match(Set dst (LShiftVB src (LShiftCntV shift)));
3822 ins_cost(SVE_COST);
3823 format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (B)" %}
3824 ins_encode %{
3825 int con = (int)$shift$$constant;
3826 if (con >= 8) {
3827 __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3828 as_FloatRegister($src$$reg));
3829 return;
3830 }
3831 __ sve_lsl(as_FloatRegister($dst$$reg), __ B,
3832 as_FloatRegister($src$$reg), con);
3833 %}
3834 ins_pipe(pipe_slow);
3835 %}
3836
3837 instruct vlslS_imm(vReg dst, vReg src, immI shift) %{
3838 predicate(UseSVE > 0);
3839 match(Set dst (LShiftVS src (LShiftCntV shift)));
3840 ins_cost(SVE_COST);
3841 format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (H)" %}
3842 ins_encode %{
3843 int con = (int)$shift$$constant;
3844 if (con >= 16) {
3845 __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
3846 as_FloatRegister($src$$reg));
3847 return;
3848 }
3849 __ sve_lsl(as_FloatRegister($dst$$reg), __ H,
3850 as_FloatRegister($src$$reg), con);
3851 %}
3852 ins_pipe(pipe_slow);
3853 %}
3854
3855 instruct vlslI_imm(vReg dst, vReg src, immI shift) %{
3856 predicate(UseSVE > 0);
3857 match(Set dst (LShiftVI src (LShiftCntV shift)));
3858 ins_cost(SVE_COST);
3859 format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (S)" %}
3860 ins_encode %{
3861 int con = (int)$shift$$constant;
3862 __ sve_lsl(as_FloatRegister($dst$$reg), __ S,
3863 as_FloatRegister($src$$reg), con);
3864 %}
3865 ins_pipe(pipe_slow);
3866 %}
3867
3868 instruct vlslL_imm(vReg dst, vReg src, immI shift) %{
3869 predicate(UseSVE > 0);
3870 match(Set dst (LShiftVL src (LShiftCntV shift)));
3871 ins_cost(SVE_COST);
3872 format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (D)" %}
3873 ins_encode %{
3874 int con = (int)$shift$$constant;
3875 __ sve_lsl(as_FloatRegister($dst$$reg), __ D,
3876 as_FloatRegister($src$$reg), con);
3877 %}
3878 ins_pipe(pipe_slow);
3879 %}
3880
3881 instruct vshiftcntB(vReg dst, iRegIorL2I cnt) %{
3882 predicate(UseSVE > 0 &&
3883 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE));
3884 match(Set dst (LShiftCntV cnt));
3885 match(Set dst (RShiftCntV cnt));
3886 format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (B)" %}
3887 ins_encode %{
3888 __ sve_dup(as_FloatRegister($dst$$reg), __ B, as_Register($cnt$$reg));
3889 %}
3890 ins_pipe(pipe_slow);
3891 %}
3892
3893 instruct vshiftcntS(vReg dst, iRegIorL2I cnt) %{
3894 predicate(UseSVE > 0 &&
3895 (n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
3896 (n->bottom_type()->is_vect()->element_basic_type() == T_CHAR)));
3897 match(Set dst (LShiftCntV cnt));
3898 match(Set dst (RShiftCntV cnt));
3899 format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (H)" %}
3900 ins_encode %{
3901 __ sve_dup(as_FloatRegister($dst$$reg), __ H, as_Register($cnt$$reg));
3902 %}
3903 ins_pipe(pipe_slow);
3904 %}
3905
3906 instruct vshiftcntI(vReg dst, iRegIorL2I cnt) %{
3907 predicate(UseSVE > 0 &&
3908 (n->bottom_type()->is_vect()->element_basic_type() == T_INT));
3909 match(Set dst (LShiftCntV cnt));
3910 match(Set dst (RShiftCntV cnt));
3911 format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (S)" %}
3912 ins_encode %{
3913 __ sve_dup(as_FloatRegister($dst$$reg), __ S, as_Register($cnt$$reg));
3914 %}
3915 ins_pipe(pipe_slow);
3916 %}
3917
3918 instruct vshiftcntL(vReg dst, iRegIorL2I cnt) %{
3919 predicate(UseSVE > 0 &&
3920 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
3921 match(Set dst (LShiftCntV cnt));
3922 match(Set dst (RShiftCntV cnt));
3923 format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (D)" %}
3924 ins_encode %{
3925 __ sve_dup(as_FloatRegister($dst$$reg), __ D, as_Register($cnt$$reg));
3926 %}
3927 ins_pipe(pipe_slow);
3928 %}
3929
3930 // vector shift - predicated
3931
3932 instruct vasrB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3933 predicate(UseSVE > 0);
3934 match(Set dst_src1 (RShiftVB (Binary dst_src1 src2) pg));
3935 ins_cost(SVE_COST);
3936 format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
3937 ins_encode %{
3938 __ sve_asr(as_FloatRegister($dst_src1$$reg), __ B,
3939 as_PRegister($pg$$reg),
3940 as_FloatRegister($src2$$reg));
3941 %}
3942 ins_pipe(pipe_slow);
3943 %}
3944
3945 instruct vasrS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3946 predicate(UseSVE > 0);
3947 match(Set dst_src1 (RShiftVS (Binary dst_src1 src2) pg));
3948 ins_cost(SVE_COST);
3949 format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
3950 ins_encode %{
3951 __ sve_asr(as_FloatRegister($dst_src1$$reg), __ H,
3952 as_PRegister($pg$$reg),
3953 as_FloatRegister($src2$$reg));
3954 %}
3955 ins_pipe(pipe_slow);
3956 %}
3957
3958 instruct vasrI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3959 predicate(UseSVE > 0);
3960 match(Set dst_src1 (RShiftVI (Binary dst_src1 src2) pg));
3961 ins_cost(SVE_COST);
3962 format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
3963 ins_encode %{
3964 __ sve_asr(as_FloatRegister($dst_src1$$reg), __ S,
3965 as_PRegister($pg$$reg),
3966 as_FloatRegister($src2$$reg));
3967 %}
3968 ins_pipe(pipe_slow);
3969 %}
3970
3971 instruct vasrL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3972 predicate(UseSVE > 0);
3973 match(Set dst_src1 (RShiftVL (Binary dst_src1 src2) pg));
3974 ins_cost(SVE_COST);
3975 format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
3976 ins_encode %{
3977 __ sve_asr(as_FloatRegister($dst_src1$$reg), __ D,
3978 as_PRegister($pg$$reg),
3979 as_FloatRegister($src2$$reg));
3980 %}
3981 ins_pipe(pipe_slow);
3982 %}
3983
3984 instruct vlslB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3985 predicate(UseSVE > 0);
3986 match(Set dst_src1 (LShiftVB (Binary dst_src1 src2) pg));
3987 ins_cost(SVE_COST);
3988 format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
3989 ins_encode %{
3990 __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ B,
3991 as_PRegister($pg$$reg),
3992 as_FloatRegister($src2$$reg));
3993 %}
3994 ins_pipe(pipe_slow);
3995 %}
3996
3997 instruct vlslS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
3998 predicate(UseSVE > 0);
3999 match(Set dst_src1 (LShiftVS (Binary dst_src1 src2) pg));
4000 ins_cost(SVE_COST);
4001 format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
4002 ins_encode %{
4003 __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ H,
4004 as_PRegister($pg$$reg),
4005 as_FloatRegister($src2$$reg));
4006 %}
4007 ins_pipe(pipe_slow);
4008 %}
4009
4010 instruct vlslI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4011 predicate(UseSVE > 0);
4012 match(Set dst_src1 (LShiftVI (Binary dst_src1 src2) pg));
4013 ins_cost(SVE_COST);
4014 format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
4015 ins_encode %{
4016 __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ S,
4017 as_PRegister($pg$$reg),
4018 as_FloatRegister($src2$$reg));
4019 %}
4020 ins_pipe(pipe_slow);
4021 %}
4022
4023 instruct vlslL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4024 predicate(UseSVE > 0);
4025 match(Set dst_src1 (LShiftVL (Binary dst_src1 src2) pg));
4026 ins_cost(SVE_COST);
4027 format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
4028 ins_encode %{
4029 __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ D,
4030 as_PRegister($pg$$reg),
4031 as_FloatRegister($src2$$reg));
4032 %}
4033 ins_pipe(pipe_slow);
4034 %}
4035
4036 instruct vlsrB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4037 predicate(UseSVE > 0);
4038 match(Set dst_src1 (URShiftVB (Binary dst_src1 src2) pg));
4039 ins_cost(SVE_COST);
4040 format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
4041 ins_encode %{
4042 __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ B,
4043 as_PRegister($pg$$reg),
4044 as_FloatRegister($src2$$reg));
4045 %}
4046 ins_pipe(pipe_slow);
4047 %}
4048
4049 instruct vlsrS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4050 predicate(UseSVE > 0);
4051 match(Set dst_src1 (URShiftVS (Binary dst_src1 src2) pg));
4052 ins_cost(SVE_COST);
4053 format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
4054 ins_encode %{
4055 __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ H,
4056 as_PRegister($pg$$reg),
4057 as_FloatRegister($src2$$reg));
4058 %}
4059 ins_pipe(pipe_slow);
4060 %}
4061
4062 instruct vlsrI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4063 predicate(UseSVE > 0);
4064 match(Set dst_src1 (URShiftVI (Binary dst_src1 src2) pg));
4065 ins_cost(SVE_COST);
4066 format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
4067 ins_encode %{
4068 __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ S,
4069 as_PRegister($pg$$reg),
4070 as_FloatRegister($src2$$reg));
4071 %}
4072 ins_pipe(pipe_slow);
4073 %}
4074
4075 instruct vlsrL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4076 predicate(UseSVE > 0);
4077 match(Set dst_src1 (URShiftVL (Binary dst_src1 src2) pg));
4078 ins_cost(SVE_COST);
4079 format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
4080 ins_encode %{
4081 __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ D,
4082 as_PRegister($pg$$reg),
4083 as_FloatRegister($src2$$reg));
4084 %}
4085 ins_pipe(pipe_slow);
4086 %}
4087
4088 instruct vasrB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4089 predicate(UseSVE > 0);
4090 match(Set dst_src (RShiftVB (Binary dst_src (RShiftCntV shift)) pg));
4091 ins_cost(SVE_COST);
4092 format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %}
4093 ins_encode %{
4094 int con = (int)$shift$$constant;
4095 assert(con > 0 && con < 8, "invalid shift immediate");
4096 __ sve_asr(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con);
4097 %}
4098 ins_pipe(pipe_slow);
4099 %}
4100
4101 instruct vasrS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4102 predicate(UseSVE > 0);
4103 match(Set dst_src (RShiftVS (Binary dst_src (RShiftCntV shift)) pg));
4104 ins_cost(SVE_COST);
4105 format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %}
4106 ins_encode %{
4107 int con = (int)$shift$$constant;
4108 assert(con > 0 && con < 16, "invalid shift immediate");
4109 __ sve_asr(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con);
4110 %}
4111 ins_pipe(pipe_slow);
4112 %}
4113
4114 instruct vasrI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4115 predicate(UseSVE > 0);
4116 match(Set dst_src (RShiftVI (Binary dst_src (RShiftCntV shift)) pg));
4117 ins_cost(SVE_COST);
4118 format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %}
4119 ins_encode %{
4120 int con = (int)$shift$$constant;
4121 assert(con > 0 && con < 32, "invalid shift immediate");
4122 __ sve_asr(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con);
4123 %}
4124 ins_pipe(pipe_slow);
4125 %}
4126
4127 instruct vasrL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4128 predicate(UseSVE > 0);
4129 match(Set dst_src (RShiftVL (Binary dst_src (RShiftCntV shift)) pg));
4130 ins_cost(SVE_COST);
4131 format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %}
4132 ins_encode %{
4133 int con = (int)$shift$$constant;
4134 assert(con > 0 && con < 64, "invalid shift immediate");
4135 __ sve_asr(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con);
4136 %}
4137 ins_pipe(pipe_slow);
4138 %}
4139
4140 instruct vlsrB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4141 predicate(UseSVE > 0);
4142 match(Set dst_src (URShiftVB (Binary dst_src (RShiftCntV shift)) pg));
4143 ins_cost(SVE_COST);
4144 format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %}
4145 ins_encode %{
4146 int con = (int)$shift$$constant;
4147 assert(con > 0 && con < 8, "invalid shift immediate");
4148 __ sve_lsr(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con);
4149 %}
4150 ins_pipe(pipe_slow);
4151 %}
4152
4153 instruct vlsrS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4154 predicate(UseSVE > 0);
4155 match(Set dst_src (URShiftVS (Binary dst_src (RShiftCntV shift)) pg));
4156 ins_cost(SVE_COST);
4157 format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %}
4158 ins_encode %{
4159 int con = (int)$shift$$constant;
4160 assert(con > 0 && con < 16, "invalid shift immediate");
4161 __ sve_lsr(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con);
4162 %}
4163 ins_pipe(pipe_slow);
4164 %}
4165
4166 instruct vlsrI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4167 predicate(UseSVE > 0);
4168 match(Set dst_src (URShiftVI (Binary dst_src (RShiftCntV shift)) pg));
4169 ins_cost(SVE_COST);
4170 format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %}
4171 ins_encode %{
4172 int con = (int)$shift$$constant;
4173 assert(con > 0 && con < 32, "invalid shift immediate");
4174 __ sve_lsr(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con);
4175 %}
4176 ins_pipe(pipe_slow);
4177 %}
4178
4179 instruct vlsrL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4180 predicate(UseSVE > 0);
4181 match(Set dst_src (URShiftVL (Binary dst_src (RShiftCntV shift)) pg));
4182 ins_cost(SVE_COST);
4183 format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %}
4184 ins_encode %{
4185 int con = (int)$shift$$constant;
4186 assert(con > 0 && con < 64, "invalid shift immediate");
4187 __ sve_lsr(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con);
4188 %}
4189 ins_pipe(pipe_slow);
4190 %}
4191
4192 instruct vlslB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4193 predicate(UseSVE > 0);
4194 match(Set dst_src (LShiftVB (Binary dst_src (LShiftCntV shift)) pg));
4195 ins_cost(SVE_COST);
4196 format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %}
4197 ins_encode %{
4198 int con = (int)$shift$$constant;
4199 assert(con >= 0 && con < 8, "invalid shift immediate");
4200 __ sve_lsl(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con);
4201 %}
4202 ins_pipe(pipe_slow);
4203 %}
4204
4205 instruct vlslS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4206 predicate(UseSVE > 0);
4207 match(Set dst_src (LShiftVS (Binary dst_src (LShiftCntV shift)) pg));
4208 ins_cost(SVE_COST);
4209 format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %}
4210 ins_encode %{
4211 int con = (int)$shift$$constant;
4212 assert(con >= 0 && con < 16, "invalid shift immediate");
4213 __ sve_lsl(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con);
4214 %}
4215 ins_pipe(pipe_slow);
4216 %}
4217
4218 instruct vlslI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4219 predicate(UseSVE > 0);
4220 match(Set dst_src (LShiftVI (Binary dst_src (LShiftCntV shift)) pg));
4221 ins_cost(SVE_COST);
4222 format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %}
4223 ins_encode %{
4224 int con = (int)$shift$$constant;
4225 assert(con >= 0 && con < 32, "invalid shift immediate");
4226 __ sve_lsl(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con);
4227 %}
4228 ins_pipe(pipe_slow);
4229 %}
4230
4231 instruct vlslL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
4232 predicate(UseSVE > 0);
4233 match(Set dst_src (LShiftVL (Binary dst_src (LShiftCntV shift)) pg));
4234 ins_cost(SVE_COST);
4235 format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %}
4236 ins_encode %{
4237 int con = (int)$shift$$constant;
4238 assert(con >= 0 && con < 64, "invalid shift immediate");
4239 __ sve_lsl(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con);
4240 %}
4241 ins_pipe(pipe_slow);
4242 %}
4243
4244 // vector sqrt
4245
4246 instruct vsqrtF(vReg dst, vReg src) %{
4247 predicate(UseSVE > 0 &&
4248 !n->as_Vector()->is_predicated_vector());
4249 match(Set dst (SqrtVF src));
4250 ins_cost(SVE_COST);
4251 format %{ "sve_fsqrt $dst, $src\t# vector (sve) (S)" %}
4252 ins_encode %{
4253 __ sve_fsqrt(as_FloatRegister($dst$$reg), __ S,
4254 ptrue, as_FloatRegister($src$$reg));
4255 %}
4256 ins_pipe(pipe_slow);
4257 %}
4258
4259 instruct vsqrtD(vReg dst, vReg src) %{
4260 predicate(UseSVE > 0 &&
4261 !n->as_Vector()->is_predicated_vector());
4262 match(Set dst (SqrtVD src));
4263 ins_cost(SVE_COST);
4264 format %{ "sve_fsqrt $dst, $src\t# vector (sve) (D)" %}
4265 ins_encode %{
4266 __ sve_fsqrt(as_FloatRegister($dst$$reg), __ D,
4267 ptrue, as_FloatRegister($src$$reg));
4268 %}
4269 ins_pipe(pipe_slow);
4270 %}
4271
4272 // vector sqrt - predicated
4273
4274 instruct vsqrtF_masked(vReg dst_src, pRegGov pg) %{
4275 predicate(UseSVE > 0);
4276 match(Set dst_src (SqrtVF dst_src pg));
4277 ins_cost(SVE_COST);
4278 format %{ "sve_fsqrt $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
4279 ins_encode %{
4280 __ sve_fsqrt(as_FloatRegister($dst_src$$reg), __ S,
4281 as_PRegister($pg$$reg),
4282 as_FloatRegister($dst_src$$reg));
4283 %}
4284 ins_pipe(pipe_slow);
4285 %}
4286
4287 instruct vsqrtD_masked(vReg dst_src, pRegGov pg) %{
4288 predicate(UseSVE > 0);
4289 match(Set dst_src (SqrtVD dst_src pg));
4290 ins_cost(SVE_COST);
4291 format %{ "sve_fsqrt $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
4292 ins_encode %{
4293 __ sve_fsqrt(as_FloatRegister($dst_src$$reg), __ D,
4294 as_PRegister($pg$$reg),
4295 as_FloatRegister($dst_src$$reg));
4296 %}
4297 ins_pipe(pipe_slow);
4298 %}
4299
4300 // vector sub
4301
4302 instruct vsubB(vReg dst, vReg src1, vReg src2) %{
4303 predicate(UseSVE > 0);
4304 match(Set dst (SubVB src1 src2));
4305 ins_cost(SVE_COST);
4306 format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (B)" %}
4307 ins_encode %{
4308 __ sve_sub(as_FloatRegister($dst$$reg), __ B,
4309 as_FloatRegister($src1$$reg),
4310 as_FloatRegister($src2$$reg));
4311 %}
4312 ins_pipe(pipe_slow);
4313 %}
4314
4315 instruct vsubS(vReg dst, vReg src1, vReg src2) %{
4316 predicate(UseSVE > 0);
4317 match(Set dst (SubVS src1 src2));
4318 ins_cost(SVE_COST);
4319 format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (H)" %}
4320 ins_encode %{
4321 __ sve_sub(as_FloatRegister($dst$$reg), __ H,
4322 as_FloatRegister($src1$$reg),
4323 as_FloatRegister($src2$$reg));
4324 %}
4325 ins_pipe(pipe_slow);
4326 %}
4327
4328 instruct vsubI(vReg dst, vReg src1, vReg src2) %{
4329 predicate(UseSVE > 0);
4330 match(Set dst (SubVI src1 src2));
4331 ins_cost(SVE_COST);
4332 format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (S)" %}
4333 ins_encode %{
4334 __ sve_sub(as_FloatRegister($dst$$reg), __ S,
4335 as_FloatRegister($src1$$reg),
4336 as_FloatRegister($src2$$reg));
4337 %}
4338 ins_pipe(pipe_slow);
4339 %}
4340
4341 instruct vsubL(vReg dst, vReg src1, vReg src2) %{
4342 predicate(UseSVE > 0);
4343 match(Set dst (SubVL src1 src2));
4344 ins_cost(SVE_COST);
4345 format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (D)" %}
4346 ins_encode %{
4347 __ sve_sub(as_FloatRegister($dst$$reg), __ D,
4348 as_FloatRegister($src1$$reg),
4349 as_FloatRegister($src2$$reg));
4350 %}
4351 ins_pipe(pipe_slow);
4352 %}
4353
4354 instruct vsubF(vReg dst, vReg src1, vReg src2) %{
4355 predicate(UseSVE > 0);
4356 match(Set dst (SubVF src1 src2));
4357 ins_cost(SVE_COST);
4358 format %{ "sve_fsub $dst, $src1, $src2\t # vector (sve) (S)" %}
4359 ins_encode %{
4360 __ sve_fsub(as_FloatRegister($dst$$reg), __ S,
4361 as_FloatRegister($src1$$reg),
4362 as_FloatRegister($src2$$reg));
4363 %}
4364 ins_pipe(pipe_slow);
4365 %}
4366
4367 instruct vsubD(vReg dst, vReg src1, vReg src2) %{
4368 predicate(UseSVE > 0);
4369 match(Set dst (SubVD src1 src2));
4370 ins_cost(SVE_COST);
4371 format %{ "sve_fsub $dst, $src1, $src2\t # vector (sve) (D)" %}
4372 ins_encode %{
4373 __ sve_fsub(as_FloatRegister($dst$$reg), __ D,
4374 as_FloatRegister($src1$$reg),
4375 as_FloatRegister($src2$$reg));
4376 %}
4377 ins_pipe(pipe_slow);
4378 %}
4379
4380 // vector sub - predicated
4381
4382 instruct vsubB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4383 predicate(UseSVE > 0);
4384 match(Set dst_src1 (SubVB (Binary dst_src1 src2) pg));
4385 ins_cost(SVE_COST);
4386 format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
4387 ins_encode %{
4388 __ sve_sub(as_FloatRegister($dst_src1$$reg), __ B,
4389 as_PRegister($pg$$reg),
4390 as_FloatRegister($src2$$reg));
4391 %}
4392 ins_pipe(pipe_slow);
4393 %}
4394
4395 instruct vsubS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4396 predicate(UseSVE > 0);
4397 match(Set dst_src1 (SubVS (Binary dst_src1 src2) pg));
4398 ins_cost(SVE_COST);
4399 format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
4400 ins_encode %{
4401 __ sve_sub(as_FloatRegister($dst_src1$$reg), __ H,
4402 as_PRegister($pg$$reg),
4403 as_FloatRegister($src2$$reg));
4404 %}
4405 ins_pipe(pipe_slow);
4406 %}
4407
4408 instruct vsubI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4409 predicate(UseSVE > 0);
4410 match(Set dst_src1 (SubVI (Binary dst_src1 src2) pg));
4411 ins_cost(SVE_COST);
4412 format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
4413 ins_encode %{
4414 __ sve_sub(as_FloatRegister($dst_src1$$reg), __ S,
4415 as_PRegister($pg$$reg),
4416 as_FloatRegister($src2$$reg));
4417 %}
4418 ins_pipe(pipe_slow);
4419 %}
4420
4421 instruct vsubL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4422 predicate(UseSVE > 0);
4423 match(Set dst_src1 (SubVL (Binary dst_src1 src2) pg));
4424 ins_cost(SVE_COST);
4425 format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
4426 ins_encode %{
4427 __ sve_sub(as_FloatRegister($dst_src1$$reg), __ D,
4428 as_PRegister($pg$$reg),
4429 as_FloatRegister($src2$$reg));
4430 %}
4431 ins_pipe(pipe_slow);
4432 %}
4433
4434 instruct vsubF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4435 predicate(UseSVE > 0);
4436 match(Set dst_src1 (SubVF (Binary dst_src1 src2) pg));
4437 ins_cost(SVE_COST);
4438 format %{ "sve_fsub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
4439 ins_encode %{
4440 __ sve_fsub(as_FloatRegister($dst_src1$$reg), __ S,
4441 as_PRegister($pg$$reg),
4442 as_FloatRegister($src2$$reg));
4443 %}
4444 ins_pipe(pipe_slow);
4445 %}
4446
4447 instruct vsubD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
4448 predicate(UseSVE > 0);
4449 match(Set dst_src1 (SubVD (Binary dst_src1 src2) pg));
4450 ins_cost(SVE_COST);
4451 format %{ "sve_fsub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
4452 ins_encode %{
4453 __ sve_fsub(as_FloatRegister($dst_src1$$reg), __ D,
4454 as_PRegister($pg$$reg),
4455 as_FloatRegister($src2$$reg));
4456 %}
4457 ins_pipe(pipe_slow);
4458 %}
4459
4460 // ------------------------------ Vector mask cast --------------------------
4461
4462 instruct vmaskcast(pRegGov dst_src) %{
4463 predicate(UseSVE > 0 &&
4464 n->bottom_type()->is_vect()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
4465 n->bottom_type()->is_vect()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
4466 match(Set dst_src (VectorMaskCast dst_src));
4467 ins_cost(0);
4468 format %{ "vmaskcast $dst_src\t# empty (sve)" %}
4469 ins_encode %{
4470 // empty
4471 %}
4472 ins_pipe(pipe_class_empty);
4473 %}
4474
4475 instruct vmaskcast_extend(pRegGov dst, pReg src)
4476 %{
4477 predicate(UseSVE > 0 &&
4478 (Matcher::vector_length_in_bytes(n) == 2 * Matcher::vector_length_in_bytes(n->in(1)) ||
4479 Matcher::vector_length_in_bytes(n) == 4 * Matcher::vector_length_in_bytes(n->in(1)) ||
4480 Matcher::vector_length_in_bytes(n) == 8 * Matcher::vector_length_in_bytes(n->in(1))));
4481 match(Set dst (VectorMaskCast src));
4482 ins_cost(SVE_COST * 3);
4483 format %{ "sve_vmaskcast_extend $dst, $src\t# extend predicate $src" %}
4484 ins_encode %{
4485 __ sve_vmaskcast_extend(as_PRegister($dst$$reg), as_PRegister($src$$reg),
4486 Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
4487 %}
4488 ins_pipe(pipe_slow);
4489 %}
4490
4491 instruct vmaskcast_narrow(pRegGov dst, pReg src)
4492 %{
4493 predicate(UseSVE > 0 &&
4494 (Matcher::vector_length_in_bytes(n) * 2 == Matcher::vector_length_in_bytes(n->in(1)) ||
4495 Matcher::vector_length_in_bytes(n) * 4 == Matcher::vector_length_in_bytes(n->in(1)) ||
4496 Matcher::vector_length_in_bytes(n) * 8 == Matcher::vector_length_in_bytes(n->in(1))));
4497 match(Set dst (VectorMaskCast src));
4498 ins_cost(SVE_COST * 3);
4499 format %{ "sve_vmaskcast_narrow $dst, $src\t# narrow predicate $src" %}
4500 ins_encode %{
4501 __ sve_vmaskcast_narrow(as_PRegister($dst$$reg), as_PRegister($src$$reg),
4502 Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
4503 %}
4504 ins_pipe(pipe_slow);
4505 %}
4506
4507 // ------------------------------ Vector cast -------------------------------
4508
4509 instruct vcvtBtoX_extend(vReg dst, vReg src)
4510 %{
4511 predicate(UseSVE > 0);
4512 match(Set dst (VectorCastB2X src));
4513 ins_cost(2 * SVE_COST);
4514 format %{ "sve_vectorcast_b2x $dst, $src\t# convert B to X vector (extend)" %}
4515 ins_encode %{
4516 BasicType to_bt = Matcher::vector_element_basic_type(this);
4517 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4518 __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ B);
4519 if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
4520 __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
4521 }
4522 %}
4523 ins_pipe(pipe_slow);
4524 %}
4525
4526 instruct vcvtStoB(vReg dst, vReg src, vReg tmp)
4527 %{
4528 predicate(UseSVE > 0 &&
4529 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
4530 match(Set dst (VectorCastS2X src));
4531 effect(TEMP tmp);
4532 ins_cost(2 * SVE_COST);
4533 format %{ "sve_vectorcast_s2b $dst, $src\t# convert H to B vector" %}
4534 ins_encode %{
4535 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
4536 as_FloatRegister($src$$reg), __ H, as_FloatRegister($tmp$$reg));
4537 %}
4538 ins_pipe(pipe_slow);
4539 %}
4540
4541 instruct vcvtStoX_extend(vReg dst, vReg src)
4542 %{
4543 predicate(UseSVE > 0 &&
4544 type2aelembytes(Matcher::vector_element_basic_type(n)) > 2);
4545 match(Set dst (VectorCastS2X src));
4546 ins_cost(2 * SVE_COST);
4547 format %{ "sve_vectorcast_s2x $dst, $src\t# convert H to X vector (extend)" %}
4548 ins_encode %{
4549 BasicType to_bt = Matcher::vector_element_basic_type(this);
4550 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4551 __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ H);
4552 if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
4553 __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
4554 }
4555 %}
4556 ins_pipe(pipe_slow);
4557 %}
4558
4559 instruct vcvtItoB(vReg dst, vReg src, vReg tmp)
4560 %{
4561 predicate(UseSVE > 0 &&
4562 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
4563 match(Set dst (VectorCastI2X src));
4564 effect(TEMP_DEF dst, TEMP tmp);
4565 ins_cost(3 * SVE_COST);
4566 format %{ "sve_vectorcast_i2b $dst, $src\t# convert I to B vector" %}
4567 ins_encode %{
4568 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
4569 as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
4570 %}
4571 ins_pipe(pipe_slow);
4572 %}
4573
4574 instruct vcvtItoS(vReg dst, vReg src, vReg tmp)
4575 %{
4576 predicate(UseSVE > 0 &&
4577 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
4578 match(Set dst (VectorCastI2X src));
4579 effect(TEMP tmp);
4580 ins_cost(2 * SVE_COST);
4581 format %{ "sve_vectorcast_i2s $dst, $src\t# convert I to H vector" %}
4582 ins_encode %{
4583 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ H,
4584 as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
4585 %}
4586 ins_pipe(pipe_slow);
4587 %}
4588
4589 instruct vcvtItoL(vReg dst, vReg src)
4590 %{
4591 predicate(UseSVE > 0 &&
4592 n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
4593 match(Set dst (VectorCastI2X src));
4594 ins_cost(SVE_COST);
4595 format %{ "sve_vectorcast_i2l $dst, $src\t# convert I to L vector" %}
4596 ins_encode %{
4597 __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
4598 %}
4599 ins_pipe(pipe_slow);
4600 %}
4601
4602 instruct vcvtItoF(vReg dst, vReg src)
4603 %{
4604 predicate(UseSVE > 0 &&
4605 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4606 match(Set dst (VectorCastI2X src));
4607 ins_cost(SVE_COST);
4608 format %{ "sve_vectorcast_i2f $dst, $src\t# convert I to F vector" %}
4609 ins_encode %{
4610 __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
4611 %}
4612 ins_pipe(pipe_slow);
4613 %}
4614
4615 instruct vcvtItoD(vReg dst, vReg src)
4616 %{
4617 predicate(UseSVE > 0 &&
4618 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
4619 match(Set dst (VectorCastI2X src));
4620 ins_cost(2 * SVE_COST);
4621 format %{ "sve_vectorcast_i2d $dst, $src\t# convert I to D vector" %}
4622 ins_encode %{
4623 __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
4624 __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ D);
4625 %}
4626 ins_pipe(pipe_slow);
4627 %}
4628
4629 instruct vcvtLtoX_narrow(vReg dst, vReg src, vReg tmp)
4630 %{
4631 predicate(UseSVE > 0 && is_integral_type(Matcher::vector_element_basic_type(n)));
4632 match(Set dst (VectorCastL2X src));
4633 effect(TEMP_DEF dst, TEMP tmp);
4634 ins_cost(2 * SVE_COST);
4635 format %{ "sve_vectorcast_l2x $dst, $src\t# convert L to B/H/S vector (narrow)" %}
4636 ins_encode %{
4637 BasicType to_bt = Matcher::vector_element_basic_type(this);
4638 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4639 __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
4640 as_FloatRegister($src$$reg), __ D, as_FloatRegister($tmp$$reg));
4641 %}
4642 ins_pipe(pipe_slow);
4643 %}
4644
4645 instruct vcvtLtoF(vReg dst, vReg src, vReg tmp)
4646 %{
4647 predicate(UseSVE > 0 &&
4648 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4649 match(Set dst (VectorCastL2X src));
4650 effect(TEMP_DEF dst, TEMP tmp);
4651 ins_cost(3 * SVE_COST);
4652 format %{ "sve_vectorcast_l2f $dst, $src\t# convert L to F vector" %}
4653 ins_encode %{
4654 __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
4655 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
4656 as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
4657
4658 %}
4659 ins_pipe(pipe_slow);
4660 %}
4661
4662 instruct vcvtLtoD(vReg dst, vReg src)
4663 %{
4664 predicate(UseSVE > 0 &&
4665 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
4666 match(Set dst (VectorCastL2X src));
4667 ins_cost(SVE_COST);
4668 format %{ "sve_vectorcast_l2d $dst, $src\t# convert L to D vector" %}
4669 ins_encode %{
4670 __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
4671 %}
4672 ins_pipe(pipe_slow);
4673 %}
4674
4675 instruct vcvtFtoX_narrow(vReg dst, vReg src, vReg tmp)
4676 %{
4677 predicate(UseSVE > 0 &&
4678 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4679 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT));
4680 match(Set dst (VectorCastF2X src));
4681 effect(TEMP_DEF dst, TEMP tmp);
4682 ins_cost(3 * SVE_COST);
4683 format %{ "sve_vectorcast_f2x $dst, $src\t# convert F to B/H vector" %}
4684 ins_encode %{
4685 BasicType to_bt = Matcher::vector_element_basic_type(this);
4686 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4687 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
4688 __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
4689 as_FloatRegister($dst$$reg), __ S, as_FloatRegister($tmp$$reg));
4690 %}
4691 ins_pipe(pipe_slow);
4692 %}
4693
4694 instruct vcvtFtoI(vReg dst, vReg src)
4695 %{
4696 predicate(UseSVE > 0 &&
4697 (n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4698 match(Set dst (VectorCastF2X src));
4699 ins_cost(SVE_COST);
4700 format %{ "sve_vectorcast_f2x $dst, $src\t# convert F to I vector" %}
4701 ins_encode %{
4702 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
4703 %}
4704 ins_pipe(pipe_slow);
4705 %}
4706
4707 instruct vcvtFtoL(vReg dst, vReg src)
4708 %{
4709 predicate(UseSVE > 0 &&
4710 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
4711 match(Set dst (VectorCastF2X src));
4712 ins_cost(SVE_COST * 2);
4713 format %{ "sve_vectorcast_f2x $dst, $src\t# convert F to L vector" %}
4714 ins_encode %{
4715 __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
4716 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);
4717 %}
4718 ins_pipe(pipe_slow);
4719 %}
4720
4721 instruct vcvtFtoD(vReg dst, vReg src)
4722 %{
4723 predicate(UseSVE > 0 &&
4724 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
4725 match(Set dst (VectorCastF2X src));
4726 ins_cost(2 * SVE_COST);
4727 format %{ "sve_vectorcast_f2d $dst, $dst\t# convert F to D vector" %}
4728 ins_encode %{
4729 __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
4730 __ sve_fcvt(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);
4731 %}
4732 ins_pipe(pipe_slow);
4733 %}
4734
4735 instruct vcvtDtoX_narrow(vReg dst, vReg src, vReg tmp)
4736 %{
4737 predicate(UseSVE > 0 &&
4738 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4739 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
4740 n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4741 match(Set dst (VectorCastD2X src));
4742 effect(TEMP_DEF dst, TEMP tmp);
4743 ins_cost(3 * SVE_COST);
4744 format %{ "sve_vectorcast_d2x $dst, $src\t# convert D to X vector (narrow)" %}
4745 ins_encode %{
4746 BasicType to_bt = Matcher::vector_element_basic_type(this);
4747 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
4748 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
4749 __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
4750 as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
4751 %}
4752 ins_pipe(pipe_slow);
4753 %}
4754
4755 instruct vcvtDtoL(vReg dst, vReg src)
4756 %{
4757 predicate(UseSVE > 0 &&
4758 n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
4759 match(Set dst (VectorCastD2X src));
4760 ins_cost(SVE_COST);
4761 format %{ "sve_vectorcast_d2l $dst, $src\t# convert D to L vector" %}
4762 ins_encode %{
4763 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
4764 %}
4765 ins_pipe(pipe_slow);
4766 %}
4767
4768 instruct vcvtDtoF(vReg dst, vReg src, vReg tmp)
4769 %{
4770 predicate(UseSVE > 0 &&
4771 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
4772 match(Set dst (VectorCastD2X src));
4773 effect(TEMP_DEF dst, TEMP tmp);
4774 ins_cost(3 * SVE_COST);
4775 format %{ "sve_vectorcast_d2f $dst, S, $dst\t# convert D to F vector" %}
4776 ins_encode %{
4777 __ sve_fcvt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
4778 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
4779 as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
4780 %}
4781 ins_pipe(pipe_slow);
4782 %}
4783
4784 // ------------------------------ Vector extract ---------------------------------
4785
4786 instruct extractB(iRegINoSp dst, vReg src, immI idx, vReg vtmp)
4787 %{
4788 predicate(UseSVE > 0 && n->in(2)->get_int() >= 16);
4789 match(Set dst (ExtractB src idx));
4790 effect(TEMP vtmp);
4791 ins_cost(2 * SVE_COST);
4792 format %{ "sve_extract_integral $dst, B, $src, $idx\t# extract from vector(B)" %}
4793 ins_encode %{
4794 __ sve_extract_integral(as_Register($dst$$reg), __ B, as_FloatRegister($src$$reg),
4795 (int)($idx$$constant), /* is_signed */ true, as_FloatRegister($vtmp$$reg));
4796 %}
4797 ins_pipe(pipe_slow);
4798 %}
4799
4800 instruct extractS(iRegINoSp dst, vReg src, immI idx, vReg vtmp)
4801 %{
4802 predicate(UseSVE > 0 && n->in(2)->get_int() >= 8);
4803 match(Set dst (ExtractS src idx));
4804 effect(TEMP vtmp);
4805 ins_cost(2 * SVE_COST);
4806 format %{ "sve_extract_integral $dst, H, $src, $idx\t# extract from vector(S)" %}
4807 ins_encode %{
4808 __ sve_extract_integral(as_Register($dst$$reg), __ H, as_FloatRegister($src$$reg),
4809 (int)($idx$$constant), /* is_signed */ true, as_FloatRegister($vtmp$$reg));
4810 %}
4811 ins_pipe(pipe_slow);
4812 %}
4813
4814 instruct extractI(iRegINoSp dst, vReg src, immI idx, vReg vtmp)
4815 %{
4816 predicate(UseSVE > 0 && n->in(2)->get_int() >= 4);
4817 match(Set dst (ExtractI src idx));
4818 effect(TEMP vtmp);
4819 ins_cost(2 * SVE_COST);
4820 format %{ "sve_extract_integral $dst, S, $src, $idx\t# extract from vector(I)" %}
4821 ins_encode %{
4822 __ sve_extract_integral(as_Register($dst$$reg), __ S, as_FloatRegister($src$$reg),
4823 (int)($idx$$constant), /* is_signed */ true, as_FloatRegister($vtmp$$reg));
4824 %}
4825 ins_pipe(pipe_slow);
4826 %}
4827
4828 instruct extractL(iRegLNoSp dst, vReg src, immI idx, vReg vtmp)
4829 %{
4830 predicate(UseSVE > 0 && n->in(2)->get_int() >= 2);
4831 match(Set dst (ExtractL src idx));
4832 effect(TEMP vtmp);
4833 ins_cost(2 * SVE_COST);
4834 format %{ "sve_extract_integral $dst, D, $src, $idx\t# extract from vector(L)" %}
4835 ins_encode %{
4836 __ sve_extract_integral(as_Register($dst$$reg), __ D, as_FloatRegister($src$$reg),
4837 (int)($idx$$constant), /* is_signed */ false, as_FloatRegister($vtmp$$reg));
4838 %}
4839 ins_pipe(pipe_slow);
4840 %}
4841
4842 instruct extractB_LT16(iRegINoSp dst, vReg src, immI idx)
4843 %{
4844 predicate(UseSVE > 0 && n->in(2)->get_int() < 16);
4845 match(Set dst (ExtractB src idx));
4846 ins_cost(INSN_COST);
4847 format %{ "smov $dst, B, $src, $idx\t# extract from vector(B)" %}
4848 ins_encode %{
4849 __ smov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ B, $idx$$constant);
4850 %}
4851 ins_pipe(pipe_class_default);
4852 %}
4853
4854 instruct extractS_LT8(iRegINoSp dst, vReg src, immI idx)
4855 %{
4856 predicate(UseSVE > 0 && n->in(2)->get_int() < 8);
4857 match(Set dst (ExtractS src idx));
4858 ins_cost(INSN_COST);
4859 format %{ "smov $dst, H, $src, $idx\t# extract from vector(S)" %}
4860 ins_encode %{
4861 __ smov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ H, $idx$$constant);
4862 %}
4863 ins_pipe(pipe_class_default);
4864 %}
4865
4866 instruct extractI_LT4(iRegINoSp dst, vReg src, immI idx)
4867 %{
4868 predicate(UseSVE > 0 && n->in(2)->get_int() < 4);
4869 match(Set dst (ExtractI src idx));
4870 ins_cost(INSN_COST);
4871 format %{ "smov $dst, S, $src, $idx\t# extract from vector(I)" %}
4872 ins_encode %{
4873 __ smov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ S, $idx$$constant);
4874 %}
4875 ins_pipe(pipe_class_default);
4876 %}
4877
4878 instruct extractL_LT2(iRegLNoSp dst, vReg src, immI idx)
4879 %{
4880 predicate(UseSVE > 0 && n->in(2)->get_int() < 2);
4881 match(Set dst (ExtractL src idx));
4882 ins_cost(INSN_COST);
4883 format %{ "umov $dst, D, $src, $idx\t# extract from vector(L)" %}
4884 ins_encode %{
4885 __ umov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ D, $idx$$constant);
4886 %}
4887 ins_pipe(pipe_class_default);
4888 %}
4889
4890 instruct extractF(vRegF dst, vReg src, immI idx)
4891 %{
4892 predicate(UseSVE > 0);
4893 match(Set dst (ExtractF src idx));
4894 ins_cost(2 * SVE_COST);
4895 format %{ "sve_extract_f $dst, S, $src, $idx\t# extract from vector(F)" %}
4896 ins_encode %{
4897 if ((as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg)) && ($idx$$constant == 0)) {
4898 /* empty */
4899 } else if ($idx$$constant == 0) {
4900 __ fmovs(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
4901 } else if ($idx$$constant < 4) {
4902 __ ins(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($src$$reg), 0, (int)($idx$$constant));
4903 } else {
4904 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4905 __ sve_ext(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), $idx$$constant << 2);
4906 }
4907 %}
4908 ins_pipe(pipe_slow);
4909 %}
4910
4911 instruct extractD(vRegD dst, vReg src, immI idx)
4912 %{
4913 predicate(UseSVE > 0);
4914 match(Set dst (ExtractD src idx));
4915 ins_cost(2 * SVE_COST);
4916 format %{ "sve_extract_d $dst, D, $src, $idx\t# extract from vector(D)" %}
4917 ins_encode %{
4918 if ((as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg)) && ($idx$$constant == 0)) {
4919 /* empty */
4920 } else if ($idx$$constant == 0) {
4921 __ fmovd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
4922 } else if ($idx$$constant == 1) {
4923 __ ins(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), 0, 1);
4924 } else {
4925 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4926 __ sve_ext(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), $idx$$constant << 3);
4927 }
4928 %}
4929 ins_pipe(pipe_slow);
4930 %}
4931
4932 // ------------------------------- VectorTest ----------------------------------
4933
4934 instruct vtest_alltrue(iRegINoSp dst, pRegGov src1, pRegGov src2, pReg ptmp, rFlagsReg cr)
4935 %{
4936 predicate(UseSVE > 0 &&
4937 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
4938 match(Set dst (VectorTest src1 src2));
4939 effect(TEMP ptmp, KILL cr);
4940 ins_cost(SVE_COST);
4941 format %{ "sve_eors $ptmp, $src1, $src2\t# $src2 is all true mask\n"
4942 "csetw $dst, EQ\t# VectorTest (sve) - alltrue" %}
4943 ins_encode %{
4944 __ sve_eors(as_PRegister($ptmp$$reg), ptrue,
4945 as_PRegister($src1$$reg), as_PRegister($src2$$reg));
4946 __ csetw(as_Register($dst$$reg), Assembler::EQ);
4947 %}
4948 ins_pipe(pipe_slow);
4949 %}
4950
4951 instruct vtest_anytrue(iRegINoSp dst, pRegGov src1, pRegGov src2, rFlagsReg cr)
4952 %{
4953 predicate(UseSVE > 0 &&
4954 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
4955 match(Set dst (VectorTest src1 src2));
4956 effect(KILL cr);
4957 ins_cost(SVE_COST);
4958 format %{ "sve_ptest $src1\n\t"
4959 "csetw $dst, NE\t# VectorTest (sve) - anytrue" %}
4960 ins_encode %{
4961 // "src2" is not used for sve.
4962 __ sve_ptest(ptrue, as_PRegister($src1$$reg));
4963 __ csetw(as_Register($dst$$reg), Assembler::NE);
4964 %}
4965 ins_pipe(pipe_slow);
4966 %}
4967 // ------------------------------ Vector insert ---------------------------------
4968
4969 instruct insertI_le128bits(vReg dst, vReg src, iRegIorL2I val, immI idx) %{
4970 predicate(UseSVE > 0 &&
4971 (Matcher::vector_length_in_bytes(n) == 8 || Matcher::vector_length_in_bytes(n) == 16) &&
4972 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4973 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
4974 n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4975 match(Set dst (VectorInsert (Binary src val) idx));
4976 ins_cost(2 * INSN_COST);
4977 format %{ "orr $dst, T8/16B, $src, $src\n\t"
4978 "mov $dst, B/H/S, $idx, $val\t# insertI into vector(64/128bits)" %}
4979 ins_encode %{
4980 BasicType bt = Matcher::vector_element_basic_type(this);
4981 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
4982 __ orr(as_FloatRegister($dst$$reg), Matcher::vector_length_in_bytes(this) == 8 ? __ T8B : __ T16B,
4983 as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
4984 }
4985 __ mov(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(Matcher::vector_element_basic_type(this)),
4986 $idx$$constant, $val$$Register);
4987 %}
4988 ins_pipe(pipe_slow);
4989 %}
4990
4991 instruct insertI_small_index(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg vtmp, pRegGov pgtmp, rFlagsReg cr) %{
4992 predicate(UseSVE > 0 && n->in(2)->get_int() < 32 &&
4993 Matcher::vector_length_in_bytes(n) > 16 &&
4994 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
4995 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
4996 n->bottom_type()->is_vect()->element_basic_type() == T_INT));
4997 match(Set dst (VectorInsert (Binary src val) idx));
4998 effect(TEMP vtmp, TEMP pgtmp, KILL cr);
4999 ins_cost(4 * SVE_COST);
5000 format %{ "sve_index $vtmp, -16, 1\t# (B/H/S)\n\t"
5001 "sve_cmpeq $pgtmp, $vtmp, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
5002 "sve_orr $dst, $src, $src\n\t"
5003 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
5004 ins_encode %{
5005 BasicType bt = Matcher::vector_element_basic_type(this, $src);
5006 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5007 __ block_comment("insert into vector (B/H/S) {");
5008 __ sve_index(as_FloatRegister($vtmp$$reg), size, -16, 1);
5009 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
5010 as_FloatRegister($vtmp$$reg), (int)($idx$$constant) - 16);
5011 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
5012 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
5013 }
5014 __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), $val$$Register);
5015 __ block_comment("} insert into vector (B/H/S)");
5016 %}
5017 ins_pipe(pipe_slow);
5018 %}
5019
5020 instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg vtmp1, vReg vtmp2, pRegGov pgtmp, rFlagsReg cr) %{
5021 predicate(UseSVE > 0 && n->in(2)->get_int() >= 32 &&
5022 Matcher::vector_length_in_bytes(n) > 16 &&
5023 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
5024 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
5025 n->bottom_type()->is_vect()->element_basic_type() == T_INT));
5026 match(Set dst (VectorInsert (Binary src val) idx));
5027 effect(TEMP vtmp1, TEMP vtmp2, TEMP pgtmp, KILL cr);
5028 ins_cost(5 * SVE_COST);
5029 format %{ "sve_index $vtmp1, 0, 1\t# (B/H/S)\n\t"
5030 "sve_dup $vtmp2, $idx\t# (B/H/S)\n\t"
5031 "sve_cmpeq $pgtmp, $vtmp1, $vtmp2\n\t"
5032 "sve_orr $dst, $src, $src\n\t"
5033 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
5034 ins_encode %{
5035 BasicType bt = Matcher::vector_element_basic_type(this, $src);
5036 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5037 __ block_comment("insert into vector (B/H/S) {");
5038 __ sve_index(as_FloatRegister($vtmp1$$reg), size, 0, 1);
5039 __ sve_dup(as_FloatRegister($vtmp2$$reg), size, (int)($idx$$constant));
5040 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
5041 as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg));
5042 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
5043 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
5044 }
5045 __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), $val$$Register);
5046 __ block_comment("} insert into vector (B/H/S)");
5047 %}
5048 ins_pipe(pipe_slow);
5049 %}
5050
5051 instruct insertL_128bits(vReg dst, vReg src, iRegL val, immI idx) %{
5052 predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) == 16 &&
5053 n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
5054 match(Set dst (VectorInsert (Binary src val) idx));
5055 ins_cost(2 * INSN_COST);
5056 format %{ "orr $dst, T16B, $src, $src\n\t"
5057 "mov $dst, D, $idx, $val\t# insertL into vector(128bits)" %}
5058 ins_encode %{
5059 BasicType bt = Matcher::vector_element_basic_type(this);
5060 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
5061 __ orr(as_FloatRegister($dst$$reg), __ T16B,
5062 as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
5063 }
5064 __ mov(as_FloatRegister($dst$$reg), __ D, $idx$$constant, $val$$Register);
5065 %}
5066 ins_pipe(pipe_slow);
5067 %}
5068
5069 instruct insertL(vReg dst, vReg src, iRegL val, immI idx, vReg vtmp, pRegGov pgtmp, rFlagsReg cr) %{
5070 predicate(UseSVE > 0 &&
5071 Matcher::vector_length_in_bytes(n) > 16 &&
5072 n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
5073 match(Set dst (VectorInsert (Binary src val) idx));
5074 effect(TEMP vtmp, TEMP pgtmp, KILL cr);
5075 ins_cost(4 * SVE_COST);
5076 format %{ "sve_index $vtmp, D, -16, 1\n\t"
5077 "sve_cmpeq $pgtmp, $vtmp, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
5078 "sve_orr $dst, $src, $src\n\t"
5079 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (L)" %}
5080 ins_encode %{
5081 __ block_comment("insert into vector (L) {");
5082 __ sve_index(as_FloatRegister($vtmp$$reg), __ D, -16, 1);
5083 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
5084 as_FloatRegister($vtmp$$reg), (int)($idx$$constant) - 16);
5085 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
5086 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
5087 }
5088 __ sve_cpy(as_FloatRegister($dst$$reg), __ D,
5089 as_PRegister($pgtmp$$reg), $val$$Register);
5090 __ block_comment("} insert into vector (L)");
5091 %}
5092 ins_pipe(pipe_slow);
5093 %}
5094
5095 instruct insertF_le128bits(vReg dst, vReg src, vRegF val, immI idx) %{
5096 predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
5097 (Matcher::vector_length_in_bytes(n) == 8 || Matcher::vector_length_in_bytes(n) == 16));
5098 match(Set dst (VectorInsert (Binary src val) idx));
5099 ins_cost(2 * INSN_COST);
5100 effect(TEMP_DEF dst);
5101 format %{ "orr $dst, T8/16B, $src, $src\n\t"
5102 "ins $dst, S, $val, $idx, 0\t# insertF into vector(64/128bits)" %}
5103 ins_encode %{
5104 __ orr(as_FloatRegister($dst$$reg), Matcher::vector_length_in_bytes(this) == 8 ? __ T8B : __ T16B,
5105 as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
5106 __ ins(as_FloatRegister($dst$$reg), __ S,
5107 as_FloatRegister($val$$reg), $idx$$constant, 0);
5108 %}
5109 ins_pipe(pipe_slow);
5110 %}
5111
5112 instruct insertF_small_index(vReg dst, vReg src, vRegF val, immI idx, pRegGov pgtmp, rFlagsReg cr) %{
5113 predicate(UseSVE > 0 && n->in(2)->get_int() < 32 &&
5114 Matcher::vector_length_in_bytes(n) > 16 &&
5115 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
5116 match(Set dst (VectorInsert (Binary src val) idx));
5117 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
5118 ins_cost(4 * SVE_COST);
5119 format %{ "sve_index $dst, S, -16, 1\n\t"
5120 "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
5121 "sve_orr $dst, $src, $src\n\t"
5122 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
5123 ins_encode %{
5124 __ block_comment("insert into vector (F) {");
5125 __ sve_index(as_FloatRegister($dst$$reg), __ S, -16, 1);
5126 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
5127 as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
5128 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
5129 __ sve_cpy(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
5130 __ block_comment("} insert into vector (F)");
5131 %}
5132 ins_pipe(pipe_slow);
5133 %}
5134
5135 instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr) %{
5136 predicate(UseSVE > 0 && n->in(2)->get_int() >= 32 &&
5137 Matcher::vector_length_in_bytes(n) > 16 &&
5138 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
5139 match(Set dst (VectorInsert (Binary src val) idx));
5140 effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
5141 ins_cost(5 * SVE_COST);
5142 format %{ "sve_index $tmp1, S, 0, 1\n\t"
5143 "sve_dup $dst, S, $idx\n\t"
5144 "sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
5145 "sve_orr $dst, $src, $src\n\t"
5146 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
5147 ins_encode %{
5148 __ block_comment("insert into vector (F) {");
5149 __ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
5150 __ sve_dup(as_FloatRegister($dst$$reg), __ S, (int)($idx$$constant));
5151 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
5152 as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
5153 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
5154 as_FloatRegister($src$$reg));
5155 __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
5156 as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
5157 __ block_comment("} insert into vector (F)");
5158 %}
5159 ins_pipe(pipe_slow);
5160 %}
5161
5162 instruct insertD_128bits(vReg dst, vReg src, vRegD val, immI idx) %{
5163 predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) == 16 &&
5164 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
5165 match(Set dst (VectorInsert (Binary src val) idx));
5166 ins_cost(2 * INSN_COST);
5167 effect(TEMP_DEF dst);
5168 format %{ "orr $dst, T16B, $src, $src\n\t"
5169 "ins $dst, D, $val, $idx, 0\t# insertD into vector(128bits)" %}
5170 ins_encode %{
5171 __ orr(as_FloatRegister($dst$$reg), __ T16B,
5172 as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
5173 __ ins(as_FloatRegister($dst$$reg), __ D,
5174 as_FloatRegister($val$$reg), $idx$$constant, 0);
5175 %}
5176 ins_pipe(pipe_slow);
5177 %}
5178
5179 instruct insertD(vReg dst, vReg src, vRegD val, immI idx, pRegGov pgtmp, rFlagsReg cr) %{
5180 predicate(UseSVE > 0 &&
5181 Matcher::vector_length_in_bytes(n) > 16 &&
5182 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
5183 match(Set dst (VectorInsert (Binary src val) idx));
5184 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
5185 ins_cost(4 * SVE_COST);
5186 format %{ "sve_index $dst, D, -16, 1\n\t"
5187 "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
5188 "sve_orr $dst, $src, $src\n\t"
5189 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (D)" %}
5190 ins_encode %{
5191 __ block_comment("insert into vector (D) {");
5192 __ sve_index(as_FloatRegister($dst$$reg), __ D, -16, 1);
5193 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
5194 as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
5195 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
5196 as_FloatRegister($src$$reg));
5197 __ sve_cpy(as_FloatRegister($dst$$reg), __ D,
5198 as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
5199 __ block_comment("} insert into vector (D)");
5200 %}
5201 ins_pipe(pipe_slow);
5202 %}
5203
5204 // ------------------------------ Vector shuffle -------------------------------
5205
5206 instruct loadshuffle(vReg dst, vReg src) %{
5207 predicate(UseSVE > 0);
5208 match(Set dst (VectorLoadShuffle src));
5209 ins_cost(SVE_COST);
5210 format %{ "sve_loadshuffle $dst, $src\t# vector load shuffle (B/H/S/D)" %}
5211 ins_encode %{
5212 BasicType bt = Matcher::vector_element_basic_type(this);
5213 if (bt == T_BYTE) {
5214 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
5215 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
5216 as_FloatRegister($src$$reg));
5217 }
5218 } else {
5219 __ sve_vector_extend(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
5220 as_FloatRegister($src$$reg), __ B);
5221 }
5222 %}
5223 ins_pipe(pipe_slow);
5224 %}
5225
5226 // ------------------------------ Vector rearrange -------------------------------
5227
5228 instruct rearrange(vReg dst, vReg src, vReg shuffle)
5229 %{
5230 predicate(UseSVE > 0);
5231 match(Set dst (VectorRearrange src shuffle));
5232 ins_cost(SVE_COST);
5233 format %{ "sve_tbl $dst, $src, $shuffle\t# vector rearrange" %}
5234 ins_encode %{
5235 BasicType bt = Matcher::vector_element_basic_type(this, $src);
5236 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5237 __ sve_tbl(as_FloatRegister($dst$$reg), size,
5238 as_FloatRegister($src$$reg), as_FloatRegister($shuffle$$reg));
5239 %}
5240 ins_pipe(pipe_slow);
5241 %}
5242
5243 // ------------------------------ Vector Load Gather ---------------------------------
5244
5245 instruct gatherI(vReg dst, indirect mem, vReg idx) %{
5246 predicate(UseSVE > 0 &&
5247 n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
5248 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5249 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5250 match(Set dst (LoadVectorGather mem idx));
5251 ins_cost(SVE_COST);
5252 format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (S)" %}
5253 ins_encode %{
5254 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), ptrue,
5255 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5256 %}
5257 ins_pipe(pipe_slow);
5258 %}
5259
5260 instruct gatherL(vReg dst, indirect mem, vReg idx) %{
5261 predicate(UseSVE > 0 &&
5262 n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
5263 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5264 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5265 match(Set dst (LoadVectorGather mem idx));
5266 ins_cost(2 * SVE_COST);
5267 format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (D)" %}
5268 ins_encode %{
5269 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5270 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), ptrue, as_Register($mem$$base),
5271 as_FloatRegister($idx$$reg));
5272 %}
5273 ins_pipe(pipe_slow);
5274 %}
5275
5276 // ------------------------------ Vector Load Gather Partial-------------------------------
5277
5278 instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5279 predicate(UseSVE > 0 &&
5280 n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
5281 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5282 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5283 match(Set dst (LoadVectorGather mem idx));
5284 effect(TEMP ptmp, KILL cr);
5285 ins_cost(2 * SVE_COST + INSN_COST);
5286 format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (S)" %}
5287 ins_encode %{
5288 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this));
5289 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
5290 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5291 %}
5292 ins_pipe(pipe_slow);
5293 %}
5294
5295 instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5296 predicate(UseSVE > 0 &&
5297 n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
5298 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5299 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5300 match(Set dst (LoadVectorGather mem idx));
5301 effect(TEMP ptmp, KILL cr);
5302 ins_cost(3 * SVE_COST + INSN_COST);
5303 format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (D)" %}
5304 ins_encode %{
5305 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this));
5306 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5307 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
5308 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5309 %}
5310 ins_pipe(pipe_slow);
5311 %}
5312
5313 // ------------------------------ Vector Load Gather Predicated -------------------------------
5314
5315 instruct gatherI_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
5316 predicate(UseSVE > 0 &&
5317 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5318 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5319 match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
5320 ins_cost(SVE_COST);
5321 format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (S)" %}
5322 ins_encode %{
5323 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
5324 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5325 %}
5326 ins_pipe(pipe_slow);
5327 %}
5328
5329 instruct gatherL_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
5330 predicate(UseSVE > 0 &&
5331 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5332 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5333 match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
5334 ins_cost(2 * SVE_COST);
5335 format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (D)" %}
5336 ins_encode %{
5337 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5338 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
5339 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5340 %}
5341 ins_pipe(pipe_slow);
5342 %}
5343
5344 // ------------------------------ Vector Store Scatter -------------------------------
5345
5346 instruct scatterI(indirect mem, vReg src, vReg idx) %{
5347 predicate(UseSVE > 0 &&
5348 n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
5349 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5350 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5351 match(Set mem (StoreVectorScatter mem (Binary src idx)));
5352 ins_cost(SVE_COST);
5353 format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (S)" %}
5354 ins_encode %{
5355 __ sve_st1w_scatter(as_FloatRegister($src$$reg), ptrue,
5356 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5357 %}
5358 ins_pipe(pipe_slow);
5359 %}
5360
5361 instruct scatterL(indirect mem, vReg src, vReg idx) %{
5362 predicate(UseSVE > 0 &&
5363 n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
5364 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5365 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5366 match(Set mem (StoreVectorScatter mem (Binary src idx)));
5367 ins_cost(2 * SVE_COST);
5368 format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (D)" %}
5369 ins_encode %{
5370 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5371 __ sve_st1d_scatter(as_FloatRegister($src$$reg), ptrue,
5372 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5373 %}
5374 ins_pipe(pipe_slow);
5375 %}
5376
5377 // ------------------------------ Vector Store Scatter Partial -------------------------------
5378
5379 instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5380 predicate(UseSVE > 0 &&
5381 n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
5382 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5383 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5384 match(Set mem (StoreVectorScatter mem (Binary src idx)));
5385 effect(TEMP ptmp, KILL cr);
5386 ins_cost(2 * SVE_COST + INSN_COST);
5387 format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (S)" %}
5388 ins_encode %{
5389 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src));
5390 __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
5391 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5392 %}
5393 ins_pipe(pipe_slow);
5394 %}
5395
5396 instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
5397 predicate(UseSVE > 0 &&
5398 n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
5399 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5400 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5401 match(Set mem (StoreVectorScatter mem (Binary src idx)));
5402 effect(TEMP ptmp, KILL cr);
5403 ins_cost(3 * SVE_COST + INSN_COST);
5404 format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (D)" %}
5405 ins_encode %{
5406 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src));
5407 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5408 __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
5409 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5410 %}
5411 ins_pipe(pipe_slow);
5412 %}
5413
5414 // ------------------------------ Vector Store Scatter Predicated -------------------------------
5415
5416 instruct scatterI_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
5417 predicate(UseSVE > 0 &&
5418 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5419 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
5420 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
5421 ins_cost(SVE_COST);
5422 format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicate (S)" %}
5423 ins_encode %{
5424 __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
5425 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5426 %}
5427 ins_pipe(pipe_slow);
5428 %}
5429
5430 instruct scatterL_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
5431 predicate(UseSVE > 0 &&
5432 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5433 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5434 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
5435 ins_cost(2 * SVE_COST);
5436 format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated (D)" %}
5437 ins_encode %{
5438 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
5439 __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
5440 as_Register($mem$$base), as_FloatRegister($idx$$reg));
5441 %}
5442 ins_pipe(pipe_slow);
5443 %}
5444
5445 // ------------------------------ Vector Load Const -------------------------------
5446
5447 instruct loadconB(vReg dst, immI0 src) %{
5448 predicate(UseSVE > 0 &&
5449 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
5450 match(Set dst (VectorLoadConst src));
5451 ins_cost(SVE_COST);
5452 format %{ "sve_index $dst, 0, 1\t# generate iota indices" %}
5453 ins_encode %{
5454 __ sve_index(as_FloatRegister($dst$$reg), __ B, 0, 1);
5455 %}
5456 ins_pipe(pipe_slow);
5457 %}
5458
5459 // Intrisics for String.indexOf(char)
5460
5461
5462 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
5463 iRegI_R0 result, vReg ztmp1, vReg ztmp2,
5464 pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
5465 %{
5466 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
5467 predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
5468 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
5469
5470 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
5471
5472 ins_encode %{
5473 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
5474 as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
5475 as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), true /* isL */);
5476 %}
5477 ins_pipe(pipe_class_memory);
5478 %}
5479
5480 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
5481 iRegI_R0 result, vReg ztmp1, vReg ztmp2,
5482 pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
5483 %{
5484 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
5485 predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
5486 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
5487
5488 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
5489
5490 ins_encode %{
5491 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
5492 as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
5493 as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), false /* isL */);
5494 %}
5495 ins_pipe(pipe_class_memory);
5496 %}
5497
5498 // ---------------------------- Vector mask reductions ---------------------------
5499 instruct vmask_truecount(iRegINoSp dst, pReg src) %{
5500 predicate(UseSVE > 0);
5501 match(Set dst (VectorMaskTrueCount src));
5502 ins_cost(SVE_COST);
5503 format %{ "vmask_truecount $dst, $src\t# vector mask truecount (sve)" %}
5504 ins_encode %{
5505 BasicType bt = Matcher::vector_element_basic_type(this, $src);
5506 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5507 __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($src$$reg));
5508 %}
5509 ins_pipe(pipe_slow);
5510 %}
5511
5512 // Return the index of the first mask lane that is set, or vector length if none of
5513 // them are set.
5514 instruct vmask_firsttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
5515 predicate(UseSVE > 0);
5516 match(Set dst (VectorMaskFirstTrue src));
5517 effect(TEMP ptmp);
5518 ins_cost(3 * SVE_COST);
5519 format %{ "vmask_firsttrue $dst, $src\t# vector mask firsttrue (sve)" %}
5520 ins_encode %{
5521 uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
5522 BasicType bt = Matcher::vector_element_basic_type(this, $src);
5523 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5524 // When the input predicate is all-false, the result should be the vector length
5525 // instead of max vector register size.
5526 if (length_in_bytes == MaxVectorSize) {
5527 __ sve_brkb(as_PRegister($ptmp$$reg), ptrue, as_PRegister($src$$reg), false);
5528 } else {
5529 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
5530 __ sve_brkb(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg), as_PRegister($src$$reg), false);
5531 }
5532 __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($ptmp$$reg));
5533 %}
5534 ins_pipe(pipe_slow);
5535 %}
5536
5537 instruct vmask_lasttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
5538 predicate(UseSVE > 0);
5539 match(Set dst (VectorMaskLastTrue src));
5540 effect(TEMP ptmp);
5541 ins_cost(3 * SVE_COST);
5542 format %{ "vmask_lasttrue $dst, $src\t# vector mask lasttrue (sve)" %}
5543 ins_encode %{
5544 BasicType bt = Matcher::vector_element_basic_type(this, $src);
5545 __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($src$$reg), as_PRegister($ptmp$$reg));
5546 %}
5547 ins_pipe(pipe_slow);
5548 %}
5549
5550 instruct vmask_tolong(iRegLNoSp dst, pReg src, vReg vtmp1, vReg vtmp2) %{
5551 predicate(UseSVE > 0 &&
5552 n->in(1)->bottom_type()->is_vect()->length() <= 64);
5553 match(Set dst (VectorMaskToLong src));
5554 effect(TEMP vtmp1, TEMP vtmp2);
5555 ins_cost(13 * SVE_COST);
5556 format %{ "vmask_tolong $dst, $src\t# vector mask tolong (sve)" %}
5557 ins_encode %{
5558 __ sve_vmask_tolong(as_Register($dst$$reg), as_PRegister($src$$reg),
5559 Matcher::vector_element_basic_type(this, $src),
5560 Matcher::vector_length(this, $src),
5561 as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg));
5562 %}
5563 ins_pipe(pipe_slow);
5564 %}
5565
5566 // ---------------------------- Vector mask generation ---------------------------
5567 // The rules below set predicate registers. They can guarantee the high bits of dst
5568 // are cleared with zero when the vector length is less than the full size of
5569 // hardware vector register width.
5570
5571
5572 // maskAll (full or partial predicate size)
5573
5574 instruct vmaskAll_immI(pRegGov dst, immI src) %{
5575 predicate(UseSVE > 0);
5576 match(Set dst (MaskAll src));
5577 ins_cost(SVE_COST);
5578 format %{ "sve_ptrue_lanecnt/sve_pfalse $dst\t# mask all (sve) (B/H/S)" %}
5579 ins_encode %{
5580 int con = (int)$src$$constant;
5581 if (con == 0) {
5582 __ sve_pfalse(as_PRegister($dst$$reg));
5583 } else {
5584 assert(con == -1, "invalid constant value for mask");
5585 BasicType bt = Matcher::vector_element_basic_type(this);
5586 __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
5587 Matcher::vector_length(this));
5588 }
5589 %}
5590 ins_pipe(pipe_slow);
5591 %}
5592
5593 instruct vmaskAllI(pRegGov dst, iRegIorL2I src, vReg tmp, rFlagsReg cr) %{
5594 predicate(UseSVE > 0);
5595 match(Set dst (MaskAll src));
5596 effect(TEMP tmp, KILL cr);
5597 ins_cost(3 * SVE_COST);
5598 format %{ "sve_dup $tmp, $src\n\t"
5599 "sve_ptrue $dst, vector_length\n\t"
5600 "sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) (B/H/S)" %}
5601 ins_encode %{
5602 BasicType bt = Matcher::vector_element_basic_type(this);
5603 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5604 uint length_in_bytes = Matcher::vector_length_in_bytes(this);
5605 __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
5606 if (length_in_bytes < MaxVectorSize) {
5607 __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), size, Matcher::vector_length(this));
5608 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size,
5609 as_PRegister($dst$$reg), as_FloatRegister($tmp$$reg), 0);
5610 } else {
5611 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
5612 }
5613 %}
5614 ins_pipe(pipe_slow);
5615 %}
5616
5617 instruct vmaskAll_immL(pRegGov dst, immL src) %{
5618 predicate(UseSVE > 0);
5619 match(Set dst (MaskAll src));
5620 ins_cost(SVE_COST);
5621 format %{ "sve_ptrue_lanecnt/sve_pfalse $dst\t# mask all (sve) (D)" %}
5622 ins_encode %{
5623 long con = (long)$src$$constant;
5624 if (con == 0) {
5625 __ sve_pfalse(as_PRegister($dst$$reg));
5626 } else {
5627 assert(con == -1, "invalid constant value for mask");
5628 BasicType bt = Matcher::vector_element_basic_type(this);
5629 __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
5630 Matcher::vector_length(this));
5631 }
5632 %}
5633 ins_pipe(pipe_slow);
5634 %}
5635
5636 instruct vmaskAllL(pRegGov dst, iRegL src, vReg tmp, rFlagsReg cr) %{
5637 predicate(UseSVE > 0);
5638 match(Set dst (MaskAll src));
5639 effect(TEMP tmp, KILL cr);
5640 ins_cost(3 * SVE_COST);
5641 format %{ "sve_dup $tmp, $src\n\t"
5642 "sve_ptrue $dst, vector_length\n\t"
5643 "sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) (D)" %}
5644 ins_encode %{
5645 BasicType bt = Matcher::vector_element_basic_type(this);
5646 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5647 uint length_in_bytes = Matcher::vector_length_in_bytes(this);
5648 __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
5649 if (length_in_bytes < MaxVectorSize) {
5650 __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), size, Matcher::vector_length(this));
5651 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size,
5652 as_PRegister($dst$$reg), as_FloatRegister($tmp$$reg), 0);
5653 } else {
5654 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
5655 }
5656 %}
5657 ins_pipe(pipe_slow);
5658 %}
5659
5660 // vector mask compare
5661
5662 instruct vmaskcmp(pRegGov dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %{
5663 predicate(UseSVE > 0);
5664 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
5665 effect(KILL cr);
5666 ins_cost(2 * SVE_COST);
5667 format %{ "sve_cmp $dst, $src1, $src2\t# vector mask cmp (sve)" %}
5668 ins_encode %{
5669 uint length_in_bytes = Matcher::vector_length_in_bytes(this);
5670 BasicType bt = Matcher::vector_element_basic_type(this);
5671 if (length_in_bytes == MaxVectorSize) {
5672 __ sve_compare(as_PRegister($dst$$reg), bt, ptrue, as_FloatRegister($src1$$reg),
5673 as_FloatRegister($src2$$reg), (int)$cond$$constant);
5674 } else {
5675 __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
5676 Matcher::vector_length(this));
5677 __ sve_compare(as_PRegister($dst$$reg), bt, as_PRegister($dst$$reg), as_FloatRegister($src1$$reg),
5678 as_FloatRegister($src2$$reg), (int)$cond$$constant);
5679 }
5680 %}
5681 ins_pipe(pipe_slow);
5682 %}
5683
5684 instruct vmaskcmp_masked(pRegGov dst, vReg src1, vReg src2, immI cond, pRegGov pg, rFlagsReg cr) %{
5685 predicate(UseSVE > 0);
5686 match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond pg)));
5687 effect(KILL cr);
5688 ins_cost(SVE_COST);
5689 format %{ "sve_cmp $dst, $pg, $src1, $src2\t# vector mask cmp (sve)" %}
5690 ins_encode %{
5691 BasicType bt = Matcher::vector_element_basic_type(this);
5692 __ sve_compare(as_PRegister($dst$$reg), bt, as_PRegister($pg$$reg), as_FloatRegister($src1$$reg),
5693 as_FloatRegister($src2$$reg), (int)$cond$$constant);
5694 %}
5695 ins_pipe(pipe_slow);
5696 %}
5697
5698 // vector load mask
5699
5700 instruct vloadmaskB(pRegGov dst, vReg src, rFlagsReg cr) %{
5701 predicate(UseSVE > 0 &&
5702 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
5703 match(Set dst (VectorLoadMask src));
5704 effect(KILL cr);
5705 ins_cost(SVE_COST);
5706 format %{ "vloadmaskB $dst, $src\t# vector load mask (sve) (B)" %}
5707 ins_encode %{
5708 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ B,
5709 ptrue, as_FloatRegister($src$$reg), 0);
5710 %}
5711 ins_pipe(pipe_slow);
5712 %}
5713
5714 instruct vloadmask_extend(pRegGov dst, vReg src, vReg tmp, rFlagsReg cr) %{
5715 predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() != T_BYTE);
5716 match(Set dst (VectorLoadMask src));
5717 effect(TEMP tmp, KILL cr);
5718 ins_cost(3 * SVE_COST);
5719 format %{ "vloadmask $dst, $src\t# vector load mask (sve) (H/S/D)" %}
5720 ins_encode %{
5721 BasicType bt = Matcher::vector_element_basic_type(this);
5722 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5723 __ sve_vector_extend(as_FloatRegister($tmp$$reg), size, as_FloatRegister($src$$reg), __ B);
5724 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
5725 %}
5726 ins_pipe(pipe_slow);
5727 %}
5728
5729 // ---------------------------- Compress/Expand Operations ---------------------------
5730
5731 instruct mcompress(pReg dst, pReg pg, rFlagsReg cr) %{
5732 predicate(UseSVE > 0);
5733 match(Set dst (CompressM pg));
5734 effect(KILL cr);
5735 ins_cost(2 * SVE_COST);
5736 format %{ "sve_cntp rscratch1, $pg\n\t"
5737 "sve_whilelo $dst, zr, rscratch1\t# mask compress (B/H/S/D)" %}
5738 ins_encode %{
5739 BasicType bt = Matcher::vector_element_basic_type(this);
5740 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5741 __ sve_cntp(rscratch1, size, ptrue, as_PRegister($pg$$reg));
5742 __ sve_whilelo(as_PRegister($dst$$reg), size, zr, rscratch1);
5743 %}
5744 ins_pipe(pipe_slow);
5745 %}
5746
5747 instruct vcompress(vReg dst, vReg src, pRegGov pg) %{
5748 predicate(UseSVE > 0 &&
5749 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
5750 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT ||
5751 n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
5752 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
5753 match(Set dst (CompressV src pg));
5754 ins_cost(SVE_COST);
5755 format %{ "sve_compact $dst, $src, $pg\t# vector compress (S/D)" %}
5756 ins_encode %{
5757 BasicType bt = Matcher::vector_element_basic_type(this);
5758 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5759 __ sve_compact(as_FloatRegister($dst$$reg), size, as_FloatRegister($src$$reg), as_PRegister($pg$$reg));
5760 %}
5761 ins_pipe(pipe_slow);
5762 %}
5763
5764 instruct vcompressB(vReg dst, vReg src, pReg pg, vReg vtmp1, vReg vtmp2, vReg vtmp3, vReg vtmp4,
5765 pReg ptmp, pRegGov pgtmp) %{
5766 predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
5767 effect(TEMP_DEF dst, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP ptmp, TEMP pgtmp);
5768 match(Set dst (CompressV src pg));
5769 ins_cost(13 * SVE_COST);
5770 format %{ "sve_compact $dst, $src, $pg\t# vector compress (B)" %}
5771 ins_encode %{
5772 __ sve_compress_byte(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
5773 as_FloatRegister($vtmp1$$reg),as_FloatRegister($vtmp2$$reg),
5774 as_FloatRegister($vtmp3$$reg),as_FloatRegister($vtmp4$$reg),
5775 as_PRegister($ptmp$$reg), as_PRegister($pgtmp$$reg));
5776 %}
5777 ins_pipe(pipe_slow);
5778 %}
5779
5780 instruct vcompressS(vReg dst, vReg src, pReg pg, vReg vtmp1, vReg vtmp2, pRegGov pgtmp) %{
5781 predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
5782 effect(TEMP_DEF dst, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp);
5783 match(Set dst (CompressV src pg));
5784 ins_cost(38 * SVE_COST);
5785 format %{ "sve_compact $dst, $src, $pg\t# vector compress (H)" %}
5786 ins_encode %{
5787 __ sve_compress_short(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
5788 as_FloatRegister($vtmp1$$reg),as_FloatRegister($vtmp2$$reg), as_PRegister($pgtmp$$reg));
5789 %}
5790 ins_pipe(pipe_slow);
5791 %}
5792
5793 instruct vexpand(vReg dst, vReg src, pRegGov pg) %{
5794 match(Set dst (ExpandV src pg));
5795 effect(TEMP_DEF dst);
5796 ins_cost(4 * SVE_COST);
5797 format %{ "sve_dup $dst, S/D, 0\n\t"
5798 "sve_histcnt $dst, S/D, $pg, $dst, $dst\n\t"
5799 "sve_sub $dst, S/D, 1\n\t"
5800 "sve_tbl $dst, S/D, $src, $dst\t# vector expand (S/D)" %}
5801 ins_encode %{
5802 // Example input: src = 1 2 3 4 5 6 7 8
5803 // pg = 1 0 0 1 1 0 1 1
5804 // Expected result: dst = 4 0 0 5 6 0 7 8
5805
5806 // The basic idea is to use TBL which can shuffle the elements in the given
5807 // vector flexibly. HISTCNT + SUB is used to generate the second source input
5808 // for TBL whose value is used to select the indexed element from src vector.
5809
5810 BasicType bt = Matcher::vector_element_basic_type(this);
5811 assert(UseSVE == 2 && !is_subword_type(bt), "unsupported");
5812 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5813 // dst = 0 0 0 0 0 0 0 0
5814 __ sve_dup(as_FloatRegister($dst$$reg), size, 0);
5815 // dst = 5 0 0 4 3 0 2 1
5816 __ sve_histcnt(as_FloatRegister($dst$$reg), size, as_PRegister($pg$$reg),
5817 as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg));
5818 // dst = 4 -1 -1 3 2 -1 1 0
5819 __ sve_sub(as_FloatRegister($dst$$reg), size, 1);
5820 // dst = 4 0 0 5 6 0 7 8
5821 __ sve_tbl(as_FloatRegister($dst$$reg), size, as_FloatRegister($src$$reg),
5822 as_FloatRegister($dst$$reg));
5823 %}
5824 ins_pipe(pipe_slow);
5825 %}
5826
5827 instruct vmask_gen(pRegGov pg, iRegL len, rFlagsReg cr) %{
5828 predicate(UseSVE > 0);
5829 match(Set pg (VectorMaskGen len));
5830 effect(KILL cr);
5831 ins_cost(SVE_COST);
5832 format %{ "sve_whilelo $pg, zr, $len\t # sve" %}
5833 ins_encode %{
5834 BasicType bt = Matcher::vector_element_basic_type(this);
5835 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5836 __ sve_whilelo(as_PRegister($pg$$reg), size, zr, as_Register($len$$reg));
5837 %}
5838 ins_pipe(pipe_slow);
5839 %}
5840
5841 // ------------------------------ CountLeadingZerosV ------------------------------
5842
5843 instruct vcountLeadingZeros(vReg dst, vReg src) %{
5844 predicate(UseSVE > 0 &&
5845 !n->as_Vector()->is_predicated_vector());
5846 match(Set dst (CountLeadingZerosV src));
5847 ins_cost(SVE_COST);
5848 format %{ "sve_clz $dst, $src\t# vector (sve)" %}
5849 ins_encode %{
5850 BasicType bt = Matcher::vector_element_basic_type(this);
5851 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5852 __ sve_clz(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($src$$reg));
5853 %}
5854 ins_pipe(pipe_slow);
5855 %}
5856
5857 // The dst and src should use the same register to make sure the
5858 // inactive lanes in dst save the same elements as src.
5859 instruct vcountLeadingZeros_masked(vReg dst_src, pRegGov pg) %{
5860 predicate(UseSVE > 0);
5861 match(Set dst_src (CountLeadingZerosV dst_src pg));
5862 ins_cost(SVE_COST);
5863 format %{ "sve_clz $dst_src, $pg, $dst_src\t# vector (sve)" %}
5864 ins_encode %{
5865 BasicType bt = Matcher::vector_element_basic_type(this);
5866 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5867 __ sve_clz(as_FloatRegister($dst_src$$reg), size,
5868 as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
5869 %}
5870 ins_pipe(pipe_slow);
5871 %}
5872
5873 // ------------------------------ CountTrailingZerosV -----------------------------
5874
5875 instruct vcountTrailingZeros(vReg dst, vReg src) %{
5876 predicate(UseSVE > 0 &&
5877 !n->as_Vector()->is_predicated_vector());
5878 match(Set dst (CountTrailingZerosV src));
5879 ins_cost(2 * SVE_COST);
5880 format %{ "sve_rbit $dst, $src\n\t"
5881 "sve_clz $dst, $dst\t# vector (sve)" %}
5882 ins_encode %{
5883 BasicType bt = Matcher::vector_element_basic_type(this);
5884 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5885 __ sve_rbit(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($src$$reg));
5886 __ sve_clz(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($dst$$reg));
5887 %}
5888 ins_pipe(pipe_slow);
5889 %}
5890
5891 // The dst and src should use the same register to make sure the
5892 // inactive lanes in dst save the same elements as src.
5893 instruct vcountTrailingZeros_masked(vReg dst_src, pRegGov pg) %{
5894 predicate(UseSVE > 0);
5895 match(Set dst_src (CountTrailingZerosV dst_src pg));
5896 ins_cost(2 * SVE_COST);
5897 format %{ "sve_rbit $dst_src, $pg, $dst_src\n\t"
5898 "sve_clz $dst_src, $pg, $dst_src\t# vector (sve)" %}
5899 ins_encode %{
5900 BasicType bt = Matcher::vector_element_basic_type(this);
5901 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5902 __ sve_rbit(as_FloatRegister($dst_src$$reg), size,
5903 as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
5904 __ sve_clz(as_FloatRegister($dst_src$$reg), size,
5905 as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
5906 %}
5907 ins_pipe(pipe_slow);
5908 %}
5909
5910 // ---------------------------------- ReverseV ------------------------------------
5911
5912 instruct vreverse(vReg dst, vReg src) %{
5913 predicate(UseSVE > 0 &&
5914 !n->as_Vector()->is_predicated_vector());
5915 match(Set dst (ReverseV src));
5916 ins_cost(SVE_COST);
5917 format %{ "sve_rbit $dst, $src\t# vector (sve)" %}
5918 ins_encode %{
5919 BasicType bt = Matcher::vector_element_basic_type(this);
5920 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5921 __ sve_rbit(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($src$$reg));
5922 %}
5923 ins_pipe(pipe_slow);
5924 %}
5925
5926 // The dst and src should use the same register to make sure the
5927 // inactive lanes in dst save the same elements as src.
5928 instruct vreverse_masked(vReg dst_src, pRegGov pg) %{
5929 predicate(UseSVE > 0);
5930 match(Set dst_src (ReverseV dst_src pg));
5931 ins_cost(SVE_COST);
5932 format %{ "sve_rbit $dst_src, $pg, $dst_src\t# vector (sve)" %}
5933 ins_encode %{
5934 BasicType bt = Matcher::vector_element_basic_type(this);
5935 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5936 __ sve_rbit(as_FloatRegister($dst_src$$reg), size,
5937 as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
5938 %}
5939 ins_pipe(pipe_slow);
5940 %}
5941
5942 // -------------------------------- ReverseBytesV ---------------------------------
5943
5944 instruct vreverseBytes(vReg dst, vReg src) %{
5945 predicate(UseSVE > 0 &&
5946 !n->as_Vector()->is_predicated_vector());
5947 match(Set dst (ReverseBytesV src));
5948 ins_cost(SVE_COST);
5949 format %{ "sve_revb $dst, $src\t# vector (sve)" %}
5950 ins_encode %{
5951 BasicType bt = Matcher::vector_element_basic_type(this);
5952 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5953 if (bt == T_BYTE) {
5954 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
5955 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
5956 }
5957 } else {
5958 __ sve_revb(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($src$$reg));
5959 }
5960 %}
5961 ins_pipe(pipe_slow);
5962 %}
5963
5964 // The dst and src should use the same register to make sure the
5965 // inactive lanes in dst save the same elements as src.
5966 instruct vreverseBytes_masked(vReg dst_src, pRegGov pg) %{
5967 predicate(UseSVE > 0);
5968 match(Set dst_src (ReverseBytesV dst_src pg));
5969 ins_cost(SVE_COST);
5970 format %{ "sve_revb $dst_src, $pg, $dst_src\t# vector (sve)" %}
5971 ins_encode %{
5972 BasicType bt = Matcher::vector_element_basic_type(this);
5973 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
5974 if (bt == T_BYTE) {
5975 // do nothing
5976 } else {
5977 __ sve_revb(as_FloatRegister($dst_src$$reg), size,
5978 as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
5979 }
5980 %}
5981 ins_pipe(pipe_slow);
5982 %}
5983