66 constraint(ALLOC_IN_RC(ptr_reg));
67 match(AddP reg off);
68 op_cost(0);
69 format %{ "[$reg, $off]" %}
70 interface(MEMORY_INTER) %{
71 base($reg);
72 `index'(0xffffffff);
73 scale(0x0);
74 disp($off);
75 %}
76 %}')dnl
77 OPERAND_VMEMORYA_INDIRECT_OFFSET(I, 4)
78 OPERAND_VMEMORYA_INDIRECT_OFFSET(L, 4)
79
80 // The indOff of vmemA is valid only when the vector element (load to/store from)
81 // size equals to memory element (load from/store to) size.
82 opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);
83
84 source_hpp %{
85 bool op_sve_supported(int opcode, int vlen, BasicType bt);
86 %}
87
88 source %{
89
90 typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
91 PRegister Pg, const Address &adr);
92
93 // Predicated load/store, with optional ptrue to all elements of given predicate register.
94 static void loadStoreA_predicated(C2_MacroAssembler masm, bool is_store, FloatRegister reg,
95 PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
96 int opcode, Register base, int index, int size, int disp) {
97 sve_mem_insn_predicate insn;
98 int mesize = type2aelembytes(mem_elem_bt);
99 if (index == -1) {
100 assert(size == 0, "unsupported address mode: scale size = %d", size);
101 switch(mesize) {
102 case 1:
103 insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
104 break;
105 case 2:
122 ShouldNotReachHere();
123 }
124 }
125
126 bool op_sve_supported(int opcode, int vlen, BasicType bt) {
127 int length_in_bytes = vlen * type2aelembytes(bt);
128 switch (opcode) {
129 case Op_MulAddVS2VI:
130 // No multiply reduction instructions
131 case Op_MulReductionVD:
132 case Op_MulReductionVF:
133 case Op_MulReductionVI:
134 case Op_MulReductionVL:
135 // Others
136 case Op_ExtractC:
137 case Op_ExtractUB:
138 return false;
139 // Vector API specific
140 case Op_VectorLoadShuffle:
141 case Op_VectorRearrange:
142 if (vlen < 4 || length_in_bytes > MaxVectorSize) {
143 return false;
144 } else {
145 return true;
146 }
147 case Op_LoadVector:
148 case Op_StoreVector:
149 return Matcher::vector_size_supported(bt, vlen);
150 default:
151 break;
152 }
153 // By default, we only support vector operations with no less than 8 bytes and 2 elements.
154 return 8 <= length_in_bytes && length_in_bytes <= MaxVectorSize && vlen >= 2;
155 }
156 %}
157
158 definitions %{
159 int_def SVE_COST (200, 200);
160 %}
161
162 dnl
163 dnl ELEMENT_SHORT_CHART($1, $2)
164 dnl ELEMENT_SHORT_CHART(etype, node)
165 define(`ELEMENT_SHORT_CHAR',`ifelse(`$1', `T_SHORT',
166 `($2->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
167 ($2->bottom_type()->is_vect()->element_basic_type() == T_CHAR))',
168 `($2->bottom_type()->is_vect()->element_basic_type() == $1)')')dnl
169 dnl
170
171 // All SVE instructions
172
173 // vector load/store
174
175 // Unpredicated vector load/store
214 match(Set ifelse(load, $3, dst (LoadVector mem), mem (StoreVector mem src)));
215 ins_cost(4 * INSN_COST);
216 format %{ "$1 ifelse(load, $3, `$dst,$mem', `$mem,$src')\t# vector ($6 bits)" %}
217 ins_encode( `aarch64_enc_'ifelse(load, $3, ldr, str)v$2($7, mem) );
218 ins_pipe(v$3`_reg_mem'ifelse(eval($4 * 8), 128, 128, 64));
219 %}')dnl
220 dnl $1 $2 $3 $4 $5 $6 $7
221 VLoadStore(ldrh, H, load, 2, D, 16, dst)
222 VLoadStore(strh, H, store, 2, D, 16, src)
223 VLoadStore(ldrs, S, load, 4, D, 32, dst)
224 VLoadStore(strs, S, store, 4, D, 32, src)
225 VLoadStore(ldrd, D, load, 8, D, 64, dst)
226 VLoadStore(strd, D, store, 8, D, 64, src)
227 VLoadStore(ldrq, Q, load, 16, X, 128, dst)
228 VLoadStore(strq, Q, store, 16, X, 128, src)
229
230 // Predicated vector load/store, based on the vector length of the node.
231 // Only load/store values in the range of the memory_size. This is needed
232 // when the memory_size is lower than the hardware supported max vector size.
233 // And this might happen for Vector API mask vector load/store.
234 instruct loadV_partial(vReg dst, vmemA mem, pRegGov pTmp, rFlagsReg cr) %{
235 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() > 16 &&
236 n->as_LoadVector()->memory_size() < MaxVectorSize);
237 match(Set dst (LoadVector mem));
238 effect(TEMP pTmp, KILL cr);
239 ins_cost(6 * SVE_COST);
240 format %{ "sve_whilelo_zr_imm $pTmp, vector_length\n\t"
241 "sve_ldr $dst, $pTmp, $mem\t# load vector predicated" %}
242 ins_encode %{
243 BasicType bt = Matcher::vector_element_basic_type(this);
244 __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), __ elemType_to_regVariant(bt),
245 Matcher::vector_length(this));
246 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
247 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg,
248 as_PRegister($pTmp$$reg), bt, bt, $mem->opcode(),
249 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
250 %}
251 ins_pipe(pipe_slow);
252 %}
253
254 instruct storeV_partial(vReg src, vmemA mem, pRegGov pTmp, rFlagsReg cr) %{
255 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() > 16 &&
256 n->as_StoreVector()->memory_size() < MaxVectorSize);
257 match(Set mem (StoreVector mem src));
258 effect(TEMP pTmp, KILL cr);
259 ins_cost(5 * SVE_COST);
260 format %{ "sve_whilelo_zr_imm $pTmp, vector_length\n\t"
261 "sve_str $src, $pTmp, $mem\t# store vector predicated" %}
262 ins_encode %{
263 BasicType bt = Matcher::vector_element_basic_type(this, $src);
264 __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), __ elemType_to_regVariant(bt),
265 Matcher::vector_length(this, $src));
266 FloatRegister src_reg = as_FloatRegister($src$$reg);
267 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg,
268 as_PRegister($pTmp$$reg), bt, bt, $mem->opcode(),
269 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
270 %}
271 ins_pipe(pipe_slow);
272 %}dnl
273
274
275 // vector reinterpret
276
277 instruct reinterpret(vReg dst) %{
278 predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() ==
279 n->in(1)->bottom_type()->is_vect()->length_in_bytes()); // src == dst
280 match(Set dst (VectorReinterpret dst));
281 ins_cost(0);
282 format %{ "# reinterpret $dst\t# do nothing" %}
283 ins_encode %{
284 // empty
285 %}
286 ins_pipe(pipe_class_empty);
287 %}
288
289 instruct reinterpretResize(vReg dst, vReg src, pRegGov pTmp, rFlagsReg cr) %{
290 predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() !=
291 n->in(1)->bottom_type()->is_vect()->length_in_bytes()); // src != dst
292 match(Set dst (VectorReinterpret src));
293 effect(TEMP_DEF dst, TEMP pTmp, KILL cr);
294 ins_cost(3 * SVE_COST);
295 format %{ "reinterpretResize $dst, $src\t# vector (sve)" %}
296 ins_encode %{
297 uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
298 uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
299 uint length_in_bytes_resize = length_in_bytes_src < length_in_bytes_dst ?
300 length_in_bytes_src : length_in_bytes_dst;
301 assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
302 "invalid vector length");
303 __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), __ B, length_in_bytes_resize);
304 __ sve_dup(as_FloatRegister($dst$$reg), __ B, 0);
305 __ sve_sel(as_FloatRegister($dst$$reg), __ B, as_PRegister($pTmp$$reg),
306 as_FloatRegister($src$$reg), as_FloatRegister($dst$$reg));
307 %}
308 ins_pipe(pipe_slow);
309 %}
310 dnl
311 dnl UNARY_OP_TRUE_PREDICATE_ETYPE($1, $2, $3, $4, $5, %6 )
312 dnl UNARY_OP_TRUE_PREDICATE_ETYPE(insn_name, op_name, element_type, size, min_vec_len, insn)
313 define(`UNARY_OP_TRUE_PREDICATE_ETYPE', `
314 instruct $1(vReg dst, vReg src) %{
315 predicate(UseSVE > 0 &&
316 n->bottom_type()->is_vect()->element_basic_type() == $3);
317 match(Set dst ($2 src));
318 ins_cost(SVE_COST);
319 format %{ "$6 $dst, $src\t# vector (sve) ($4)" %}
320 ins_encode %{
321 __ $6(as_FloatRegister($dst$$reg), __ $4,
322 ptrue, as_FloatRegister($src$$reg));
323 %}
324 ins_pipe(pipe_slow);
325 %}')dnl
326 dnl
327
328 // vector abs
329 UNARY_OP_TRUE_PREDICATE_ETYPE(vabsB, AbsVB, T_BYTE, B, 16, sve_abs)
330 UNARY_OP_TRUE_PREDICATE_ETYPE(vabsS, AbsVS, T_SHORT, H, 8, sve_abs)
331 UNARY_OP_TRUE_PREDICATE_ETYPE(vabsI, AbsVI, T_INT, S, 4, sve_abs)
332 UNARY_OP_TRUE_PREDICATE_ETYPE(vabsL, AbsVL, T_LONG, D, 2, sve_abs)
333 UNARY_OP_TRUE_PREDICATE_ETYPE(vabsF, AbsVF, T_FLOAT, S, 4, sve_fabs)
334 UNARY_OP_TRUE_PREDICATE_ETYPE(vabsD, AbsVD, T_DOUBLE, D, 2, sve_fabs)
335 dnl
336 dnl BINARY_OP_UNPREDICATED($1, $2 $3, $4 $5 )
337 dnl BINARY_OP_UNPREDICATED(insn_name, op_name, size, min_vec_len, insn)
338 define(`BINARY_OP_UNPREDICATED', `
339 instruct $1(vReg dst, vReg src1, vReg src2) %{
340 predicate(UseSVE > 0);
341 match(Set dst ($2 src1 src2));
342 ins_cost(SVE_COST);
343 format %{ "$5 $dst, $src1, $src2\t # vector (sve) ($3)" %}
344 ins_encode %{
345 __ $5(as_FloatRegister($dst$$reg), __ $3,
346 as_FloatRegister($src1$$reg),
347 as_FloatRegister($src2$$reg));
348 %}
349 ins_pipe(pipe_slow);
350 %}')dnl
351
352 // vector add
353 BINARY_OP_UNPREDICATED(vaddB, AddVB, B, 16, sve_add)
354 BINARY_OP_UNPREDICATED(vaddS, AddVS, H, 8, sve_add)
355 BINARY_OP_UNPREDICATED(vaddI, AddVI, S, 4, sve_add)
356 BINARY_OP_UNPREDICATED(vaddL, AddVL, D, 2, sve_add)
357 BINARY_OP_UNPREDICATED(vaddF, AddVF, S, 4, sve_fadd)
358 BINARY_OP_UNPREDICATED(vaddD, AddVD, D, 2, sve_fadd)
359 dnl
360 dnl BINARY_OP_UNSIZED($1, $2, $3, $4 )
361 dnl BINARY_OP_UNSIZED(insn_name, op_name, min_vec_len, insn)
362 define(`BINARY_OP_UNSIZED', `
363 instruct $1(vReg dst, vReg src1, vReg src2) %{
364 predicate(UseSVE > 0);
365 match(Set dst ($2 src1 src2));
366 ins_cost(SVE_COST);
367 format %{ "$4 $dst, $src1, $src2\t# vector (sve)" %}
368 ins_encode %{
369 __ $4(as_FloatRegister($dst$$reg),
370 as_FloatRegister($src1$$reg),
371 as_FloatRegister($src2$$reg));
372 %}
373 ins_pipe(pipe_slow);
374 %}')dnl
375
376 // vector and
377 BINARY_OP_UNSIZED(vand, AndV, 16, sve_and)
378
379 // vector or
380 BINARY_OP_UNSIZED(vor, OrV, 16, sve_orr)
381
382 // vector xor
383 BINARY_OP_UNSIZED(vxor, XorV, 16, sve_eor)
384
385 // vector not
386 dnl
387 define(`MATCH_RULE', `ifelse($1, I,
388 `match(Set dst (XorV src (ReplicateB m1)));
389 match(Set dst (XorV src (ReplicateS m1)));
390 match(Set dst (XorV src (ReplicateI m1)));',
391 `match(Set dst (XorV src (ReplicateL m1)));')')dnl
392 dnl
393 define(`VECTOR_NOT', `
394 instruct vnot$1`'(vReg dst, vReg src, imm$1_M1 m1) %{
395 predicate(UseSVE > 0);
396 MATCH_RULE($1)
397 ins_cost(SVE_COST);
398 format %{ "sve_not $dst, $src\t# vector (sve) $2" %}
399 ins_encode %{
400 __ sve_not(as_FloatRegister($dst$$reg), __ D,
401 ptrue, as_FloatRegister($src$$reg));
402 %}
403 ins_pipe(pipe_slow);
404 %}')dnl
405 dnl $1,$2
406 VECTOR_NOT(I, B/H/S)
407 VECTOR_NOT(L, D)
408 undefine(MATCH_RULE)
409
410 // vector and_not
411 dnl
412 define(`MATCH_RULE', `ifelse($1, I,
413 `match(Set dst (AndV src1 (XorV src2 (ReplicateB m1))));
414 match(Set dst (AndV src1 (XorV src2 (ReplicateS m1))));
415 match(Set dst (AndV src1 (XorV src2 (ReplicateI m1))));',
416 `match(Set dst (AndV src1 (XorV src2 (ReplicateL m1))));')')dnl
417 dnl
418 define(`VECTOR_AND_NOT', `
419 instruct vand_not$1`'(vReg dst, vReg src1, vReg src2, imm$1_M1 m1) %{
420 predicate(UseSVE > 0);
421 MATCH_RULE($1)
422 ins_cost(SVE_COST);
423 format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) $2" %}
424 ins_encode %{
425 __ sve_bic(as_FloatRegister($dst$$reg),
426 as_FloatRegister($src1$$reg),
427 as_FloatRegister($src2$$reg));
428 %}
429 ins_pipe(pipe_slow);
430 %}')dnl
431 dnl $1,$2
432 VECTOR_AND_NOT(I, B/H/S)
433 VECTOR_AND_NOT(L, D)
434 undefine(MATCH_RULE)
435 dnl
436 dnl VDIVF($1, $2 , $3 )
437 dnl VDIVF(name_suffix, size, min_vec_len)
438 define(`VDIVF', `
439 instruct vdiv$1(vReg dst_src1, vReg src2) %{
440 predicate(UseSVE > 0);
441 match(Set dst_src1 (DivV$1 dst_src1 src2));
442 ins_cost(SVE_COST);
443 format %{ "sve_fdiv $dst_src1, $dst_src1, $src2\t# vector (sve) ($2)" %}
444 ins_encode %{
445 __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ $2,
446 ptrue, as_FloatRegister($src2$$reg));
447 %}
448 ins_pipe(pipe_slow);
449 %}')dnl
450
451 // vector float div
452 VDIVF(F, S, 4)
453 VDIVF(D, D, 2)
454
455 // vector min/max
456
457 instruct vmin(vReg dst_src1, vReg src2) %{
458 predicate(UseSVE > 0);
459 match(Set dst_src1 (MinV dst_src1 src2));
460 ins_cost(SVE_COST);
461 format %{ "sve_min $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
462 ins_encode %{
463 BasicType bt = Matcher::vector_element_basic_type(this);
464 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
465 if (is_floating_point_type(bt)) {
466 __ sve_fmin(as_FloatRegister($dst_src1$$reg), size,
467 ptrue, as_FloatRegister($src2$$reg));
468 } else {
469 assert(is_integral_type(bt), "Unsupported type");
470 __ sve_smin(as_FloatRegister($dst_src1$$reg), size,
471 ptrue, as_FloatRegister($src2$$reg));
472 }
473 %}
474 ins_pipe(pipe_slow);
475 %}
476
477 instruct vmax(vReg dst_src1, vReg src2) %{
478 predicate(UseSVE > 0);
479 match(Set dst_src1 (MaxV dst_src1 src2));
480 ins_cost(SVE_COST);
481 format %{ "sve_max $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
482 ins_encode %{
483 BasicType bt = Matcher::vector_element_basic_type(this);
484 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
485 if (is_floating_point_type(bt)) {
486 __ sve_fmax(as_FloatRegister($dst_src1$$reg), size,
487 ptrue, as_FloatRegister($src2$$reg));
488 } else {
489 assert(is_integral_type(bt), "Unsupported type");
490 __ sve_smax(as_FloatRegister($dst_src1$$reg), size,
491 ptrue, as_FloatRegister($src2$$reg));
492 }
493 %}
494 ins_pipe(pipe_slow);
495 %}
496
497 dnl
498 dnl VFMLA($1 $2 $3 )
499 dnl VFMLA(name_suffix, size, min_vec_len)
500 define(`VFMLA', `
501 // dst_src1 = dst_src1 + src2 * src3
502 instruct vfmla$1(vReg dst_src1, vReg src2, vReg src3) %{
503 predicate(UseFMA && UseSVE > 0);
504 match(Set dst_src1 (FmaV$1 dst_src1 (Binary src2 src3)));
505 ins_cost(SVE_COST);
506 format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
507 ins_encode %{
508 __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ $2,
509 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
510 %}
511 ins_pipe(pipe_slow);
512 %}')dnl
513 dnl
514 // vector fmla
515 VFMLA(F, S, 4)
516 VFMLA(D, D, 2)
517
518 dnl
519 dnl VFMLS($1 $2 $3 )
520 dnl VFMLS(name_suffix, size, min_vec_len)
521 define(`VFMLS', `
522 // dst_src1 = dst_src1 + -src2 * src3
523 // dst_src1 = dst_src1 + src2 * -src3
524 instruct vfmls$1(vReg dst_src1, vReg src2, vReg src3) %{
525 predicate(UseFMA && UseSVE > 0);
526 match(Set dst_src1 (FmaV$1 dst_src1 (Binary (NegV$1 src2) src3)));
527 match(Set dst_src1 (FmaV$1 dst_src1 (Binary src2 (NegV$1 src3))));
528 ins_cost(SVE_COST);
529 format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
530 ins_encode %{
531 __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ $2,
532 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
533 %}
534 ins_pipe(pipe_slow);
535 %}')dnl
536 dnl
537 // vector fmls
628 VMLS(B, B, 16)
629 VMLS(S, H, 8)
630 VMLS(I, S, 4)
631 VMLS(L, D, 2)
632
633 dnl
634 dnl BINARY_OP_TRUE_PREDICATE($1, $2, $3, $4, $5 )
635 dnl BINARY_OP_TRUE_PREDICATE(insn_name, op_name, size, min_vec_len, insn)
636 define(`BINARY_OP_TRUE_PREDICATE', `
637 instruct $1(vReg dst_src1, vReg src2) %{
638 predicate(UseSVE > 0);
639 match(Set dst_src1 ($2 dst_src1 src2));
640 ins_cost(SVE_COST);
641 format %{ "$5 $dst_src1, $dst_src1, $src2\t # vector (sve) ($3)" %}
642 ins_encode %{
643 __ $5(as_FloatRegister($dst_src1$$reg), __ $3,
644 ptrue, as_FloatRegister($src2$$reg));
645 %}
646 ins_pipe(pipe_slow);
647 %}')dnl
648
649 // vector mul
650 BINARY_OP_TRUE_PREDICATE(vmulB, MulVB, B, 16, sve_mul)
651 BINARY_OP_TRUE_PREDICATE(vmulS, MulVS, H, 8, sve_mul)
652 BINARY_OP_TRUE_PREDICATE(vmulI, MulVI, S, 4, sve_mul)
653 BINARY_OP_TRUE_PREDICATE(vmulL, MulVL, D, 2, sve_mul)
654 BINARY_OP_UNPREDICATED(vmulF, MulVF, S, 4, sve_fmul)
655 BINARY_OP_UNPREDICATED(vmulD, MulVD, D, 2, sve_fmul)
656
657 dnl
658 dnl UNARY_OP_TRUE_PREDICATE($1, $2, $3, $4, $5 )
659 dnl UNARY_OP_TRUE_PREDICATE(insn_name, op_name, size, min_vec_bytes, insn)
660 define(`UNARY_OP_TRUE_PREDICATE', `
661 instruct $1(vReg dst, vReg src) %{
662 predicate(UseSVE > 0);
663 match(Set dst ($2 src));
664 ins_cost(SVE_COST);
665 format %{ "$5 $dst, $src\t# vector (sve) ($3)" %}
666 ins_encode %{
667 __ $5(as_FloatRegister($dst$$reg), __ $3,
668 ptrue, as_FloatRegister($src$$reg));
669 %}
670 ins_pipe(pipe_slow);
671 %}')dnl
672 dnl
673 // vector fneg
674 UNARY_OP_TRUE_PREDICATE(vnegF, NegVF, S, 16, sve_fneg)
675 UNARY_OP_TRUE_PREDICATE(vnegD, NegVD, D, 16, sve_fneg)
676
677 // popcount vector
678
679 instruct vpopcountI(vReg dst, vReg src) %{
680 predicate(UseSVE > 0);
681 match(Set dst (PopCountVI src));
682 format %{ "sve_cnt $dst, $src\t# vector (sve) (S)\n\t" %}
683 ins_encode %{
684 __ sve_cnt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg));
685 %}
686 ins_pipe(pipe_slow);
687 %}
688
689 // vector mask compare
690
691 instruct vmaskcmp(vReg dst, vReg src1, vReg src2, immI cond, pRegGov pTmp, rFlagsReg cr) %{
692 predicate(UseSVE > 0);
693 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
694 effect(TEMP pTmp, KILL cr);
695 ins_cost(2 * SVE_COST);
696 format %{ "sve_cmp $pTmp, $src1, $src2\n\t"
697 "sve_cpy $dst, $pTmp, -1\t# vector mask cmp (sve)" %}
698 ins_encode %{
699 BasicType bt = Matcher::vector_element_basic_type(this);
700 __ sve_compare(as_PRegister($pTmp$$reg), bt, ptrue, as_FloatRegister($src1$$reg),
701 as_FloatRegister($src2$$reg), (int)$cond$$constant);
702 __ sve_cpy(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
703 as_PRegister($pTmp$$reg), -1, false);
704 %}
705 ins_pipe(pipe_slow);
706 %}
707
708 // vector blend
709
710 instruct vblend(vReg dst, vReg src1, vReg src2, vReg src3, pRegGov pTmp, rFlagsReg cr) %{
711 predicate(UseSVE > 0);
712 match(Set dst (VectorBlend (Binary src1 src2) src3));
713 effect(TEMP pTmp, KILL cr);
714 ins_cost(2 * SVE_COST);
715 format %{ "sve_cmpeq $pTmp, $src3, -1\n\t"
716 "sve_sel $dst, $pTmp, $src2, $src1\t# vector blend (sve)" %}
717 ins_encode %{
718 Assembler::SIMD_RegVariant size =
719 __ elemType_to_regVariant(Matcher::vector_element_basic_type(this));
720 __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), size,
721 ptrue, as_FloatRegister($src3$$reg), -1);
722 __ sve_sel(as_FloatRegister($dst$$reg), size, as_PRegister($pTmp$$reg),
723 as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
724 %}
725 ins_pipe(pipe_slow);
726 %}
727
728 // vector blend with compare
729
730 instruct vblend_maskcmp(vReg dst, vReg src1, vReg src2, vReg src3,
731 vReg src4, pRegGov pTmp, immI cond, rFlagsReg cr) %{
732 predicate(UseSVE > 0);
733 match(Set dst (VectorBlend (Binary src1 src2) (VectorMaskCmp (Binary src3 src4) cond)));
734 effect(TEMP pTmp, KILL cr);
735 ins_cost(2 * SVE_COST);
736 format %{ "sve_cmp $pTmp, $src3, $src4\t# vector cmp (sve)\n\t"
737 "sve_sel $dst, $pTmp, $src2, $src1\t# vector blend (sve)" %}
738 ins_encode %{
739 BasicType bt = Matcher::vector_element_basic_type(this);
740 __ sve_compare(as_PRegister($pTmp$$reg), bt, ptrue, as_FloatRegister($src3$$reg),
741 as_FloatRegister($src4$$reg), (int)$cond$$constant);
742 __ sve_sel(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
743 as_PRegister($pTmp$$reg), as_FloatRegister($src2$$reg),
744 as_FloatRegister($src1$$reg));
745 %}
746 ins_pipe(pipe_slow);
747 %}
748
749 // vector load mask
750
751 instruct vloadmaskB(vReg dst, vReg src) %{
752 predicate(UseSVE > 0 &&
753 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
754 match(Set dst (VectorLoadMask src));
755 ins_cost(SVE_COST);
756 format %{ "sve_neg $dst, $src\t# vector load mask (B)" %}
757 ins_encode %{
758 __ sve_neg(as_FloatRegister($dst$$reg), __ B, ptrue, as_FloatRegister($src$$reg));
759 %}
760 ins_pipe(pipe_slow);
761 %}
762
763 instruct vloadmaskS(vReg dst, vReg src) %{
764 predicate(UseSVE > 0 &&
765 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
766 match(Set dst (VectorLoadMask src));
767 ins_cost(2 * SVE_COST);
768 format %{ "sve_uunpklo $dst, H, $src\n\t"
769 "sve_neg $dst, $dst\t# vector load mask (B to H)" %}
770 ins_encode %{
771 __ sve_uunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
772 __ sve_neg(as_FloatRegister($dst$$reg), __ H, ptrue, as_FloatRegister($dst$$reg));
773 %}
774 ins_pipe(pipe_slow);
775 %}
776
777 instruct vloadmaskI(vReg dst, vReg src) %{
778 predicate(UseSVE > 0 &&
779 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
780 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
781 match(Set dst (VectorLoadMask src));
782 ins_cost(3 * SVE_COST);
783 format %{ "sve_uunpklo $dst, H, $src\n\t"
784 "sve_uunpklo $dst, S, $dst\n\t"
785 "sve_neg $dst, $dst\t# vector load mask (B to S)" %}
786 ins_encode %{
787 __ sve_uunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
788 __ sve_uunpklo(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($dst$$reg));
789 __ sve_neg(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($dst$$reg));
790 %}
791 ins_pipe(pipe_slow);
792 %}
793
794 instruct vloadmaskL(vReg dst, vReg src) %{
795 predicate(UseSVE > 0 &&
796 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
797 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
798 match(Set dst (VectorLoadMask src));
799 ins_cost(4 * SVE_COST);
800 format %{ "sve_uunpklo $dst, H, $src\n\t"
801 "sve_uunpklo $dst, S, $dst\n\t"
802 "sve_uunpklo $dst, D, $dst\n\t"
803 "sve_neg $dst, $dst\t# vector load mask (B to D)" %}
804 ins_encode %{
805 __ sve_uunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
806 __ sve_uunpklo(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($dst$$reg));
807 __ sve_uunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($dst$$reg));
808 __ sve_neg(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg));
809 %}
810 ins_pipe(pipe_slow);
811 %}
812
813 // vector store mask
814
815 instruct vstoremaskB(vReg dst, vReg src, immI_1 size) %{
816 predicate(UseSVE > 0);
817 match(Set dst (VectorStoreMask src size));
818 ins_cost(SVE_COST);
819 format %{ "sve_neg $dst, $src\t# vector store mask (B)" %}
820 ins_encode %{
821 __ sve_neg(as_FloatRegister($dst$$reg), __ B, ptrue,
822 as_FloatRegister($src$$reg));
823 %}
824 ins_pipe(pipe_slow);
825 %}
826
827 instruct vstoremaskS(vReg dst, vReg src, vReg tmp, immI_2 size) %{
828 predicate(UseSVE > 0);
829 match(Set dst (VectorStoreMask src size));
830 effect(TEMP_DEF dst, TEMP tmp);
831 ins_cost(3 * SVE_COST);
832 format %{ "sve_dup $tmp, H, 0\n\t"
833 "sve_uzp1 $dst, B, $src, $tmp\n\t"
834 "sve_neg $dst, B, $dst\t# vector store mask (sve) (H to B)" %}
835 ins_encode %{
836 __ sve_dup(as_FloatRegister($tmp$$reg), __ H, 0);
837 __ sve_uzp1(as_FloatRegister($dst$$reg), __ B,
838 as_FloatRegister($src$$reg), as_FloatRegister($tmp$$reg));
839 __ sve_neg(as_FloatRegister($dst$$reg), __ B, ptrue,
840 as_FloatRegister($dst$$reg));
841
842 %}
843 ins_pipe(pipe_slow);
844 %}
845
846 instruct vstoremaskI(vReg dst, vReg src, vReg tmp, immI_4 size) %{
847 predicate(UseSVE > 0);
848 match(Set dst (VectorStoreMask src size));
849 effect(TEMP_DEF dst, TEMP tmp);
850 ins_cost(4 * SVE_COST);
851 format %{ "sve_dup $tmp, S, 0\n\t"
852 "sve_uzp1 $dst, H, $src, $tmp\n\t"
853 "sve_uzp1 $dst, B, $dst, $tmp\n\t"
854 "sve_neg $dst, B, $dst\t# vector store mask (sve) (S to B)" %}
855 ins_encode %{
856 __ sve_dup(as_FloatRegister($tmp$$reg), __ S, 0);
857 __ sve_uzp1(as_FloatRegister($dst$$reg), __ H,
858 as_FloatRegister($src$$reg), as_FloatRegister($tmp$$reg));
859 __ sve_uzp1(as_FloatRegister($dst$$reg), __ B,
860 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
861 __ sve_neg(as_FloatRegister($dst$$reg), __ B, ptrue,
862 as_FloatRegister($dst$$reg));
863 %}
864 ins_pipe(pipe_slow);
865 %}
866
867 instruct vstoremaskL(vReg dst, vReg src, vReg tmp, immI_8 size) %{
868 predicate(UseSVE > 0);
869 match(Set dst (VectorStoreMask src size));
870 effect(TEMP_DEF dst, TEMP tmp);
871 ins_cost(5 * SVE_COST);
872 format %{ "sve_dup $tmp, D, 0\n\t"
873 "sve_uzp1 $dst, S, $src, $tmp\n\t"
874 "sve_uzp1 $dst, H, $dst, $tmp\n\t"
875 "sve_uzp1 $dst, B, $dst, $tmp\n\t"
876 "sve_neg $dst, B, $dst\t# vector store mask (sve) (D to B)" %}
877 ins_encode %{
878 __ sve_dup(as_FloatRegister($tmp$$reg), __ D, 0);
879 __ sve_uzp1(as_FloatRegister($dst$$reg), __ S,
880 as_FloatRegister($src$$reg), as_FloatRegister($tmp$$reg));
881 __ sve_uzp1(as_FloatRegister($dst$$reg), __ H,
882 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
883 __ sve_uzp1(as_FloatRegister($dst$$reg), __ B,
884 as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
885 __ sve_neg(as_FloatRegister($dst$$reg), __ B, ptrue,
886 as_FloatRegister($dst$$reg));
887 %}
888 ins_pipe(pipe_slow);
889 %}
890 dnl
891 dnl
892 dnl VLOADMASK_LOADV($1, $2 )
893 dnl VLOADMASK_LOADV(esize, cond)
894 define(`VLOADMASK_LOADV', `
895 instruct vloadmask_loadV_$1(vReg dst, ifelse($1, `byte', vmemA, indirect) mem) %{
896 predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() == MaxVectorSize &&
897 type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) $2);
898 match(Set dst (VectorLoadMask (LoadVector mem)));
899 ins_cost(5 * SVE_COST);
900 format %{ "sve_ld1b $dst, $mem\n\t"
901 "sve_neg $dst, $dst\t# load vector mask (sve)" %}
902 ins_encode %{
903 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
904 BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
905 Assembler::SIMD_RegVariant to_vect_variant = __ elemType_to_regVariant(to_vect_bt);
906 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg, ptrue,
907 T_BOOLEAN, to_vect_bt, $mem->opcode(),
908 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
909 __ sve_neg(dst_reg, to_vect_variant, ptrue, dst_reg);
910 %}
911 ins_pipe(pipe_slow);
912 %}')dnl
913 dnl
914 define(`ARGLIST',
915 `ifelse($1, `byte', vmemA, indirect) mem, vReg src, vReg tmp, ifelse($1, `byte', immI_1, immI_gt_1) esize')
916 dnl
917 dnl STOREV_VSTOREMASK($1, )
918 dnl STOREV_VSTOREMASK(esize)
919 define(`STOREV_VSTOREMASK', `
920 instruct storeV_vstoremask_$1(ARGLIST($1)) %{
921 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() *
922 n->as_StoreVector()->in(MemNode::ValueIn)->in(2)->get_int() == MaxVectorSize);
923 match(Set mem (StoreVector mem (VectorStoreMask src esize)));
924 effect(TEMP tmp);
925 ins_cost(5 * SVE_COST);
926 format %{ "sve_neg $tmp, $src\n\t"
927 "sve_st1b $tmp, $mem\t# store vector mask (sve)" %}
928 ins_encode %{
929 BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
930 assert(type2aelembytes(from_vect_bt) == (int)$esize$$constant, "unsupported type.");
931 Assembler::SIMD_RegVariant from_vect_variant = __ elemBytes_to_regVariant($esize$$constant);
932 __ sve_neg(as_FloatRegister($tmp$$reg), from_vect_variant, ptrue,
933 as_FloatRegister($src$$reg));
934 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($tmp$$reg),
935 ptrue, T_BOOLEAN, from_vect_bt, $mem->opcode(),
936 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
937 %}
938 ins_pipe(pipe_slow);
939 %}')dnl
940 undefine(ARGLIST)dnl
941 dnl
942 // load/store mask vector
943 VLOADMASK_LOADV(byte, == 1)
944 VLOADMASK_LOADV(non_byte, > 1)
945 STOREV_VSTOREMASK(byte)
946 STOREV_VSTOREMASK(non_byte)
947
948 // vector add reduction
949
950 instruct reduce_addI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp) %{
951 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
952 match(Set dst (AddReductionVI src1 src2));
953 effect(TEMP_DEF dst, TEMP vtmp);
954 ins_cost(SVE_COST);
955 format %{ "sve_reduce_addI $dst, $src1, $src2\t# addB/S/I reduction (sve) (may extend)" %}
956 ins_encode %{
957 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
958 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
959 __ sve_uaddv(as_FloatRegister($vtmp$$reg), variant, ptrue, as_FloatRegister($src2$$reg));
960 __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
961 __ addw($dst$$Register, $dst$$Register, $src1$$Register);
962 if (bt == T_BYTE) {
963 __ sxtb($dst$$Register, $dst$$Register);
964 } else if (bt == T_SHORT) {
965 __ sxth($dst$$Register, $dst$$Register);
966 } else {
967 assert(bt == T_INT, "unsupported type");
968 }
969 %}
970 ins_pipe(pipe_slow);
971 %}
972
973 instruct reduce_addI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
974 pRegGov ptmp, rFlagsReg cr) %{
975 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
976 match(Set dst (AddReductionVI src1 src2));
977 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
978 ins_cost(SVE_COST);
979 format %{ "sve_reduce_addI $dst, $src1, $src2\t# addI reduction partial (sve) (may extend)" %}
980 ins_encode %{
981 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
982 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
983 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
984 Matcher::vector_length(this, $src2));
985 __ sve_uaddv(as_FloatRegister($vtmp$$reg), variant,
986 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
987 __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
988 __ addw($dst$$Register, $dst$$Register, $src1$$Register);
989 if (bt == T_BYTE) {
990 __ sxtb($dst$$Register, $dst$$Register);
991 } else if (bt == T_SHORT) {
992 __ sxth($dst$$Register, $dst$$Register);
993 } else {
994 assert(bt == T_INT, "unsupported type");
995 }
996 %}
997 ins_pipe(pipe_slow);
998 %}
999
1000 instruct reduce_addL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp) %{
1001 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1002 match(Set dst (AddReductionVL src1 src2));
1003 effect(TEMP_DEF dst, TEMP vtmp);
1004 ins_cost(SVE_COST);
1005 format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction (sve)" %}
1006 ins_encode %{
1007 __ sve_uaddv(as_FloatRegister($vtmp$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
1008 __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1009 __ add($dst$$Register, $dst$$Register, $src1$$Register);
1010 %}
1011 ins_pipe(pipe_slow);
1012 %}
1013
1014 instruct reduce_addL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1015 pRegGov ptmp, rFlagsReg cr) %{
1016 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1017 match(Set dst (AddReductionVL src1 src2));
1018 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1019 ins_cost(SVE_COST);
1020 format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction partial (sve)" %}
1021 ins_encode %{
1022 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1023 Matcher::vector_length(this, $src2));
1024 __ sve_uaddv(as_FloatRegister($vtmp$$reg), __ D,
1025 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1026 __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1027 __ add($dst$$Register, $dst$$Register, $src1$$Register);
1028 %}
1029 ins_pipe(pipe_slow);
1030 %}
1031
1032 dnl
1033 dnl REDUCE_ADDF($1, $2, $3, $4 )
1034 dnl REDUCE_ADDF(insn_name, op_name, reg_dst, size)
1035 define(`REDUCE_ADDF', `
1036 instruct $1($3 src1_dst, vReg src2) %{
1037 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1038 match(Set src1_dst (AddReductionV$2 src1_dst src2));
1039 ins_cost(SVE_COST);
1040 format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) ($4)" %}
1041 ins_encode %{
1042 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1043 ptrue, as_FloatRegister($src2$$reg));
1044 %}
1045 ins_pipe(pipe_slow);
1046 %}')dnl
1047 dnl
1048 dnl
1049 dnl REDUCE_ADDF_PARTIAL($1, $2, $3, $4 )
1050 dnl REDUCE_ADDF_PARTIAL(insn_name, suffix, reg_dst, size)
1051 define(`REDUCE_ADDF_PARTIAL', `
1052 instruct $1($3 src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
1053 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1054 match(Set src1_dst (AddReductionV$2 src1_dst src2));
1055 ins_cost(SVE_COST);
1056 effect(TEMP ptmp, KILL cr);
1057 format %{ "sve_reduce_add$2 $src1_dst, $src1_dst, $src2\t# add$2 reduction partial (sve) ($4)" %}
1058 ins_encode %{
1059 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
1060 Matcher::vector_length(this, $src2));
1061 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1062 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1063 %}
1064 ins_pipe(pipe_slow);
1065 %}')dnl
1066 dnl
1067 REDUCE_ADDF(reduce_addF, F, vRegF, S)
1068 REDUCE_ADDF_PARTIAL(reduce_addF_partial, F, vRegF, S)
1069 REDUCE_ADDF(reduce_addD, D, vRegD, D)
1070 REDUCE_ADDF_PARTIAL(reduce_addD_partial, D, vRegD, D)
1071
1072 // vector and reduction
1073
1074 instruct reduce_andI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp) %{
1075 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1076 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1077 match(Set dst (AndReductionV src1 src2));
1078 effect(TEMP_DEF dst, TEMP vtmp);
1079 ins_cost(SVE_COST);
1080 format %{ "sve_reduce_andI $dst, $src1, $src2\t# andB/S/I reduction (sve) (may extend)" %}
1081 ins_encode %{
1082 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1083 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1084 __ sve_andv(as_FloatRegister($vtmp$$reg), variant, ptrue, as_FloatRegister($src2$$reg));
1085 __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1086 __ andw($dst$$Register, $dst$$Register, $src1$$Register);
1087 if (bt == T_BYTE) {
1088 __ sxtb($dst$$Register, $dst$$Register);
1089 } else if (bt == T_SHORT) {
1090 __ sxth($dst$$Register, $dst$$Register);
1091 } else {
1092 assert(bt == T_INT, "unsupported type");
1093 }
1094 %}
1095 ins_pipe(pipe_slow);
1096 %}
1097
1098 instruct reduce_andI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1099 pRegGov ptmp, rFlagsReg cr) %{
1100 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1101 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1102 match(Set dst (AndReductionV src1 src2));
1103 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1104 ins_cost(SVE_COST);
1105 format %{ "sve_reduce_andI $dst, $src1, $src2\t# andI reduction partial (sve) (may extend)" %}
1106 ins_encode %{
1107 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1108 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1109 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1110 Matcher::vector_length(this, $src2));
1111 __ sve_andv(as_FloatRegister($vtmp$$reg), variant,
1112 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1113 __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1114 __ andw($dst$$Register, $dst$$Register, $src1$$Register);
1115 if (bt == T_BYTE) {
1116 __ sxtb($dst$$Register, $dst$$Register);
1117 } else if (bt == T_SHORT) {
1118 __ sxth($dst$$Register, $dst$$Register);
1119 } else {
1120 assert(bt == T_INT, "unsupported type");
1121 }
1122 %}
1123 ins_pipe(pipe_slow);
1124 %}
1125
1126 instruct reduce_andL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp) %{
1127 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1128 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1129 match(Set dst (AndReductionV src1 src2));
1130 effect(TEMP_DEF dst, TEMP vtmp);
1131 ins_cost(SVE_COST);
1132 format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction (sve)" %}
1133 ins_encode %{
1134 __ sve_andv(as_FloatRegister($vtmp$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
1135 __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1136 __ andr($dst$$Register, $dst$$Register, $src1$$Register);
1137 %}
1138 ins_pipe(pipe_slow);
1139 %}
1140
1141 instruct reduce_andL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1142 pRegGov ptmp, rFlagsReg cr) %{
1143 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1144 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1145 match(Set dst (AndReductionV src1 src2));
1146 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1147 ins_cost(SVE_COST);
1148 format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction partial (sve)" %}
1149 ins_encode %{
1150 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1151 Matcher::vector_length(this, $src2));
1152 __ sve_andv(as_FloatRegister($vtmp$$reg), __ D,
1153 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1154 __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1155 __ andr($dst$$Register, $dst$$Register, $src1$$Register);
1156 %}
1157 ins_pipe(pipe_slow);
1158 %}
1159
1160 // vector or reduction
1161
1162 instruct reduce_orI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp) %{
1163 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1164 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1165 match(Set dst (OrReductionV src1 src2));
1166 effect(TEMP_DEF dst, TEMP vtmp);
1167 ins_cost(SVE_COST);
1168 format %{ "sve_reduce_orI $dst, $src1, $src2\t# orB/S/I reduction (sve) (may extend)" %}
1169 ins_encode %{
1170 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1171 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1172 __ sve_orv(as_FloatRegister($vtmp$$reg), variant, ptrue, as_FloatRegister($src2$$reg));
1173 __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1174 __ orrw($dst$$Register, $dst$$Register, $src1$$Register);
1175 if (bt == T_BYTE) {
1176 __ sxtb($dst$$Register, $dst$$Register);
1177 } else if (bt == T_SHORT) {
1178 __ sxth($dst$$Register, $dst$$Register);
1179 } else {
1180 assert(bt == T_INT, "unsupported type");
1181 }
1182 %}
1183 ins_pipe(pipe_slow);
1184 %}
1185
1186 instruct reduce_orI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1187 pRegGov ptmp, rFlagsReg cr) %{
1188 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1189 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1190 match(Set dst (OrReductionV src1 src2));
1191 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1192 ins_cost(SVE_COST);
1193 format %{ "sve_reduce_orI $dst, $src1, $src2\t# orI reduction partial (sve) (may extend)" %}
1194 ins_encode %{
1195 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1196 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1197 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1198 Matcher::vector_length(this, $src2));
1199 __ sve_orv(as_FloatRegister($vtmp$$reg), variant,
1200 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1201 __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1202 __ orrw($dst$$Register, $dst$$Register, $src1$$Register);
1203 if (bt == T_BYTE) {
1204 __ sxtb($dst$$Register, $dst$$Register);
1205 } else if (bt == T_SHORT) {
1206 __ sxth($dst$$Register, $dst$$Register);
1207 } else {
1208 assert(bt == T_INT, "unsupported type");
1209 }
1210 %}
1211 ins_pipe(pipe_slow);
1212 %}
1213
1214 instruct reduce_orL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp) %{
1215 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1216 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1217 match(Set dst (OrReductionV src1 src2));
1218 effect(TEMP_DEF dst, TEMP vtmp);
1219 ins_cost(SVE_COST);
1220 format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction (sve)" %}
1221 ins_encode %{
1222 __ sve_orv(as_FloatRegister($vtmp$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
1223 __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1224 __ orr($dst$$Register, $dst$$Register, $src1$$Register);
1225 %}
1226 ins_pipe(pipe_slow);
1227 %}
1228
1229 instruct reduce_orL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1230 pRegGov ptmp, rFlagsReg cr) %{
1231 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1232 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1233 match(Set dst (OrReductionV src1 src2));
1234 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1235 ins_cost(SVE_COST);
1236 format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction partial (sve)" %}
1237 ins_encode %{
1238 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1239 Matcher::vector_length(this, $src2));
1240 __ sve_orv(as_FloatRegister($vtmp$$reg), __ D,
1241 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1242 __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1243 __ orr($dst$$Register, $dst$$Register, $src1$$Register);
1244 %}
1245 ins_pipe(pipe_slow);
1246 %}
1247
1248 // vector xor reduction
1249
1250 instruct reduce_eorI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp) %{
1251 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1252 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1253 match(Set dst (XorReductionV src1 src2));
1254 effect(TEMP_DEF dst, TEMP vtmp);
1255 ins_cost(SVE_COST);
1256 format %{ "sve_reduce_eorI $dst, $src1, $src2\t# xorB/H/I reduction (sve) (may extend)" %}
1257 ins_encode %{
1258 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1259 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1260 __ sve_eorv(as_FloatRegister($vtmp$$reg), variant, ptrue, as_FloatRegister($src2$$reg));
1261 __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1262 __ eorw($dst$$Register, $dst$$Register, $src1$$Register);
1263 if (bt == T_BYTE) {
1264 __ sxtb($dst$$Register, $dst$$Register);
1265 } else if (bt == T_SHORT) {
1266 __ sxth($dst$$Register, $dst$$Register);
1267 } else {
1268 assert(bt == T_INT, "unsupported type");
1269 }
1270 %}
1271 ins_pipe(pipe_slow);
1272 %}
1273
1274 instruct reduce_eorI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1275 pRegGov ptmp, rFlagsReg cr) %{
1276 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1277 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1278 match(Set dst (XorReductionV src1 src2));
1279 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1280 ins_cost(SVE_COST);
1281 format %{ "sve_reduce_eorI $dst, $src1, $src2\t# xorI reduction partial (sve) (may extend)" %}
1282 ins_encode %{
1283 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1284 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1285 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1286 Matcher::vector_length(this, $src2));
1287 __ sve_eorv(as_FloatRegister($vtmp$$reg), variant,
1288 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1289 __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1290 __ eorw($dst$$Register, $dst$$Register, $src1$$Register);
1291 if (bt == T_BYTE) {
1292 __ sxtb($dst$$Register, $dst$$Register);
1293 } else if (bt == T_SHORT) {
1294 __ sxth($dst$$Register, $dst$$Register);
1295 } else {
1296 assert(bt == T_INT, "unsupported type");
1297 }
1298 %}
1299 ins_pipe(pipe_slow);
1300 %}
1301
1302 instruct reduce_eorL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp) %{
1303 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1304 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1305 match(Set dst (XorReductionV src1 src2));
1306 effect(TEMP_DEF dst, TEMP vtmp);
1307 ins_cost(SVE_COST);
1308 format %{ "sve_reduce_eorL $dst, $src1, $src2\t# xorL reduction (sve)" %}
1309 ins_encode %{
1310 __ sve_eorv(as_FloatRegister($vtmp$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
1311 __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1312 __ eor($dst$$Register, $dst$$Register, $src1$$Register);
1313 %}
1314 ins_pipe(pipe_slow);
1315 %}
1316
1317 instruct reduce_eorL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1318 pRegGov ptmp, rFlagsReg cr) %{
1319 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1320 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1321 match(Set dst (XorReductionV src1 src2));
1322 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1323 ins_cost(SVE_COST);
1324 format %{ "sve_reduce_eorL $dst, $src1, $src2\t# xorL reduction partial (sve)" %}
1325 ins_encode %{
1326 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1327 Matcher::vector_length(this, $src2));
1328 __ sve_eorv(as_FloatRegister($vtmp$$reg), __ D,
1329 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1330 __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1331 __ eor($dst$$Register, $dst$$Register, $src1$$Register);
1332 %}
1333 ins_pipe(pipe_slow);
1334 %}
1335
1336 dnl
1337 dnl REDUCE_MAXMIN_I($1, $2, $3 )
1338 dnl REDUCE_MAXMIN_I(min_max, op_mame, cmp)
1339 define(`REDUCE_MAXMIN_I', `
1340 instruct reduce_$1I(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp) %{
1341 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1342 (n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
1343 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
1344 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT));
1345 match(Set dst ($2 src1 src2));
1346 effect(TEMP_DEF dst, TEMP vtmp);
1347 ins_cost(SVE_COST);
1348 format %{ "sve_reduce_$1I $dst, $src1, $src2\t# reduce $1B/S/I (sve)" %}
1349 ins_encode %{
1350 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1351 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1352 __ sve_s$1v(as_FloatRegister($vtmp$$reg), variant, ptrue, as_FloatRegister($src2$$reg));
1353 __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1354 __ cmpw($dst$$Register, $src1$$Register);
1355 __ cselw(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($src1$$reg), Assembler::$3);
1356 %}
1357 ins_pipe(pipe_slow);
1358 %}')dnl
1359 dnl
1360 dnl REDUCE_MAXMIN_L($1, $2, $3 )
1361 dnl REDUCE_MAXMIN_L(min_max, op_name, cmp)
1362 define(`REDUCE_MAXMIN_L', `
1363 instruct reduce_$1L(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp) %{
1364 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1365 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1366 match(Set dst ($2 src1 src2));
1367 effect(TEMP_DEF dst, TEMP vtmp);
1368 ins_cost(SVE_COST);
1369 format %{ "sve_reduce_$1L $dst, $src1, $src2\t# reduce $1L partial (sve)" %}
1370 ins_encode %{
1371 __ sve_s$1v(as_FloatRegister($vtmp$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
1372 __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1373 __ cmp($dst$$Register, $src1$$Register);
1374 __ csel(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($src1$$reg), Assembler::$3);
1375 %}
1376 ins_pipe(pipe_slow);
1377 %}')dnl
1378 dnl
1379 dnl REDUCE_MAXMIN_I_PARTIAL($1, $2, $3 )
1380 dnl REDUCE_MAXMIN_I_PARTIAL(min_max, op_mame, cmp)
1381 define(`REDUCE_MAXMIN_I_PARTIAL', `
1382 instruct reduce_$1I_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1383 pRegGov ptmp, rFlagsReg cr) %{
1384 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1385 (n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
1386 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
1387 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT));
1388 match(Set dst ($2 src1 src2));
1389 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1390 ins_cost(SVE_COST);
1391 format %{ "sve_reduce_$1I $dst, $src1, $src2\t# reduce $1I partial (sve)" %}
1392 ins_encode %{
1393 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1394 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1395 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1396 Matcher::vector_length(this, $src2));
1397 __ sve_s$1v(as_FloatRegister($vtmp$$reg), variant,
1398 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1399 __ smov($dst$$Register, as_FloatRegister($vtmp$$reg), variant, 0);
1400 __ cmpw($dst$$Register, $src1$$Register);
1401 __ cselw(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($src1$$reg), Assembler::$3);
1402 %}
1403 ins_pipe(pipe_slow);
1404 %}')dnl
1405 dnl
1406 dnl REDUCE_MAXMIN_L_PARTIAL($1, $2, $3 )
1407 dnl REDUCE_MAXMIN_L_PARTIAL(min_max, op_name, cmp)
1408 define(`REDUCE_MAXMIN_L_PARTIAL', `
1409 instruct reduce_$1L_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1410 pRegGov ptmp, rFlagsReg cr) %{
1411 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1412 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1413 match(Set dst ($2 src1 src2));
1414 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1415 ins_cost(SVE_COST);
1416 format %{ "sve_reduce_$1L $dst, $src1, $src2\t# reduce $1L partial (sve)" %}
1417 ins_encode %{
1418 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1419 Matcher::vector_length(this, $src2));
1420 __ sve_s$1v(as_FloatRegister($vtmp$$reg), __ D,
1421 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1422 __ umov($dst$$Register, as_FloatRegister($vtmp$$reg), __ D, 0);
1423 __ cmp($dst$$Register, $src1$$Register);
1424 __ csel(as_Register($dst$$reg), as_Register($dst$$reg), as_Register($src1$$reg), Assembler::$3);
1425 %}
1426 ins_pipe(pipe_slow);
1427 %}')dnl
1428 dnl
1429 dnl REDUCE_FMINMAX($1, $2, $3, $4, $5 )
1430 dnl REDUCE_FMINMAX(min_max, name_suffix, element_type, size, reg_src_dst)
1431 define(`REDUCE_FMINMAX', `
1432 instruct reduce_$1$2($5 dst, $5 src1, vReg src2) %{
1433 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1434 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1435 match(Set dst (translit($1, `m', `M')ReductionV src1 src2));
1436 ins_cost(INSN_COST);
1437 effect(TEMP_DEF dst);
1438 format %{ "sve_f$1v $dst, $src2 # vector (sve) ($4)\n\t"
1439 "f$1s $dst, $dst, $src1\t# $1 reduction $2" %}
1440 ins_encode %{
1441 __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4,
1442 ptrue, as_FloatRegister($src2$$reg));
1443 __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1444 %}
1445 ins_pipe(pipe_slow);
1446 %}')dnl
1447 dnl
1448 dnl
1449 dnl REDUCE_FMINMAX_PARTIAL($1, $2, $3, $4, $5 )
1450 dnl REDUCE_FMINMAX_PARTIAL(min_max, name_suffix, element_type, size, reg_src_dst)
1451 define(`REDUCE_FMINMAX_PARTIAL', `
1452 instruct reduce_$1$2_partial($5 dst, $5 src1, vReg src2,
1453 pRegGov ptmp, rFlagsReg cr) %{
1454 predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1455 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1456 match(Set dst (translit($1, `m', `M')ReductionV src1 src2));
1457 ins_cost(INSN_COST);
1458 effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
1459 format %{ "sve_reduce_$1$2 $dst, $src1, $src2\t# reduce $1 $4 partial (sve)" %}
1460 ins_encode %{
1461 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
1462 Matcher::vector_length(this, $src2));
1463 __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4,
1464 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1465 __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1466 %}
1467 ins_pipe(pipe_slow);
1468 %}')dnl
1469
1470 // vector max reduction
1471 REDUCE_MAXMIN_I(max, MaxReductionV, GT)
1472 REDUCE_MAXMIN_I_PARTIAL(max, MaxReductionV, GT)
1473 REDUCE_MAXMIN_L(max, MaxReductionV, GT)
1474 REDUCE_MAXMIN_L_PARTIAL(max, MaxReductionV, GT)
1475 REDUCE_FMINMAX(max, F, T_FLOAT, S, vRegF)
1476 REDUCE_FMINMAX_PARTIAL(max, F, T_FLOAT, S, vRegF)
1477 REDUCE_FMINMAX(max, D, T_DOUBLE, D, vRegD)
1478 REDUCE_FMINMAX_PARTIAL(max, D, T_DOUBLE, D, vRegD)
1479
1480 // vector min reduction
1481 REDUCE_MAXMIN_I(min, MinReductionV, LT)
1482 REDUCE_MAXMIN_I_PARTIAL(min, MinReductionV, LT)
1483 REDUCE_MAXMIN_L(min, MinReductionV, LT)
1484 REDUCE_MAXMIN_L_PARTIAL(min, MinReductionV, LT)
1485 REDUCE_FMINMAX(min, F, T_FLOAT, S, vRegF)
1486 REDUCE_FMINMAX_PARTIAL(min, F, T_FLOAT, S, vRegF)
1487 REDUCE_FMINMAX(min, D, T_DOUBLE, D, vRegD)
1488 REDUCE_FMINMAX_PARTIAL(min, D, T_DOUBLE, D, vRegD)
1489
1490 // vector Math.rint, floor, ceil
1491
1492 instruct vroundD(vReg dst, vReg src, immI rmode) %{
1493 predicate(UseSVE > 0 &&
1494 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
1495 match(Set dst (RoundDoubleModeV src rmode));
1496 format %{ "sve_frint $dst, $src, $rmode\t# vector (sve) (D)" %}
1497 ins_encode %{
1498 switch ($rmode$$constant) {
1499 case RoundDoubleModeNode::rmode_rint:
1500 __ sve_frintn(as_FloatRegister($dst$$reg), __ D,
1501 ptrue, as_FloatRegister($src$$reg));
1502 break;
1503 case RoundDoubleModeNode::rmode_floor:
1504 __ sve_frintm(as_FloatRegister($dst$$reg), __ D,
1505 ptrue, as_FloatRegister($src$$reg));
1506 break;
1507 case RoundDoubleModeNode::rmode_ceil:
1508 __ sve_frintp(as_FloatRegister($dst$$reg), __ D,
1509 ptrue, as_FloatRegister($src$$reg));
1647 VSHIFT_TRUE_PREDICATE(vlsrS, URShiftVS, H, 8, sve_lsr)
1648 VSHIFT_TRUE_PREDICATE(vlsrI, URShiftVI, S, 4, sve_lsr)
1649 VSHIFT_TRUE_PREDICATE(vlsrL, URShiftVL, D, 2, sve_lsr)
1650 VSHIFT_IMM_UNPREDICATED(vasrB_imm, RShiftVB, RShiftCntV, B, 16, sve_asr)
1651 VSHIFT_IMM_UNPREDICATED(vasrS_imm, RShiftVS, RShiftCntV, H, 8, sve_asr)
1652 VSHIFT_IMM_UNPREDICATED(vasrI_imm, RShiftVI, RShiftCntV, S, 4, sve_asr)
1653 VSHIFT_IMM_UNPREDICATED(vasrL_imm, RShiftVL, RShiftCntV, D, 2, sve_asr)
1654 VSHIFT_IMM_UNPREDICATED(vlsrB_imm, URShiftVB, RShiftCntV, B, 16, sve_lsr)
1655 VSHIFT_IMM_UNPREDICATED(vlsrS_imm, URShiftVS, RShiftCntV, H, 8, sve_lsr)
1656 VSHIFT_IMM_UNPREDICATED(vlsrI_imm, URShiftVI, RShiftCntV, S, 4, sve_lsr)
1657 VSHIFT_IMM_UNPREDICATED(vlsrL_imm, URShiftVL, RShiftCntV, D, 2, sve_lsr)
1658 VSHIFT_IMM_UNPREDICATED(vlslB_imm, LShiftVB, LShiftCntV, B, 16, sve_lsl)
1659 VSHIFT_IMM_UNPREDICATED(vlslS_imm, LShiftVS, LShiftCntV, H, 8, sve_lsl)
1660 VSHIFT_IMM_UNPREDICATED(vlslI_imm, LShiftVI, LShiftCntV, S, 4, sve_lsl)
1661 VSHIFT_IMM_UNPREDICATED(vlslL_imm, LShiftVL, LShiftCntV, D, 2, sve_lsl)
1662 VSHIFT_COUNT(vshiftcntB, B, 16, T_BYTE)
1663 VSHIFT_COUNT(vshiftcntS, H, 8, T_SHORT)
1664 VSHIFT_COUNT(vshiftcntI, S, 4, T_INT)
1665 VSHIFT_COUNT(vshiftcntL, D, 2, T_LONG)
1666
1667 // vector sqrt
1668 UNARY_OP_TRUE_PREDICATE(vsqrtF, SqrtVF, S, 16, sve_fsqrt)
1669 UNARY_OP_TRUE_PREDICATE(vsqrtD, SqrtVD, D, 16, sve_fsqrt)
1670
1671 // vector sub
1672 BINARY_OP_UNPREDICATED(vsubB, SubVB, B, 16, sve_sub)
1673 BINARY_OP_UNPREDICATED(vsubS, SubVS, H, 8, sve_sub)
1674 BINARY_OP_UNPREDICATED(vsubI, SubVI, S, 4, sve_sub)
1675 BINARY_OP_UNPREDICATED(vsubL, SubVL, D, 2, sve_sub)
1676 BINARY_OP_UNPREDICATED(vsubF, SubVF, S, 4, sve_fsub)
1677 BINARY_OP_UNPREDICATED(vsubD, SubVD, D, 2, sve_fsub)
1678
1679 // vector mask cast
1680
1681 instruct vmaskcast(vReg dst) %{
1682 predicate(UseSVE > 0 && n->bottom_type()->is_vect()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
1683 n->bottom_type()->is_vect()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
1684 match(Set dst (VectorMaskCast dst));
1685 ins_cost(0);
1686 format %{ "vmaskcast $dst\t# empty (sve)" %}
1687 ins_encode %{
1688 // empty
1689 %}
1690 ins_pipe(pipe_class_empty);
1691 %}
1692
1693 // ------------------------------ Vector cast -------------------------------
1694 dnl
1695 dnl
1696 define(`VECTOR_CAST_EXTEND1', `
1697 instruct vcvt$1to$2`'(vReg dst, vReg src)
1698 %{
1699 predicate(UseSVE > 0 &&
1700 n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1701 match(Set dst (VectorCast$1`'2X src));
1702 ins_cost(SVE_COST);
1703 format %{ "sve_$3 $dst, $4, $src\t# convert $1 to $2 vector" %}
1704 ins_encode %{
1705 __ sve_$3(as_FloatRegister($dst$$reg), __ $4, as_FloatRegister($src$$reg));
1706 %}
1707 ins_pipe(pipe_slow);
1708 %}')dnl
1709 dnl
1710 dnl
1711 define(`VECTOR_CAST_EXTEND2', `
1712 instruct vcvt$1to$2`'(vReg dst, vReg src)
1713 %{
1714 predicate(UseSVE > 0 &&
1715 n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1716 match(Set dst (VectorCast$1`'2X src));
1717 ins_cost(2 * SVE_COST);
1718 format %{ "sve_$3 $dst, $4, $src\n\t"
1719 "sve_$3 $dst, $5, $dst\t# convert $1 to $2 vector" %}
1720 ins_encode %{
1721 __ sve_$3(as_FloatRegister($dst$$reg), __ $4, as_FloatRegister($src$$reg));
1722 __ sve_$3(as_FloatRegister($dst$$reg), __ $5, as_FloatRegister($dst$$reg));
1723 %}
1724 ins_pipe(pipe_slow);
1725 %}')dnl
1726 dnl
1727 dnl
1728 define(`VECTOR_CAST_EXTEND3', `
1729 instruct vcvt$1to$2`'(vReg dst, vReg src)
1730 %{
1731 predicate(UseSVE > 0 &&
1732 n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1733 match(Set dst (VectorCast$1`'2X src));
1734 ins_cost(3 * SVE_COST);
1735 format %{ "sve_$3 $dst, $4, $src\n\t"
1736 "sve_$3 $dst, $5, $dst\n\t"
1737 "sve_$3 $dst, $6, $dst\t# convert $1 to $2 vector" %}
1738 ins_encode %{
1739 __ sve_$3(as_FloatRegister($dst$$reg), __ $4, as_FloatRegister($src$$reg));
1740 __ sve_$3(as_FloatRegister($dst$$reg), __ $5, as_FloatRegister($dst$$reg));
1741 __ sve_$3(as_FloatRegister($dst$$reg), __ $6, as_FloatRegister($dst$$reg));
1742 %}
1743 ins_pipe(pipe_slow);
1744 %}')dnl
1745 dnl
1746 dnl
1747 define(`VECTOR_CAST_NARROW1', `
1748 instruct vcvt$1to$2`'(vReg dst, vReg src, vReg tmp)
1749 %{
1750 predicate(UseSVE > 0 &&
1751 n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1752 match(Set dst (VectorCast$1`'2X src));
1753 effect(TEMP tmp);
1754 ins_cost(2 * SVE_COST);
1755 format %{ "sve_$3 $tmp, $4, 0\n\t"
1756 "sve_$5 $dst, $4, $src, tmp\t# convert $1 to $2 vector" %}
1757 ins_encode %{
1758 __ sve_$3(as_FloatRegister($tmp$$reg), __ $4, 0);
1759 __ sve_$5(as_FloatRegister($dst$$reg), __ $4, as_FloatRegister($src$$reg), as_FloatRegister($tmp$$reg));
1760 %}
1761 ins_pipe(pipe_slow);
1762 %}')dnl
1763 dnl
1764 dnl
1765 define(`VECTOR_CAST_NARROW2', `
1766 instruct vcvt$1to$2`'(vReg dst, vReg src, vReg tmp)
1767 %{
1768 predicate(UseSVE > 0 &&
1769 n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1770 match(Set dst (VectorCast$1`'2X src));
1771 effect(TEMP_DEF dst, TEMP tmp);
1772 ins_cost(3 * SVE_COST);
1773 format %{ "sve_$3 $tmp, $4, 0\n\t"
1774 "sve_$5 $dst, $4, $src, tmp\n\t"
1775 "sve_$5 $dst, $6, $dst, tmp\n\t# convert $1 to $2 vector" %}
1776 ins_encode %{
1777 __ sve_$3(as_FloatRegister($tmp$$reg), __ $4, 0);
1778 __ sve_$5(as_FloatRegister($dst$$reg), __ $4, as_FloatRegister($src$$reg), as_FloatRegister($tmp$$reg));
1779 __ sve_$5(as_FloatRegister($dst$$reg), __ $6, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
1780 %}
1781 ins_pipe(pipe_slow);
1782 %}')dnl
1783 dnl
1784 dnl
1785 define(`VECTOR_CAST_NARROW3', `
1786 instruct vcvt$1to$2`'(vReg dst, vReg src, vReg tmp)
1787 %{
1788 predicate(UseSVE > 0 &&
1789 n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1790 match(Set dst (VectorCast$1`'2X src));
1791 effect(TEMP_DEF dst, TEMP tmp);
1792 ins_cost(4 * SVE_COST);
1793 format %{ "sve_$3 $tmp, $4, 0\n\t"
1794 "sve_$5 $dst, $4, $src, tmp\n\t"
1795 "sve_$5 $dst, $6, $dst, tmp\n\t"
1796 "sve_$5 $dst, $7, $dst, tmp\n\t# convert $1 to $2 vector" %}
1797 ins_encode %{
1798 __ sve_$3(as_FloatRegister($tmp$$reg), __ $4, 0);
1799 __ sve_$5(as_FloatRegister($dst$$reg), __ $4, as_FloatRegister($src$$reg), as_FloatRegister($tmp$$reg));
1800 __ sve_$5(as_FloatRegister($dst$$reg), __ $6, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
1801 __ sve_$5(as_FloatRegister($dst$$reg), __ $7, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
1802 %}
1803 ins_pipe(pipe_slow);
1804 %}')dnl
1805 dnl
1806 dnl
1807 define(`VECTOR_CAST_I2F_EXTEND2', `
1808 instruct vcvt$1to$2`'(vReg dst, vReg src)
1809 %{
1810 predicate(UseSVE > 0 &&
1811 n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1812 match(Set dst (VectorCast$1`'2X src));
1813 ins_cost(3 * SVE_COST);
1814 format %{ "sve_$3 $dst, $4, $src\n\t"
1815 "sve_$3 $dst, $5, $dst\n\t"
1816 "sve_$6 $dst, $5, $dst, $5\t# convert $1 to $2 vector" %}
1817 ins_encode %{
1818 __ sve_$3(as_FloatRegister($dst$$reg), __ $4, as_FloatRegister($src$$reg));
1819 __ sve_$3(as_FloatRegister($dst$$reg), __ $5, as_FloatRegister($dst$$reg));
1820 __ sve_$6(as_FloatRegister($dst$$reg), __ $5, ptrue, as_FloatRegister($dst$$reg), __ $5);
1821 %}
1822 ins_pipe(pipe_slow);
1823 %}')dnl
1824 dnl
1825 dnl
1826 define(`VECTOR_CAST_I2F_EXTEND3', `
1827 instruct vcvt$1to$2`'(vReg dst, vReg src)
1828 %{
1829 predicate(UseSVE > 0 &&
1830 n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1831 match(Set dst (VectorCast$1`'2X src));
1832 ins_cost(4 * SVE_COST);
1833 format %{ "sve_$3 $dst, $4, $src\n\t"
1834 "sve_$3 $dst, $5, $dst\n\t"
1835 "sve_$3 $dst, $6, $dst\n\t"
1836 "sve_$7 $dst, $6, $dst, $6\t# convert $1 to $2 vector" %}
1837 ins_encode %{
1838 __ sve_$3(as_FloatRegister($dst$$reg), __ $4, as_FloatRegister($src$$reg));
1839 __ sve_$3(as_FloatRegister($dst$$reg), __ $5, as_FloatRegister($dst$$reg));
1840 __ sve_$3(as_FloatRegister($dst$$reg), __ $6, as_FloatRegister($dst$$reg));
1841 __ sve_$7(as_FloatRegister($dst$$reg), __ $6, ptrue, as_FloatRegister($dst$$reg), __ $6);
1842 %}
1843 ins_pipe(pipe_slow);
1844 %}')dnl
1845 dnl
1846 dnl
1847 define(`VECTOR_CAST_X2F_NARROW1', `
1848 instruct vcvt$1to$2`'(vReg dst, vReg src, vReg tmp)
1849 %{
1850 predicate(UseSVE > 0 &&
1851 n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1852 match(Set dst (VectorCast$1`'2X src));
1853 effect(TEMP_DEF dst, TEMP tmp);
1854 ins_cost(3 * SVE_COST);
1855 format %{ "sve_$3 $dst, $4, $src, $5\n\t"
1856 "sve_$6 $tmp, $7, 0\n\t"
1857 "sve_$8 $dst, $7, $dst, $tmp\t# convert $1 to $2 vector" %}
1858 ins_encode %{
1859 __ sve_$3(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src$$reg), __ $5);
1860 __ sve_$6(as_FloatRegister($tmp$$reg), __ $7, 0);
1861 __ sve_$8(as_FloatRegister($dst$$reg), __ $7, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
1862 %}
1863 ins_pipe(pipe_slow);
1864 %}')dnl
1865 dnl
1866 dnl
1867 define(`VECTOR_CAST_X2X', `
1868 instruct vcvt$1to$2`'(vReg dst, vReg src)
1869 %{
1870 predicate(UseSVE > 0 &&
1871 n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1872 match(Set dst (VectorCast$1`'2X src));
1873 ins_cost(SVE_COST);
1874 format %{ "sve_$3 $dst, $4, $src, $4\t# convert $1 to $2 vector" %}
1875 ins_encode %{
1876 __ sve_$3(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src$$reg), __ $4);
1877 %}
1878 ins_pipe(pipe_slow);
1879 %}')dnl
1880 dnl
1881 dnl
1882 define(`VECTOR_CAST_X2F_EXTEND1', `
1883 instruct vcvt$1to$2`'(vReg dst, vReg src)
1884 %{
1885 predicate(UseSVE > 0 &&
1886 n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1887 match(Set dst (VectorCast$1`'2X src));
1888 ins_cost(2 * SVE_COST);
1889 format %{ "sve_$3 $dst, $4, $src\n\t"
1890 "sve_$5 $dst, $4, $dst, $6\t# convert $1 to $2 vector" %}
1891 ins_encode %{
1892 __ sve_$3(as_FloatRegister($dst$$reg), __ $4, as_FloatRegister($src$$reg));
1893 __ sve_$5(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($dst$$reg), __ $6);
1894 %}
1895 ins_pipe(pipe_slow);
1896 %}')dnl
1897 dnl
1898 dnl
1899 define(`VECTOR_CAST_F2X_NARROW1', `
1900 instruct vcvt$1to$2`'(vReg dst, vReg src, vReg tmp)
1901 %{
1902 predicate(UseSVE > 0 &&
1903 n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1904 match(Set dst (VectorCast$1`'2X src));
1905 effect(TEMP_DEF dst, TEMP tmp);
1906 ins_cost(3 * SVE_COST);
1907 format %{ "sve_$3 $dst, $4, $src, $4\n\t"
1908 "sve_$5 $tmp, $6, 0\n\t"
1909 "sve_$7 $dst, $6, $dst, tmp\t# convert $1 to $2 vector" %}
1910 ins_encode %{
1911 __ sve_$3(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src$$reg), __ $4);
1912 __ sve_$5(as_FloatRegister($tmp$$reg), __ $6, 0);
1913 __ sve_$7(as_FloatRegister($dst$$reg), __ $6, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
1914 %}
1915 ins_pipe(pipe_slow);
1916 %}')dnl
1917 dnl
1918 dnl
1919 define(`VECTOR_CAST_F2X_NARROW2', `
1920 instruct vcvt$1to$2`'(vReg dst, vReg src, vReg tmp)
1921 %{
1922 predicate(UseSVE > 0 &&
1923 n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1924 match(Set dst (VectorCast$1`'2X src));
1925 effect(TEMP_DEF dst, TEMP tmp);
1926 ins_cost(4 * SVE_COST);
1927 format %{ "sve_$3 $dst, $4, $src, $4\n\t"
1928 "sve_$5 $tmp, $6, 0\n\t"
1929 "sve_$7 $dst, $6, $dst, tmp\n\t"
1930 "sve_$7 $dst, $8, $dst, tmp\n\t# convert $1 to $2 vector" %}
1931 ins_encode %{
1932 __ sve_$3(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src$$reg), __ $4);
1933 __ sve_$5(as_FloatRegister($tmp$$reg), __ $6, 0);
1934 __ sve_$7(as_FloatRegister($dst$$reg), __ $6, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
1935 __ sve_$7(as_FloatRegister($dst$$reg), __ $8, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
1936 %}
1937 ins_pipe(pipe_slow);
1938 %}')dnl
1939 dnl
1940 dnl
1941 define(`VECTOR_CAST_F2X_EXTEND1', `
1942 instruct vcvt$1to$2`'(vReg dst, vReg src)
1943 %{
1944 predicate(UseSVE > 0 &&
1945 n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1946 match(Set dst (VectorCast$1`'2X src));
1947 ins_cost(2 * SVE_COST);
1948 format %{ "sve_$3 $dst, $4, $src, $4\n\t"
1949 "sve_$5 $dst, $6, $dst\t# convert $1 to $2 vector" %}
1950 ins_encode %{
1951 __ sve_$3(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src$$reg), __ $4);
1952 __ sve_$5(as_FloatRegister($dst$$reg), __ $6, as_FloatRegister($dst$$reg));
1953 %}
1954 ins_pipe(pipe_slow);
1955 %}')dnl
1956 dnl
1957 dnl
1958 define(`VECTOR_CAST_F2X_NARROW3', `
1959 instruct vcvt$1to$2`'(vReg dst, vReg src, vReg tmp)
1960 %{
1961 predicate(UseSVE > 0 &&
1962 n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
1963 match(Set dst (VectorCast$1`'2X src));
1964 effect(TEMP_DEF dst, TEMP tmp);
1965 ins_cost(5 * SVE_COST);
1966 format %{ "sve_$3 $dst, $4, $src, $4\n\t"
1967 "sve_$5 $tmp, $6, 0\n\t"
1968 "sve_$7 $dst, $6, $dst, tmp\n\t"
1969 "sve_$7 $dst, $8, $dst, tmp\n\t"
1970 "sve_$7 $dst, $9, $dst, tmp\n\t# convert $1 to $2 vector" %}
1971 ins_encode %{
1972 __ sve_$3(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src$$reg), __ $4);
1973 __ sve_$5(as_FloatRegister($tmp$$reg), __ $6, 0);
1974 __ sve_$7(as_FloatRegister($dst$$reg), __ $6, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
1975 __ sve_$7(as_FloatRegister($dst$$reg), __ $8, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
1976 __ sve_$7(as_FloatRegister($dst$$reg), __ $9, as_FloatRegister($dst$$reg), as_FloatRegister($tmp$$reg));
1977 %}
1978 ins_pipe(pipe_slow);
1979 %}')dnl
1980 dnl
1981 VECTOR_CAST_EXTEND1(B, S, sunpklo, H)
1982 VECTOR_CAST_EXTEND2(B, I, sunpklo, H, S)
1983 VECTOR_CAST_EXTEND3(B, L, sunpklo, H, S, D)
1984 VECTOR_CAST_I2F_EXTEND2(B, F, sunpklo, H, S, scvtf)
1985 VECTOR_CAST_I2F_EXTEND3(B, D, sunpklo, H, S, D, scvtf)
1986 dnl
1987 VECTOR_CAST_NARROW1(S, B, dup, B, uzp1)
1988 VECTOR_CAST_EXTEND1(S, I, sunpklo, S)
1989 VECTOR_CAST_EXTEND2(S, L, sunpklo, S, D)
1990 VECTOR_CAST_X2F_EXTEND1(S, F, sunpklo, S, scvtf, S)
1991 VECTOR_CAST_I2F_EXTEND2(S, D, sunpklo, S, D, scvtf)
1992 dnl
1993 VECTOR_CAST_NARROW2(I, B, dup, H, uzp1, B)
1994 VECTOR_CAST_NARROW1(I, S, dup, H, uzp1)
1995 VECTOR_CAST_EXTEND1(I, L, sunpklo, D)
1996 VECTOR_CAST_X2X(I, F, scvtf, S)
1997 VECTOR_CAST_X2F_EXTEND1(I, D, sunpklo, D, scvtf, D)
1998 dnl
1999 VECTOR_CAST_NARROW3(L, B, dup, S, uzp1, H, B)
2000 VECTOR_CAST_NARROW2(L, S, dup, S, uzp1, H)
2001 VECTOR_CAST_NARROW1(L, I, dup, S, uzp1)
2002 VECTOR_CAST_X2F_NARROW1(L, F, scvtf, S, D, dup, S, uzp1)
2003 VECTOR_CAST_X2X(L, D, scvtf, D)
2004 dnl
2005 VECTOR_CAST_F2X_NARROW2(F, B, fcvtzs, S, dup, H, uzp1, B)
2006 VECTOR_CAST_F2X_NARROW1(F, S, fcvtzs, S, dup, H, uzp1)
2007 VECTOR_CAST_X2X(F, I, fcvtzs, S)
2008 VECTOR_CAST_F2X_EXTEND1(F, L, fcvtzs, S, sunpklo, D)
2009 VECTOR_CAST_X2F_EXTEND1(F, D, sunpklo, D, fcvt, S)
2010 dnl
2011 VECTOR_CAST_F2X_NARROW3(D, B, fcvtzs, D, dup, S, uzp1, H, B)
2012 VECTOR_CAST_F2X_NARROW2(D, S, fcvtzs, D, dup, S, uzp1, H)
2013 VECTOR_CAST_F2X_NARROW1(D, I, fcvtzs, D, dup, S, uzp1)
2014 VECTOR_CAST_X2X(D, L, fcvtzs, D)
2015 VECTOR_CAST_X2F_NARROW1(D, F, fcvt, S, D, dup, S, uzp1)
2016 dnl
2017 dnl
2018 // ------------------------------ Vector extract ---------------------------------
2019 define(`VECTOR_EXTRACT_SXT', `
2020 instruct extract$1`'($2 dst, vReg src, immI idx, pRegGov pTmp, rFlagsReg cr)
2021 %{
2022 predicate(UseSVE > 0);
2023 match(Set dst (Extract$1 src idx));
2024 effect(TEMP pTmp, KILL cr);
2025 ins_cost(2 * SVE_COST);
2026 format %{ "sve_extract $dst, $3, $pTmp, $src, $idx\n\t"
2027 "sbfmw $dst, $dst, 0U, $5\t# extract from vector($1)" %}
2028 ins_encode %{
2029 __ sve_extract(as_$4($dst$$reg), __ $3, as_PRegister($pTmp$$reg),
2030 as_FloatRegister($src$$reg), (int)($idx$$constant));
2031 __ sbfmw(as_$4($dst$$reg), as_$4($dst$$reg), 0U, $5);
2032 %}
2033 ins_pipe(pipe_slow);
2034 %}')dnl
2035 dnl $1 $2 $3 $4 $5
2036 VECTOR_EXTRACT_SXT(B, iRegINoSp, B, Register, 7U)
2037 VECTOR_EXTRACT_SXT(S, iRegINoSp, H, Register, 15U)
2038
2039 dnl
2040 define(`VECTOR_EXTRACT', `
2041 instruct extract$1`'($2 dst, vReg src, immI idx, pRegGov pTmp, rFlagsReg cr)
2042 %{
2043 predicate(UseSVE > 0);
2044 match(Set dst (Extract$1 src idx));
2045 effect(TEMP pTmp, KILL cr);
2046 ins_cost(2 * SVE_COST);
2047 format %{ "sve_extract $dst, $3, $pTmp, $src, $idx\t# extract from vector($1)" %}
2048 ins_encode %{
2049 __ sve_extract(as_$4($dst$$reg), __ $3, as_PRegister($pTmp$$reg),
2050 as_FloatRegister($src$$reg), (int)($idx$$constant));
2051 %}
2052 ins_pipe(pipe_slow);
2053 %}')dnl
2054 dnl $1 $2 $3 $4
2055 VECTOR_EXTRACT(I, iRegINoSp, S, Register)
2056 VECTOR_EXTRACT(L, iRegLNoSp, D, Register)
2057 VECTOR_EXTRACT(F, vRegF, S, FloatRegister)
2058 VECTOR_EXTRACT(D, vRegD, D, FloatRegister)
2059
2060 // ------------------------------- VectorTest ----------------------------------
2061 dnl
2062 dnl VTEST($1, $2, $3, $4 )
2063 dnl VTEST(op_name, pred, imm, cond)
2064 define(`VTEST', `
2065 instruct vtest_$1`'(iRegINoSp dst, vReg src1, vReg src2, pReg pTmp, rFlagsReg cr)
2066 %{
2067 predicate(UseSVE > 0 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2068 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::$2);
2069 match(Set dst (VectorTest src1 src2));
2070 effect(TEMP pTmp, KILL cr);
2071 ins_cost(SVE_COST);
2072 format %{ "sve_cmpeq $pTmp, $src1, $3\n\t"
2073 "csetw $dst, $4\t# VectorTest (sve) - $1" %}
2074 ins_encode %{
2075 // "src2" is not used for sve.
2076 BasicType bt = Matcher::vector_element_basic_type(this, $src1);
2077 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2078 __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), size,
2079 ptrue, as_FloatRegister($src1$$reg), $3);
2080 __ csetw(as_Register($dst$$reg), Assembler::$4);
2081 %}
2082 ins_pipe(pipe_slow);
2083 %}')dnl
2084 dnl
2085 VTEST(alltrue, overflow, 0, EQ)
2086 VTEST(anytrue, ne, -1, NE)
2087 dnl
2088 dnl
2089 dnl VTEST_PARTIAL($1, $2, $3, $4 )
2090 dnl VTEST_PARTIAL(op_name, pred, imm, cond)
2091 define(`VTEST_PARTIAL', `
2092 instruct vtest_$1_partial`'(iRegINoSp dst, vReg src1, vReg src2, pRegGov pTmp, rFlagsReg cr)
2093 %{
2094 predicate(UseSVE > 0 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2095 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::$2);
2096 match(Set dst (VectorTest src1 src2));
2097 effect(TEMP pTmp, KILL cr);
2098 ins_cost(SVE_COST);
2099 format %{ "vtest_$1_partial $dst, $src1, $src2\t# VectorTest partial (sve) - $1" %}
2100 ins_encode %{
2101 // "src2" is not used for sve.
2102 BasicType bt = Matcher::vector_element_basic_type(this, $src1);
2103 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2104 __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), size,
2105 Matcher::vector_length(this, $src1));
2106 __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), size,
2107 as_PRegister($pTmp$$reg), as_FloatRegister($src1$$reg), $3);
2108 __ csetw(as_Register($dst$$reg), Assembler::$4);
2109 %}
2110 ins_pipe(pipe_slow);
2111 %}')dnl
2112 dnl
2113 VTEST_PARTIAL(alltrue, overflow, 0, EQ)
2114 VTEST_PARTIAL(anytrue, ne, -1, NE)
2115
2116 // ------------------------------ Vector insert ---------------------------------
2117
2118 instruct insertI_small(vReg dst, vReg src, iRegIorL2I val, immI idx, pRegGov pTmp, rFlagsReg cr)
2119 %{
2120 predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
2121 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2122 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2123 n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2124 match(Set dst (VectorInsert (Binary src val) idx));
2125 effect(TEMP_DEF dst, TEMP pTmp, KILL cr);
2126 ins_cost(4 * SVE_COST);
2127 format %{ "sve_index $dst, -16, 1\t# (B/S/I)\n\t"
2128 "sve_cmpeq $pTmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2129 "sve_orr $dst, $src, $src\n\t"
2130 "sve_cpy $dst, $pTmp, $val\t# insert into vector (B/S/I)" %}
2131 ins_encode %{
2132 BasicType bt = Matcher::vector_element_basic_type(this, $src);
2133 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2134 __ sve_index(as_FloatRegister($dst$$reg), size, -16, 1);
2135 __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), size, ptrue,
2136 as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2137 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2138 __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pTmp$$reg), as_Register($val$$reg));
2139 %}
2140 ins_pipe(pipe_slow);
2141 %}
2142
2143 instruct insertF_small(vReg dst, vReg src, vRegF val, immI idx, pRegGov pTmp, rFlagsReg cr)
2144 %{
2145 predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
2146 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2147 match(Set dst (VectorInsert (Binary src val) idx));
2148 effect(TEMP_DEF dst, TEMP pTmp, KILL cr);
2149 ins_cost(4 * SVE_COST);
2150 format %{ "sve_index $dst, S, -16, 1\n\t"
2151 "sve_cmpeq $pTmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2152 "sve_orr $dst, $src, $src\n\t"
2153 "sve_cpy $dst, $pTmp, $val\t# insert into vector (F)" %}
2154 ins_encode %{
2155 __ sve_index(as_FloatRegister($dst$$reg), __ S, -16, 1);
2156 __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), __ S, ptrue,
2157 as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2158 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2159 __ sve_cpy(as_FloatRegister($dst$$reg), __ S, as_PRegister($pTmp$$reg), as_FloatRegister($val$$reg));
2160 %}
2161 ins_pipe(pipe_slow);
2162 %}
2163
2164 instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp1, pRegGov pTmp, rFlagsReg cr)
2165 %{
2166 predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
2167 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2168 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2169 n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2170 match(Set dst (VectorInsert (Binary src val) idx));
2171 effect(TEMP_DEF dst, TEMP tmp1, TEMP pTmp, KILL cr);
2172 ins_cost(5 * SVE_COST);
2173 format %{ "sve_index $tmp1, 0, 1\t# (B/S/I)\n\t"
2174 "sve_dup $dst, $idx\t# (B/S/I)\n\t"
2175 "sve_cmpeq $pTmp, $tmp1, $dst\n\t"
2176 "sve_orr $dst, $src, $src\n\t"
2177 "sve_cpy $dst, $pTmp, $val\t# insert into vector (B/S/I)" %}
2178 ins_encode %{
2179 BasicType bt = Matcher::vector_element_basic_type(this, $src);
2180 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2181 __ sve_index(as_FloatRegister($tmp1$$reg), size, 0, 1);
2182 __ sve_dup(as_FloatRegister($dst$$reg), size, (int)($idx$$constant));
2183 __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), size, ptrue,
2184 as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
2185 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2186 __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pTmp$$reg), as_Register($val$$reg));
2187 %}
2188 ins_pipe(pipe_slow);
2189 %}
2190 dnl
2191 dnl
2192 define(`VECTOR_INSERT_D', `
2193 instruct insert$1`'(vReg dst, vReg src, $2 val, immI idx, pRegGov pTmp, rFlagsReg cr)
2194 %{
2195 predicate(UseSVE > 0 &&
2196 n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($1));
2197 match(Set dst (VectorInsert (Binary src val) idx));
2198 effect(TEMP_DEF dst, TEMP pTmp, KILL cr);
2199 ins_cost(4 * SVE_COST);
2200 format %{ "sve_index $dst, $3, -16, 1\n\t"
2201 "sve_cmpeq $pTmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2202 "sve_orr $dst, $src, $src\n\t"
2203 "sve_cpy $dst, $pTmp, $val\t# insert into vector ($1)" %}
2204 ins_encode %{
2205 __ sve_index(as_FloatRegister($dst$$reg), __ $3, -16, 1);
2206 __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), __ $3, ptrue,
2207 as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2208 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2209 __ sve_cpy(as_FloatRegister($dst$$reg), __ $3, as_PRegister($pTmp$$reg), as_$4($val$$reg));
2210 %}
2211 ins_pipe(pipe_slow);
2212 %}')dnl
2213 dnl $1 $2 $3 $4
2214 VECTOR_INSERT_D(L, iRegL, D, Register)
2215 VECTOR_INSERT_D(D, vRegD, D, FloatRegister)
2216
2217 instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pTmp, rFlagsReg cr)
2218 %{
2219 predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
2220 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2221 match(Set dst (VectorInsert (Binary src val) idx));
2222 effect(TEMP_DEF dst, TEMP tmp1, TEMP pTmp, KILL cr);
2223 ins_cost(5 * SVE_COST);
2224 format %{ "sve_index $tmp1, S, 0, 1\n\t"
2225 "sve_dup $dst, S, $idx\n\t"
2226 "sve_cmpeq $pTmp, $tmp1, $dst\n\t"
2227 "sve_orr $dst, $src, $src\n\t"
2228 "sve_cpy $dst, $pTmp, $val\t# insert into vector (F)" %}
2229 ins_encode %{
2230 __ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
2231 __ sve_dup(as_FloatRegister($dst$$reg), __ S, (int)($idx$$constant));
2232 __ sve_cmp(Assembler::EQ, as_PRegister($pTmp$$reg), __ S, ptrue,
2233 as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
2234 __ sve_orr(as_FloatRegister($dst$$reg),
2235 as_FloatRegister($src$$reg),
2236 as_FloatRegister($src$$reg));
2237 __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
2238 as_PRegister($pTmp$$reg), as_FloatRegister($val$$reg));
2239 %}
2240 ins_pipe(pipe_slow);
2241 %}
2242
2243 // ------------------------------ Vector shuffle -------------------------------
2244
2245 instruct loadshuffleB(vReg dst, vReg src)
2246 %{
2247 predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2248 match(Set dst (VectorLoadShuffle src));
2249 ins_cost(SVE_COST);
2250 format %{ "sve_orr $dst, $src, $src\t# vector load shuffle (B)" %}
2251 ins_encode %{
2252 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
2253 __ sve_orr(as_FloatRegister($dst$$reg),
2254 as_FloatRegister($src$$reg),
2255 as_FloatRegister($src$$reg));
2256 }
2257 %}
2258 ins_pipe(pipe_slow);
2259 %}
2260
2261 instruct loadshuffleS(vReg dst, vReg src)
2262 %{
2263 predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
2264 match(Set dst (VectorLoadShuffle src));
2265 ins_cost(SVE_COST);
2266 format %{ "sve_uunpklo $dst, $src\t# vector load shuffle (B to H)" %}
2267 ins_encode %{
2268 __ sve_uunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
2269 %}
2270 ins_pipe(pipe_slow);
2271 %}
2272
2273 instruct loadshuffleI(vReg dst, vReg src)
2274 %{
2275 predicate(UseSVE > 0 &&
2276 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2277 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2278 match(Set dst (VectorLoadShuffle src));
2279 ins_cost(2 * SVE_COST);
2280 format %{ "sve_uunpklo $dst, H, $src\n\t"
2281 "sve_uunpklo $dst, S, $dst\t# vector load shuffle (B to S)" %}
2282 ins_encode %{
2283 __ sve_uunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
2284 __ sve_uunpklo(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($dst$$reg));
2285 %}
2286 ins_pipe(pipe_slow);
2287 %}
2288
2289 instruct loadshuffleL(vReg dst, vReg src)
2290 %{
2291 predicate(UseSVE > 0 &&
2292 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2293 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2294 match(Set dst (VectorLoadShuffle src));
2295 ins_cost(3 * SVE_COST);
2296 format %{ "sve_uunpklo $dst, H, $src\n\t"
2297 "sve_uunpklo $dst, S, $dst\n\t"
2298 "sve_uunpklo $dst, D, $dst\t# vector load shuffle (B to D)" %}
2299 ins_encode %{
2300 __ sve_uunpklo(as_FloatRegister($dst$$reg), __ H, as_FloatRegister($src$$reg));
2301 __ sve_uunpklo(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($dst$$reg));
2302 __ sve_uunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($dst$$reg));
2303 %}
2304 ins_pipe(pipe_slow);
2305 %}
2306
2307 // ------------------------------ Vector rearrange -------------------------------
2308
2309 instruct rearrange(vReg dst, vReg src, vReg shuffle)
2310 %{
2311 predicate(UseSVE > 0);
2312 match(Set dst (VectorRearrange src shuffle));
2313 ins_cost(SVE_COST);
2314 format %{ "sve_tbl $dst, $src, $shuffle\t# vector rearrange" %}
2315 ins_encode %{
2316 BasicType bt = Matcher::vector_element_basic_type(this, $src);
2317 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2318 __ sve_tbl(as_FloatRegister($dst$$reg), size,
2319 as_FloatRegister($src$$reg), as_FloatRegister($shuffle$$reg));
2320 %}
2321 ins_pipe(pipe_slow);
2322 %}
2323
2324 // ------------------------------ Vector Load Gather ---------------------------------
2325
2326 instruct gatherI(vReg dst, indirect mem, vReg idx) %{
2327 predicate(UseSVE > 0 &&
2328 n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
2329 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2330 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2331 match(Set dst (LoadVectorGather mem idx));
2332 ins_cost(SVE_COST);
2333 format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (I/F)" %}
2334 ins_encode %{
2335 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), ptrue,
2336 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2337 %}
2338 ins_pipe(pipe_slow);
2339 %}
2340
2341 instruct gatherL(vReg dst, indirect mem, vReg idx) %{
2342 predicate(UseSVE > 0 &&
2343 n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
2344 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2345 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2346 match(Set dst (LoadVectorGather mem idx));
2347 ins_cost(2 * SVE_COST);
2348 format %{ "sve_uunpklo $idx, $idx\n\t"
2349 "load_vector_gather $dst, $mem, $idx\t# vector load gather (L/D)" %}
2350 ins_encode %{
2351 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2352 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), ptrue, as_Register($mem$$base), as_FloatRegister($idx$$reg));
2353 %}
2354 ins_pipe(pipe_slow);
2355 %}
2356
2357 // ------------------------------ Vector Load Gather Partial-------------------------------
2358
2359 instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov pTmp, rFlagsReg cr) %{
2360 predicate(UseSVE > 0 &&
2361 n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
2362 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2363 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2364 match(Set dst (LoadVectorGather mem idx));
2365 effect(TEMP pTmp, KILL cr);
2366 ins_cost(2 * SVE_COST + INSN_COST);
2367 format %{ "sve_whilelo_zr_imm $pTmp, vector_length\n\t"
2368 "load_vector_gather $dst, $pTmp, $mem, $idx\t# vector load gather partial (I/F)" %}
2369 ins_encode %{
2370 __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), __ S,
2371 Matcher::vector_length(this));
2372 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($pTmp$$reg),
2373 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2374 %}
2375 ins_pipe(pipe_slow);
2376 %}
2377
2378 instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov pTmp, rFlagsReg cr) %{
2379 predicate(UseSVE > 0 &&
2380 n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
2381 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2382 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2383 match(Set dst (LoadVectorGather mem idx));
2384 effect(TEMP pTmp, KILL cr);
2385 ins_cost(3 * SVE_COST + INSN_COST);
2386 format %{ "sve_whilelo_zr_imm $pTmp, vector_length\n\t"
2387 "sve_uunpklo $idx, $idx\n\t"
2388 "load_vector_gather $dst, $pTmp, $mem, $idx\t# vector load gather partial (L/D)" %}
2389 ins_encode %{
2390 __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), __ D,
2391 Matcher::vector_length(this));
2392 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2393 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($pTmp$$reg),
2394 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2395 %}
2396 ins_pipe(pipe_slow);
2397 %}
2398
2399 // ------------------------------ Vector Store Scatter -------------------------------
2400
2401 instruct scatterI(indirect mem, vReg src, vReg idx) %{
2402 predicate(UseSVE > 0 &&
2403 n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
2404 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2405 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2406 match(Set mem (StoreVectorScatter mem (Binary src idx)));
2407 ins_cost(SVE_COST);
2408 format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (I/F)" %}
2409 ins_encode %{
2410 __ sve_st1w_scatter(as_FloatRegister($src$$reg), ptrue,
2411 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2412 %}
2413 ins_pipe(pipe_slow);
2414 %}
2415
2416 instruct scatterL(indirect mem, vReg src, vReg idx) %{
2417 predicate(UseSVE > 0 &&
2418 n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
2419 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2420 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2421 match(Set mem (StoreVectorScatter mem (Binary src idx)));
2422 ins_cost(2 * SVE_COST);
2423 format %{ "sve_uunpklo $idx, $idx\n\t"
2424 "store_vector_scatter $mem, $idx, $src\t# vector store scatter (L/D)" %}
2425 ins_encode %{
2426 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D,
2427 as_FloatRegister($idx$$reg));
2428 __ sve_st1d_scatter(as_FloatRegister($src$$reg), ptrue,
2429 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2430 %}
2431 ins_pipe(pipe_slow);
2432 %}
2433
2434 // ------------------------------ Vector Store Scatter Partial-------------------------------
2435
2436 instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov pTmp, rFlagsReg cr) %{
2437 predicate(UseSVE > 0 &&
2438 n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
2439 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2440 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2441 match(Set mem (StoreVectorScatter mem (Binary src idx)));
2442 effect(TEMP pTmp, KILL cr);
2443 ins_cost(2 * SVE_COST + INSN_COST);
2444 format %{ "sve_whilelo_zr_imm $pTmp, vector_length\n\t"
2445 "store_vector_scatter $mem, $pTmp, $idx, $src\t# vector store scatter partial (I/F)" %}
2446 ins_encode %{
2447 __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), __ S,
2448 Matcher::vector_length(this, $src));
2449 __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($pTmp$$reg),
2450 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2451 %}
2452 ins_pipe(pipe_slow);
2453 %}
2454
2455 instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov pTmp, rFlagsReg cr) %{
2456 predicate(UseSVE > 0 &&
2457 n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
2458 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2459 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2460 match(Set mem (StoreVectorScatter mem (Binary src idx)));
2461 effect(TEMP pTmp, KILL cr);
2462 ins_cost(3 * SVE_COST + INSN_COST);
2463 format %{ "sve_whilelo_zr_imm $pTmp, vector_length\n\t"
2464 "sve_uunpklo $idx, $idx\n\t"
2465 "store_vector_scatter $mem, $pTmp, $idx, $src\t# vector store scatter partial (L/D)" %}
2466 ins_encode %{
2467 __ sve_whilelo_zr_imm(as_PRegister($pTmp$$reg), __ D,
2468 Matcher::vector_length(this, $src));
2469 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2470 __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($pTmp$$reg),
2471 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2472 %}
2473 ins_pipe(pipe_slow);
2474 %}
2475
2476
2477 // ------------------------------ Vector Load Const -------------------------------
2478
2479 instruct loadconB(vReg dst, immI0 src) %{
2480 predicate(UseSVE > 0 &&
2481 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2482 match(Set dst (VectorLoadConst src));
2483 ins_cost(SVE_COST);
2484 format %{ "sve_index $dst, 0, 1\t# generate iota indices" %}
2485 ins_encode %{
2486 __ sve_index(as_FloatRegister($dst$$reg), __ B, 0, 1);
2487 %}
2488 ins_pipe(pipe_slow);
2489 %}
2490
2491 // Intrisics for String.indexOf(char)
2492
2493 dnl
2494 define(`STRING_INDEXOF_CHAR', `
2495 instruct string$1_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
2496 iRegI_R0 result, vReg ztmp1, vReg ztmp2,
2497 pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
2498 %{
2499 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
2500 predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::$1));
2501 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
2502
2503 format %{ "String$2 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
2504
2505 ins_encode %{
2506 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
2507 as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
2508 as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), $3 /* isL */);
2509 %}
2510 ins_pipe(pipe_class_memory);
2511 %}')dnl
2512 dnl $1 $2 $3
2513 STRING_INDEXOF_CHAR(L, Latin1, true)
2514 STRING_INDEXOF_CHAR(U, UTF16, false)
2515
2516 dnl
2517 dnl VMASK_REDUCTION($1, $2, $3 )
2518 dnl VMASK_REDUCTION(suffix, op_name, cost)
2519 define(`VMASK_REDUCTION', `
2520 instruct vmask_$1(iRegINoSp dst, vReg src, pReg ptmp, rFlagsReg cr) %{
2521 predicate(UseSVE > 0 &&
2522 n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2523 match(Set dst ($2 src));
2524 effect(TEMP ptmp, KILL cr);
2525 ins_cost($3 * SVE_COST);
2526 format %{ "vmask_$1 $dst, $src\t# vector mask $1 (sve)" %}
2527 ins_encode %{
2528 __ sve_vmask_reduction(this->ideal_Opcode(), $dst$$Register, __ B,
2529 as_FloatRegister($src$$reg), ptrue, as_PRegister($ptmp$$reg));
2530 %}
2531 ins_pipe(pipe_slow);
2532 %}')dnl
2533 dnl
2534 // ---------------------------- Vector mask reductions ---------------------------
2535 VMASK_REDUCTION(truecount, VectorMaskTrueCount, 2)
2536 VMASK_REDUCTION(firsttrue, VectorMaskFirstTrue, 3)
2537 VMASK_REDUCTION(lasttrue, VectorMaskLastTrue, 4)
2538 dnl
2539 dnl VMASK_REDUCTION_PARTIAL($1, $2, $3 )
2540 dnl VMASK_REDUCTION_PARTIAL(suffix, op_name, cost)
2541 define(`VMASK_REDUCTION_PARTIAL', `
2542 instruct vmask_$1_partial(iRegINoSp dst, vReg src, pRegGov ifelse($1, `firsttrue', `pgtmp, pReg ptmp', `ptmp'), rFlagsReg cr) %{
2543 predicate(UseSVE > 0 &&
2544 n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2545 match(Set dst ($2 src));
2546 effect(TEMP ifelse($1, `firsttrue', `pgtmp, TEMP ptmp', `ptmp'), KILL cr);
2547 ins_cost($3 * SVE_COST);
2548 format %{ "vmask_$1 $dst, $src\t# vector mask $1 partial (sve)" %}
2549 ins_encode %{
2550 __ sve_whilelo_zr_imm(as_PRegister(ifelse($1, `firsttrue', `$pgtmp', `$ptmp')$$reg), __ B,
2551 Matcher::vector_length(this, $src));
2552 __ sve_vmask_reduction(this->ideal_Opcode(), $dst$$Register, __ B, as_FloatRegister($src$$reg),
2553 as_PRegister(ifelse($1, `firsttrue', `$pgtmp', `$ptmp')$$reg), as_PRegister($ptmp$$reg));
2554 %}
2555 ins_pipe(pipe_slow);
2556 %}')dnl
2557 dnl
2558 VMASK_REDUCTION_PARTIAL(truecount, VectorMaskTrueCount, 3)
2559 VMASK_REDUCTION_PARTIAL(firsttrue, VectorMaskFirstTrue, 4)
2560 VMASK_REDUCTION_PARTIAL(lasttrue, VectorMaskLastTrue, 5)
2561
2562 dnl
2563 dnl VSTOREMASK_REDUCTION($1, $2, $3 )
2564 dnl VSTOREMASK_REDUCTION(suffix, op_name, cost)
2565 define(`VSTOREMASK_REDUCTION', `
2566 instruct vstoremask_$1(iRegINoSp dst, vReg src, immI esize, pReg ptmp, rFlagsReg cr) %{
2567 predicate(UseSVE > 0 &&
2568 n->in(1)->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2569 match(Set dst ($2 (VectorStoreMask src esize)));
2570 effect(TEMP ptmp, KILL cr);
2571 ins_cost($3 * SVE_COST);
2572 format %{ "vstoremask_$1 $dst, $src\t# vector mask $1 (sve)" %}
2573 ins_encode %{
2574 unsigned size = $esize$$constant;
2575 assert(size == 1 || size == 2 || size == 4 || size == 8, "unsupported element size");
2576 Assembler::SIMD_RegVariant variant = __ elemBytes_to_regVariant(size);
2577 __ sve_vmask_reduction(this->ideal_Opcode(), $dst$$Register, variant, as_FloatRegister($src$$reg),
2578 ptrue, as_PRegister($ptmp$$reg), Matcher::vector_length(this, $src));
2579 %}
2580 ins_pipe(pipe_slow);
2581 %}')dnl
2582 dnl
2583 // ----------------- Vector mask reductions combined with VectorMaskStore ---------------
2584 VSTOREMASK_REDUCTION(truecount, VectorMaskTrueCount, 2)
2585 VSTOREMASK_REDUCTION(firsttrue, VectorMaskFirstTrue, 3)
2586 VSTOREMASK_REDUCTION(lasttrue, VectorMaskLastTrue, 4)
2587 dnl
2588 dnl VSTOREMASK_REDUCTION_PARTIAL($1, $2, $3 )
2589 dnl VSTOREMASK_REDUCTION_PARTIAL(suffix, op_name, cost)
2590 define(`VSTOREMASK_REDUCTION_PARTIAL', `
2591 instruct vstoremask_$1_partial(iRegINoSp dst, vReg src, immI esize, pRegGov ifelse($1, `firsttrue', `pgtmp, pReg ptmp', `ptmp'), rFlagsReg cr) %{
2592 predicate(UseSVE > 0 &&
2593 n->in(1)->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
2594 match(Set dst ($2 (VectorStoreMask src esize)));
2595 effect(TEMP ifelse($1, `firsttrue', `pgtmp, TEMP ptmp', `ptmp'), KILL cr);
2596 ins_cost($3 * SVE_COST);
2597 format %{ "vstoremask_$1 $dst, $src\t# vector mask $1 partial (sve)" %}
2598 ins_encode %{
2599 unsigned size = $esize$$constant;
2600 assert(size == 1 || size == 2 || size == 4 || size == 8, "unsupported element size");
2601 Assembler::SIMD_RegVariant variant = __ elemBytes_to_regVariant(size);
2602 __ sve_whilelo_zr_imm(as_PRegister(ifelse($1, `firsttrue', `$pgtmp', `$ptmp')$$reg), variant,
2603 Matcher::vector_length(this, $src));
2604 __ sve_vmask_reduction(this->ideal_Opcode(), $dst$$Register, variant, as_FloatRegister($src$$reg),
2605 as_PRegister(ifelse($1, `firsttrue', `$pgtmp', `$ptmp')$$reg), as_PRegister($ptmp$$reg), MaxVectorSize / size);
2606 %}
2607 ins_pipe(pipe_slow);
2608 %}')dnl
2609 dnl
2610 VSTOREMASK_REDUCTION_PARTIAL(truecount, VectorMaskTrueCount, 3)
2611 VSTOREMASK_REDUCTION_PARTIAL(firsttrue, VectorMaskFirstTrue, 4)
2612 VSTOREMASK_REDUCTION_PARTIAL(lasttrue, VectorMaskLastTrue, 5)
|
66 constraint(ALLOC_IN_RC(ptr_reg));
67 match(AddP reg off);
68 op_cost(0);
69 format %{ "[$reg, $off]" %}
70 interface(MEMORY_INTER) %{
71 base($reg);
72 `index'(0xffffffff);
73 scale(0x0);
74 disp($off);
75 %}
76 %}')dnl
77 OPERAND_VMEMORYA_INDIRECT_OFFSET(I, 4)
78 OPERAND_VMEMORYA_INDIRECT_OFFSET(L, 4)
79
80 // The indOff of vmemA is valid only when the vector element (load to/store from)
81 // size equals to memory element (load from/store to) size.
82 opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);
83
84 source_hpp %{
85 bool op_sve_supported(int opcode, int vlen, BasicType bt);
86 bool masked_op_sve_supported(int opcode, int vlen, BasicType bt);
87 %}
88
89 source %{
90
91 typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
92 PRegister Pg, const Address &adr);
93
94 // Predicated load/store, with optional ptrue to all elements of given predicate register.
95 static void loadStoreA_predicated(C2_MacroAssembler masm, bool is_store, FloatRegister reg,
96 PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
97 int opcode, Register base, int index, int size, int disp) {
98 sve_mem_insn_predicate insn;
99 int mesize = type2aelembytes(mem_elem_bt);
100 if (index == -1) {
101 assert(size == 0, "unsupported address mode: scale size = %d", size);
102 switch(mesize) {
103 case 1:
104 insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
105 break;
106 case 2:
123 ShouldNotReachHere();
124 }
125 }
126
127 bool op_sve_supported(int opcode, int vlen, BasicType bt) {
128 int length_in_bytes = vlen * type2aelembytes(bt);
129 switch (opcode) {
130 case Op_MulAddVS2VI:
131 // No multiply reduction instructions
132 case Op_MulReductionVD:
133 case Op_MulReductionVF:
134 case Op_MulReductionVI:
135 case Op_MulReductionVL:
136 // Others
137 case Op_ExtractC:
138 case Op_ExtractUB:
139 return false;
140 // Vector API specific
141 case Op_VectorLoadShuffle:
142 case Op_VectorRearrange:
143 return vlen >= 4 && length_in_bytes <= MaxVectorSize;
144 case Op_LoadVector:
145 case Op_StoreVector:
146 return Matcher::vector_size_supported(bt, vlen);
147 default:
148 break;
149 }
150 // By default, we only support vector operations with no less than 8 bytes and 2 elements.
151 return 8 <= length_in_bytes && length_in_bytes <= MaxVectorSize && vlen >= 2;
152 }
153
154 bool masked_op_sve_supported(int opcode, int vlen, BasicType bt) {
155 if (opcode == Op_VectorRearrange) {
156 return false;
157 }
158 return op_sve_supported(opcode, vlen, bt);
159 }
160
161 %}
162
163 definitions %{
164 int_def SVE_COST (200, 200);
165 %}
166
167 dnl
168 dnl ELEMENT_SHORT_CHART($1, $2)
169 dnl ELEMENT_SHORT_CHART(etype, node)
170 define(`ELEMENT_SHORT_CHAR',`ifelse(`$1', `T_SHORT',
171 `($2->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
172 ($2->bottom_type()->is_vect()->element_basic_type() == T_CHAR))',
173 `($2->bottom_type()->is_vect()->element_basic_type() == $1)')')dnl
174 dnl
175
176 // All SVE instructions
177
178 // vector load/store
179
180 // Unpredicated vector load/store
219 match(Set ifelse(load, $3, dst (LoadVector mem), mem (StoreVector mem src)));
220 ins_cost(4 * INSN_COST);
221 format %{ "$1 ifelse(load, $3, `$dst,$mem', `$mem,$src')\t# vector ($6 bits)" %}
222 ins_encode( `aarch64_enc_'ifelse(load, $3, ldr, str)v$2($7, mem) );
223 ins_pipe(v$3`_reg_mem'ifelse(eval($4 * 8), 128, 128, 64));
224 %}')dnl
225 dnl $1 $2 $3 $4 $5 $6 $7
226 VLoadStore(ldrh, H, load, 2, D, 16, dst)
227 VLoadStore(strh, H, store, 2, D, 16, src)
228 VLoadStore(ldrs, S, load, 4, D, 32, dst)
229 VLoadStore(strs, S, store, 4, D, 32, src)
230 VLoadStore(ldrd, D, load, 8, D, 64, dst)
231 VLoadStore(strd, D, store, 8, D, 64, src)
232 VLoadStore(ldrq, Q, load, 16, X, 128, dst)
233 VLoadStore(strq, Q, store, 16, X, 128, src)
234
235 // Predicated vector load/store, based on the vector length of the node.
236 // Only load/store values in the range of the memory_size. This is needed
237 // when the memory_size is lower than the hardware supported max vector size.
238 // And this might happen for Vector API mask vector load/store.
239 instruct loadV_partial(vReg dst, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
240 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() > 16 &&
241 n->as_LoadVector()->memory_size() < MaxVectorSize);
242 match(Set dst (LoadVector mem));
243 effect(TEMP pgtmp, KILL cr);
244 ins_cost(6 * SVE_COST);
245 format %{ "sve_whilelo_zr_imm $pgtmp, vector_length\n\t"
246 "sve_ldr $dst, $pgtmp, $mem\t# load vector partial" %}
247 ins_encode %{
248 BasicType bt = Matcher::vector_element_basic_type(this);
249 __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
250 Matcher::vector_length(this));
251 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
252 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg,
253 as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
254 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
255 %}
256 ins_pipe(pipe_slow);
257 %}
258
259 instruct storeV_partial(vReg src, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
260 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() > 16 &&
261 n->as_StoreVector()->memory_size() < MaxVectorSize);
262 match(Set mem (StoreVector mem src));
263 effect(TEMP pgtmp, KILL cr);
264 ins_cost(5 * SVE_COST);
265 format %{ "sve_whilelo_zr_imm $pgtmp, vector_length\n\t"
266 "sve_str $src, $pgtmp, $mem\t# store vector partial" %}
267 ins_encode %{
268 BasicType bt = Matcher::vector_element_basic_type(this, $src);
269 __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
270 Matcher::vector_length(this, $src));
271 FloatRegister src_reg = as_FloatRegister($src$$reg);
272 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg,
273 as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
274 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
275 %}
276 ins_pipe(pipe_slow);
277 %}
278
279 // vector load/store - predicated
280
281 instruct loadV_masked(vReg dst, vmemA mem, pRegGov pg) %{
282 predicate(UseSVE > 0 &&
283 n->as_LoadVector()->memory_size() == MaxVectorSize);
284 match(Set dst (LoadVectorMasked mem pg));
285 ins_cost(4 * SVE_COST);
286 format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated (sve)" %}
287 ins_encode %{
288 BasicType bt = Matcher::vector_element_basic_type(this);
289 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
290 as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
291 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
292 %}
293 ins_pipe(pipe_slow);
294 %}
295
296 instruct loadV_masked_partial(vReg dst, vmemA mem, pRegGov pg, pRegGov pgtmp, rFlagsReg cr) %{
297 predicate(UseSVE > 0 &&
298 n->as_LoadVector()->memory_size() < MaxVectorSize);
299 match(Set dst (LoadVectorMasked mem pg));
300 effect(TEMP pgtmp, KILL cr);
301 ins_cost(6 * SVE_COST);
302 format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated partial (sve)" %}
303 ins_encode %{
304 BasicType bt = Matcher::vector_element_basic_type(this);
305 __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
306 Matcher::vector_length(this));
307 __ sve_and(as_PRegister($pgtmp$$reg), as_PRegister($pgtmp$$reg),
308 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
309 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
310 as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
311 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
312 %}
313 ins_pipe(pipe_slow);
314 %}
315
316 instruct storeV_masked(vReg src, vmemA mem, pRegGov pg) %{
317 predicate(UseSVE > 0 &&
318 n->as_StoreVector()->memory_size() == MaxVectorSize);
319 match(Set mem (StoreVectorMasked mem (Binary src pg)));
320 ins_cost(4 * SVE_COST);
321 format %{ "sve_str $mem, $pg, $src\t# store vector predicated (sve)" %}
322 ins_encode %{
323 BasicType bt = Matcher::vector_element_basic_type(this, $src);
324 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
325 as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
326 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
327 %}
328 ins_pipe(pipe_slow);
329 %}
330
331 instruct storeV_masked_partial(vReg src, vmemA mem, pRegGov pg, pRegGov pgtmp, rFlagsReg cr) %{
332 predicate(UseSVE > 0 &&
333 n->as_StoreVector()->memory_size() < MaxVectorSize);
334 match(Set mem (StoreVectorMasked mem (Binary src pg)));
335 effect(TEMP pgtmp, KILL cr);
336 ins_cost(6 * SVE_COST);
337 format %{ "sve_str $mem, $pg, $src\t# store vector predicated partial (sve)" %}
338 ins_encode %{
339 BasicType bt = Matcher::vector_element_basic_type(this, $src);
340 __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
341 Matcher::vector_length(this, $src));
342 __ sve_and(as_PRegister($pgtmp$$reg), as_PRegister($pgtmp$$reg),
343 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
344 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
345 as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
346 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
347 %}
348 ins_pipe(pipe_slow);
349 %}
350
351 dnl
352 dnl MASKALL_IMM($1, $2 )
353 dnl MASKALL_IMM(type, size)
354 define(`MASKALL_IMM', `
355 instruct vmaskAll_imm$1(pRegGov dst, imm$1 src) %{
356 predicate(UseSVE > 0);
357 match(Set dst (MaskAll src));
358 ins_cost(SVE_COST);
359 format %{ "sve_ptrue/sve_pfalse $dst\t# mask all (sve) ($2)" %}
360 ins_encode %{
361 ifelse($1, `I', int, long) con = (ifelse($1, `I', int, long))$src$$constant;
362 if (con == 0) {
363 __ sve_pfalse(as_PRegister($dst$$reg));
364 } else {
365 assert(con == -1, "invalid constant value for mask");
366 BasicType bt = Matcher::vector_element_basic_type(this);
367 __ sve_ptrue(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt));
368 }
369 %}
370 ins_pipe(pipe_slow);
371 %}')dnl
372 dnl
373 dnl MASKALL($1, $2 )
374 dnl MASKALL(type, size)
375 define(`MASKALL', `
376 instruct vmaskAll$1(pRegGov dst, ifelse($1, `I', iRegIorL2I, iRegL) src, vReg tmp, rFlagsReg cr) %{
377 predicate(UseSVE > 0);
378 match(Set dst (MaskAll src));
379 effect(TEMP tmp, KILL cr);
380 ins_cost(2 * SVE_COST);
381 format %{ "sve_dup $tmp, $src\n\t"
382 "sve_cmpne $dst, $tmp, 0\t# mask all (sve) ($2)" %}
383 ins_encode %{
384 BasicType bt = Matcher::vector_element_basic_type(this);
385 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
386 __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
387 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
388 %}
389 ins_pipe(pipe_slow);
390 %}')dnl
391 dnl
392 // maskAll
393 MASKALL_IMM(I, B/H/S)
394 MASKALL(I, B/H/S)
395 MASKALL_IMM(L, D)
396 MASKALL(L, D)
397
398 dnl
399 dnl MASK_LOGICAL_OP($1, $2, $3 )
400 dnl MASK_LOGICAL_OP(insn_name, op_name, insn)
401 define(`MASK_LOGICAL_OP', `
402 instruct vmask_$1(pRegGov pd, pRegGov pn, pRegGov pm) %{
403 predicate(UseSVE > 0);
404 match(Set pd ($2 pn pm));
405 ins_cost(SVE_COST);
406 format %{ "$3 $pd, $pn, $pm\t# predicate (sve)" %}
407 ins_encode %{
408 __ $3(as_PRegister($pd$$reg), ptrue,
409 as_PRegister($pn$$reg), as_PRegister($pm$$reg));
410 %}
411 ins_pipe(pipe_slow);
412 %}')dnl
413 dnl
414 // mask logical and/or/xor
415 MASK_LOGICAL_OP(and, AndVMask, sve_and)
416 MASK_LOGICAL_OP(or, OrVMask, sve_orr)
417 MASK_LOGICAL_OP(xor, XorVMask, sve_eor)
418
419 dnl
420 dnl MASK_LOGICAL_AND_NOT($1, $2 )
421 dnl MASK_LOGICAL_AND_NOT(type, size)
422 define(`MASK_LOGICAL_AND_NOT', `
423 instruct vmask_and_not$1(pRegGov pd, pRegGov pn, pRegGov pm, imm$1_M1 m1) %{
424 predicate(UseSVE > 0);
425 match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
426 ins_cost(SVE_COST);
427 format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) ($2)" %}
428 ins_encode %{
429 __ sve_bic(as_PRegister($pd$$reg), ptrue,
430 as_PRegister($pn$$reg), as_PRegister($pm$$reg));
431 %}
432 ins_pipe(pipe_slow);
433 %}')dnl
434 dnl
435 // mask logical and_not
436 MASK_LOGICAL_AND_NOT(I, B/H/S)
437 MASK_LOGICAL_AND_NOT(L, D)
438
439 // vector reinterpret
440
441 instruct reinterpret(vReg dst) %{
442 predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() ==
443 n->in(1)->bottom_type()->is_vect()->length_in_bytes()); // src == dst
444 match(Set dst (VectorReinterpret dst));
445 ins_cost(0);
446 format %{ "# reinterpret $dst\t# do nothing" %}
447 ins_encode %{
448 // empty
449 %}
450 ins_pipe(pipe_class_empty);
451 %}
452
453 instruct reinterpretResize(vReg dst, vReg src, pRegGov pgtmp, rFlagsReg cr) %{
454 predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() !=
455 n->in(1)->bottom_type()->is_vect()->length_in_bytes()); // src != dst
456 match(Set dst (VectorReinterpret src));
457 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
458 ins_cost(3 * SVE_COST);
459 format %{ "reinterpretResize $dst, $src\t# vector (sve)" %}
460 ins_encode %{
461 uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
462 uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
463 uint length_in_bytes_resize = length_in_bytes_src < length_in_bytes_dst ?
464 length_in_bytes_src : length_in_bytes_dst;
465 assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
466 "invalid vector length");
467 __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ B, length_in_bytes_resize);
468 __ sve_dup(as_FloatRegister($dst$$reg), __ B, 0);
469 __ sve_sel(as_FloatRegister($dst$$reg), __ B, as_PRegister($pgtmp$$reg),
470 as_FloatRegister($src$$reg), as_FloatRegister($dst$$reg));
471 %}
472 ins_pipe(pipe_slow);
473 %}
474
475 // vector mask reinterpret
476
477 instruct vmask_reinterpret_same_esize(pRegGov dst_src) %{
478 predicate(UseSVE > 0 &&
479 n->as_Vector()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
480 n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
481 match(Set dst_src (VectorReinterpret dst_src));
482 ins_cost(0);
483 format %{ "# vmask_reinterpret $dst_src\t# do nothing" %}
484 ins_encode %{
485 // empty
486 %}
487 ins_pipe(pipe_class_empty);
488 %}
489
490 instruct vmask_reinterpret_diff_esize(pRegGov dst, pRegGov src, vReg tmp, rFlagsReg cr) %{
491 predicate(UseSVE > 0 &&
492 n->as_Vector()->length() != n->in(1)->bottom_type()->is_vect()->length() &&
493 n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
494 match(Set dst (VectorReinterpret src));
495 effect(TEMP tmp, KILL cr);
496 ins_cost(2 * SVE_COST);
497 format %{ "# vmask_reinterpret $dst, $src\t# vector (sve)" %}
498 ins_encode %{
499 BasicType from_bt = Matcher::vector_element_basic_type(this, $src);
500 Assembler::SIMD_RegVariant from_size = __ elemType_to_regVariant(from_bt);
501 BasicType to_bt = Matcher::vector_element_basic_type(this);
502 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
503 __ sve_cpy(as_FloatRegister($tmp$$reg), from_size, as_PRegister($src$$reg), -1, false);
504 __ sve_cmp(Assembler::EQ, as_PRegister($dst$$reg), to_size, ptrue, as_FloatRegister($tmp$$reg), -1);
505 %}
506 ins_pipe(pipe_slow);
507 %}
508 dnl
509 dnl UNARY_OP_TRUE_PREDICATE($1, $2, $3, $4 )
510 dnl UNARY_OP_TRUE_PREDICATE(insn_name, op_name, size, insn)
511 define(`UNARY_OP_TRUE_PREDICATE', `
512 instruct $1(vReg dst, vReg src) %{
513 predicate(UseSVE > 0 &&
514 !n->as_Vector()->is_predicated_vector());
515 match(Set dst ($2 src));
516 ins_cost(SVE_COST);
517 format %{ "$4 $dst, $src\t# vector (sve) ($3)" %}
518 ins_encode %{
519 __ $4(as_FloatRegister($dst$$reg), __ $3,
520 ptrue, as_FloatRegister($src$$reg));
521 %}
522 ins_pipe(pipe_slow);
523 %}')dnl
524 dnl
525
526 // vector abs
527 UNARY_OP_TRUE_PREDICATE(vabsB, AbsVB, B, sve_abs)
528 UNARY_OP_TRUE_PREDICATE(vabsS, AbsVS, H, sve_abs)
529 UNARY_OP_TRUE_PREDICATE(vabsI, AbsVI, S, sve_abs)
530 UNARY_OP_TRUE_PREDICATE(vabsL, AbsVL, D, sve_abs)
531 UNARY_OP_TRUE_PREDICATE(vabsF, AbsVF, S, sve_fabs)
532 UNARY_OP_TRUE_PREDICATE(vabsD, AbsVD, D, sve_fabs)
533
534 dnl UNARY_OP_PREDICATE($1, $2, $3, $4 )
535 dnl UNARY_OP_PREDICATE(insn_name, op_name, size, insn)
536 define(`UNARY_OP_PREDICATE', `
537 instruct $1_masked(vReg dst_src, pRegGov pg) %{
538 predicate(UseSVE > 0);
539 match(Set dst_src ($2 dst_src pg));
540 ins_cost(SVE_COST);
541 format %{ "$4 $dst_src, $pg, $dst_src\t# vector (sve) ($3)" %}
542 ins_encode %{
543 __ $4(as_FloatRegister($dst_src$$reg), __ $3,
544 as_PRegister($pg$$reg),
545 as_FloatRegister($dst_src$$reg));
546 %}
547 ins_pipe(pipe_slow);
548 %}')dnl
549 // vector abs - predicated
550 UNARY_OP_PREDICATE(vabsB, AbsVB, B, sve_abs)
551 UNARY_OP_PREDICATE(vabsS, AbsVS, H, sve_abs)
552 UNARY_OP_PREDICATE(vabsI, AbsVI, S, sve_abs)
553 UNARY_OP_PREDICATE(vabsL, AbsVL, D, sve_abs)
554 UNARY_OP_PREDICATE(vabsF, AbsVF, S, sve_fabs)
555 UNARY_OP_PREDICATE(vabsD, AbsVD, D, sve_fabs)
556
557 dnl
558 dnl BINARY_OP_UNPREDICATE($1, $2 $3, $4 $5 )
559 dnl BINARY_OP_UNPREDICATE(insn_name, op_name, size, min_vec_len, insn)
560 define(`BINARY_OP_UNPREDICATE', `
561 instruct $1(vReg dst, vReg src1, vReg src2) %{
562 predicate(UseSVE > 0);
563 match(Set dst ($2 src1 src2));
564 ins_cost(SVE_COST);
565 format %{ "$5 $dst, $src1, $src2\t # vector (sve) ($3)" %}
566 ins_encode %{
567 __ $5(as_FloatRegister($dst$$reg), __ $3,
568 as_FloatRegister($src1$$reg),
569 as_FloatRegister($src2$$reg));
570 %}
571 ins_pipe(pipe_slow);
572 %}')dnl
573 dnl
574 dnl
575 dnl BINARY_OP_PREDICATE($1, $2, $3, $4 )
576 dnl BINARY_OP_PREDICATE(insn_name, op_name, size, insn)
577 define(`BINARY_OP_PREDICATE', `
578 instruct $1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
579 predicate(UseSVE > 0);
580 match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
581 ins_cost(SVE_COST);
582 format %{ "$4 $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) ($3)" %}
583 ins_encode %{
584 __ $4(as_FloatRegister($dst_src1$$reg), __ $3,
585 as_PRegister($pg$$reg),
586 as_FloatRegister($src2$$reg));
587 %}
588 ins_pipe(pipe_slow);
589 %}')dnl
590 dnl
591 // vector add
592 BINARY_OP_UNPREDICATE(vaddB, AddVB, B, 16, sve_add)
593 BINARY_OP_UNPREDICATE(vaddS, AddVS, H, 8, sve_add)
594 BINARY_OP_UNPREDICATE(vaddI, AddVI, S, 4, sve_add)
595 BINARY_OP_UNPREDICATE(vaddL, AddVL, D, 2, sve_add)
596 BINARY_OP_UNPREDICATE(vaddF, AddVF, S, 4, sve_fadd)
597 BINARY_OP_UNPREDICATE(vaddD, AddVD, D, 2, sve_fadd)
598
599 // vector add - predicated
600 BINARY_OP_PREDICATE(vaddB, AddVB, B, sve_add)
601 BINARY_OP_PREDICATE(vaddS, AddVS, H, sve_add)
602 BINARY_OP_PREDICATE(vaddI, AddVI, S, sve_add)
603 BINARY_OP_PREDICATE(vaddL, AddVL, D, sve_add)
604 BINARY_OP_PREDICATE(vaddF, AddVF, S, sve_fadd)
605 BINARY_OP_PREDICATE(vaddD, AddVD, D, sve_fadd)
606
607 dnl
608 dnl BINARY_OP_UNSIZED($1, $2, $3 )
609 dnl BINARY_OP_UNSIZED(insn_name, op_name, insn)
610 define(`BINARY_OP_UNSIZED', `
611 instruct $1(vReg dst, vReg src1, vReg src2) %{
612 predicate(UseSVE > 0);
613 match(Set dst ($2 src1 src2));
614 ins_cost(SVE_COST);
615 format %{ "$3 $dst, $src1, $src2\t# vector (sve)" %}
616 ins_encode %{
617 __ $3(as_FloatRegister($dst$$reg),
618 as_FloatRegister($src1$$reg),
619 as_FloatRegister($src2$$reg));
620 %}
621 ins_pipe(pipe_slow);
622 %}')dnl
623 dnl
624 // vector and
625 BINARY_OP_UNSIZED(vand, AndV, sve_and)
626
627 // vector or
628 BINARY_OP_UNSIZED(vor, OrV, sve_orr)
629
630 // vector xor
631 BINARY_OP_UNSIZED(vxor, XorV, sve_eor)
632
633 dnl BINARY_LOGIC_OP_PREDICATE($1, $2, $3 )
634 dnl BINARY_LOGIC_OP_PREDICATE(insn_name, op_name, insn)
635 define(`BINARY_LOGIC_OP_PREDICATE', `
636 instruct $1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
637 predicate(UseSVE > 0);
638 match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
639 ins_cost(SVE_COST);
640 format %{ "$3 $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
641 ins_encode %{
642 BasicType bt = Matcher::vector_element_basic_type(this);
643 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
644 __ $3(as_FloatRegister($dst_src1$$reg), size,
645 as_PRegister($pg$$reg),
646 as_FloatRegister($src2$$reg));
647 %}
648 ins_pipe(pipe_slow);
649 %}')dnl
650 dnl
651 // vector and - predicated
652 BINARY_LOGIC_OP_PREDICATE(vand, AndV, sve_and)
653
654 // vector or - predicated
655 BINARY_LOGIC_OP_PREDICATE(vor, OrV, sve_orr)
656
657 // vector xor - predicated
658 BINARY_LOGIC_OP_PREDICATE(vxor, XorV, sve_eor)
659
660 // vector not
661 dnl
662 define(`MATCH_RULE', `ifelse($1, I,
663 `match(Set dst (XorV src (ReplicateB m1)));
664 match(Set dst (XorV src (ReplicateS m1)));
665 match(Set dst (XorV src (ReplicateI m1)));',
666 `match(Set dst (XorV src (ReplicateL m1)));')')dnl
667 dnl
668 define(`VECTOR_NOT', `
669 instruct vnot$1`'(vReg dst, vReg src, imm$1_M1 m1) %{
670 predicate(UseSVE > 0);
671 MATCH_RULE($1)
672 ins_cost(SVE_COST);
673 format %{ "sve_not $dst, $src\t# vector (sve) $2" %}
674 ins_encode %{
675 __ sve_not(as_FloatRegister($dst$$reg), __ D,
676 ptrue, as_FloatRegister($src$$reg));
677 %}
678 ins_pipe(pipe_slow);
679 %}')dnl
680 dnl $1,$2
681 VECTOR_NOT(I, B/H/S)
682 VECTOR_NOT(L, D)
683 undefine(MATCH_RULE)
684 dnl
685 // vector and_not
686 dnl
687 define(`MATCH_RULE', `ifelse($1, I,
688 `match(Set dst (AndV src1 (XorV src2 (ReplicateB m1))));
689 match(Set dst (AndV src1 (XorV src2 (ReplicateS m1))));
690 match(Set dst (AndV src1 (XorV src2 (ReplicateI m1))));',
691 `match(Set dst (AndV src1 (XorV src2 (ReplicateL m1))));')')dnl
692 dnl
693 define(`VECTOR_AND_NOT', `
694 instruct vand_not$1`'(vReg dst, vReg src1, vReg src2, imm$1_M1 m1) %{
695 predicate(UseSVE > 0);
696 MATCH_RULE($1)
697 ins_cost(SVE_COST);
698 format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) $2" %}
699 ins_encode %{
700 __ sve_bic(as_FloatRegister($dst$$reg),
701 as_FloatRegister($src1$$reg),
702 as_FloatRegister($src2$$reg));
703 %}
704 ins_pipe(pipe_slow);
705 %}')dnl
706 dnl $1,$2
707 VECTOR_AND_NOT(I, B/H/S)
708 VECTOR_AND_NOT(L, D)
709 undefine(MATCH_RULE)
710 dnl
711 dnl VDIVF($1, $2 , $3 )
712 dnl VDIVF(name_suffix, size, min_vec_len)
713 define(`VDIVF', `
714 instruct vdiv$1(vReg dst_src1, vReg src2) %{
715 predicate(UseSVE > 0);
716 match(Set dst_src1 (DivV$1 dst_src1 src2));
717 ins_cost(SVE_COST);
718 format %{ "sve_fdiv $dst_src1, $dst_src1, $src2\t# vector (sve) ($2)" %}
719 ins_encode %{
720 __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ $2,
721 ptrue, as_FloatRegister($src2$$reg));
722 %}
723 ins_pipe(pipe_slow);
724 %}')dnl
725 dnl
726 // vector float div
727 VDIVF(F, S, 4)
728 VDIVF(D, D, 2)
729
730 // vector float div - predicated
731 BINARY_OP_PREDICATE(vfdivF, DivVF, S, sve_fdiv)
732 BINARY_OP_PREDICATE(vfdivD, DivVD, D, sve_fdiv)
733
734 dnl
735 dnl VMINMAX($1 , $2, $3 , $4 )
736 dnl VMINMAX(op_name, op, finsn, insn)
737 define(`VMINMAX', `
738 instruct v$1(vReg dst_src1, vReg src2) %{
739 predicate(UseSVE > 0);
740 match(Set dst_src1 ($2 dst_src1 src2));
741 ins_cost(SVE_COST);
742 format %{ "sve_$1 $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
743 ins_encode %{
744 BasicType bt = Matcher::vector_element_basic_type(this);
745 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
746 if (is_floating_point_type(bt)) {
747 __ $3(as_FloatRegister($dst_src1$$reg), size,
748 ptrue, as_FloatRegister($src2$$reg));
749 } else {
750 assert(is_integral_type(bt), "unsupported type");
751 __ $4(as_FloatRegister($dst_src1$$reg), size,
752 ptrue, as_FloatRegister($src2$$reg));
753 }
754 %}
755 ins_pipe(pipe_slow);
756 %}')dnl
757 dnl
758 // vector min/max
759 VMINMAX(min, MinV, sve_fmin, sve_smin)
760 VMINMAX(max, MaxV, sve_fmax, sve_smax)
761
762 dnl
763 dnl VMINMAX_PREDICATE($1 , $2, $3 , $4 )
764 dnl VMINMAX_PREDICATE(op_name, op, finsn, insn)
765 define(`VMINMAX_PREDICATE', `
766 instruct v$1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
767 predicate(UseSVE > 0);
768 match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
769 ins_cost(SVE_COST);
770 format %{ "sve_$1 $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %}
771 ins_encode %{
772 BasicType bt = Matcher::vector_element_basic_type(this);
773 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
774 if (is_floating_point_type(bt)) {
775 __ $3(as_FloatRegister($dst_src1$$reg), size,
776 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
777 } else {
778 assert(is_integral_type(bt), "unsupported type");
779 __ $4(as_FloatRegister($dst_src1$$reg), size,
780 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
781 }
782 %}
783 ins_pipe(pipe_slow);
784 %}')dnl
785 dnl
786 // vector min/max - predicated
787 VMINMAX_PREDICATE(min, MinV, sve_fmin, sve_smin)
788 VMINMAX_PREDICATE(max, MaxV, sve_fmax, sve_smax)
789
790 dnl
791 dnl VFMLA($1 $2 $3 )
792 dnl VFMLA(name_suffix, size, min_vec_len)
793 define(`VFMLA', `
794 // dst_src1 = dst_src1 + src2 * src3
795 instruct vfmla$1(vReg dst_src1, vReg src2, vReg src3) %{
796 predicate(UseFMA && UseSVE > 0);
797 match(Set dst_src1 (FmaV$1 dst_src1 (Binary src2 src3)));
798 ins_cost(SVE_COST);
799 format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
800 ins_encode %{
801 __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ $2,
802 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
803 %}
804 ins_pipe(pipe_slow);
805 %}')dnl
806 dnl
807 // vector fmla
808 VFMLA(F, S, 4)
809 VFMLA(D, D, 2)
810
811 dnl
812 dnl VFMLA_PREDICATE($1, $2 )
813 dnl VFMLA_PREDICATE(type, size)
814 define(`VFMLA_PREDICATE', `
815 // dst_src1 = dst_src1 * src2 + src3
816 instruct vfmla$1_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
817 predicate(UseFMA && UseSVE > 0);
818 match(Set dst_src1 (FmaV$1 (Binary dst_src1 src2) (Binary src3 pg)));
819 ins_cost(SVE_COST);
820 format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) ($2)" %}
821 ins_encode %{
822 __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ $2, as_PRegister($pg$$reg),
823 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
824 %}
825 ins_pipe(pipe_slow);
826 %}')dnl
827 dnl
828 // vector fmla - predicated
829 VFMLA_PREDICATE(F, S)
830 VFMLA_PREDICATE(D, D)
831
832 dnl
833 dnl VFMLS($1 $2 $3 )
834 dnl VFMLS(name_suffix, size, min_vec_len)
835 define(`VFMLS', `
836 // dst_src1 = dst_src1 + -src2 * src3
837 // dst_src1 = dst_src1 + src2 * -src3
838 instruct vfmls$1(vReg dst_src1, vReg src2, vReg src3) %{
839 predicate(UseFMA && UseSVE > 0);
840 match(Set dst_src1 (FmaV$1 dst_src1 (Binary (NegV$1 src2) src3)));
841 match(Set dst_src1 (FmaV$1 dst_src1 (Binary src2 (NegV$1 src3))));
842 ins_cost(SVE_COST);
843 format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
844 ins_encode %{
845 __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ $2,
846 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
847 %}
848 ins_pipe(pipe_slow);
849 %}')dnl
850 dnl
851 // vector fmls
942 VMLS(B, B, 16)
943 VMLS(S, H, 8)
944 VMLS(I, S, 4)
945 VMLS(L, D, 2)
946
947 dnl
948 dnl BINARY_OP_TRUE_PREDICATE($1, $2, $3, $4, $5 )
949 dnl BINARY_OP_TRUE_PREDICATE(insn_name, op_name, size, min_vec_len, insn)
950 define(`BINARY_OP_TRUE_PREDICATE', `
951 instruct $1(vReg dst_src1, vReg src2) %{
952 predicate(UseSVE > 0);
953 match(Set dst_src1 ($2 dst_src1 src2));
954 ins_cost(SVE_COST);
955 format %{ "$5 $dst_src1, $dst_src1, $src2\t # vector (sve) ($3)" %}
956 ins_encode %{
957 __ $5(as_FloatRegister($dst_src1$$reg), __ $3,
958 ptrue, as_FloatRegister($src2$$reg));
959 %}
960 ins_pipe(pipe_slow);
961 %}')dnl
962 dnl
963 // vector mul
964 BINARY_OP_TRUE_PREDICATE(vmulB, MulVB, B, 16, sve_mul)
965 BINARY_OP_TRUE_PREDICATE(vmulS, MulVS, H, 8, sve_mul)
966 BINARY_OP_TRUE_PREDICATE(vmulI, MulVI, S, 4, sve_mul)
967 BINARY_OP_TRUE_PREDICATE(vmulL, MulVL, D, 2, sve_mul)
968 BINARY_OP_UNPREDICATE(vmulF, MulVF, S, 4, sve_fmul)
969 BINARY_OP_UNPREDICATE(vmulD, MulVD, D, 2, sve_fmul)
970
971 // vector mul - predicated
972 BINARY_OP_PREDICATE(vmulB, MulVB, B, sve_mul)
973 BINARY_OP_PREDICATE(vmulS, MulVS, H, sve_mul)
974 BINARY_OP_PREDICATE(vmulI, MulVI, S, sve_mul)
975 BINARY_OP_PREDICATE(vmulL, MulVL, D, sve_mul)
976 BINARY_OP_PREDICATE(vmulF, MulVF, S, sve_fmul)
977 BINARY_OP_PREDICATE(vmulD, MulVD, D, sve_fmul)
978
979 // vector fneg
980 UNARY_OP_TRUE_PREDICATE(vnegF, NegVF, S, sve_fneg)
981 UNARY_OP_TRUE_PREDICATE(vnegD, NegVD, D, sve_fneg)
982
983 // vector fneg - predicated
984 UNARY_OP_PREDICATE(vnegF, NegVF, S, sve_fneg)
985 UNARY_OP_PREDICATE(vnegD, NegVD, D, sve_fneg)
986
987 // popcount vector
988
989 instruct vpopcountI(vReg dst, vReg src) %{
990 predicate(UseSVE > 0);
991 match(Set dst (PopCountVI src));
992 format %{ "sve_cnt $dst, $src\t# vector (sve) (S)\n\t" %}
993 ins_encode %{
994 __ sve_cnt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg));
995 %}
996 ins_pipe(pipe_slow);
997 %}
998
999 // vector mask compare
1000
1001 instruct vmaskcmp(pRegGov dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %{
1002 predicate(UseSVE > 0);
1003 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
1004 effect(KILL cr);
1005 ins_cost(SVE_COST);
1006 format %{ "sve_cmp $dst, $src1, $src2\t# vector mask cmp (sve)" %}
1007 ins_encode %{
1008 BasicType bt = Matcher::vector_element_basic_type(this);
1009 __ sve_compare(as_PRegister($dst$$reg), bt, ptrue, as_FloatRegister($src1$$reg),
1010 as_FloatRegister($src2$$reg), (int)$cond$$constant);
1011 %}
1012 ins_pipe(pipe_slow);
1013 %}
1014
1015 instruct vmaskcmp_masked(pRegGov dst, vReg src1, vReg src2, immI cond, pRegGov pg, rFlagsReg cr) %{
1016 predicate(UseSVE > 0);
1017 match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond pg)));
1018 effect(KILL cr);
1019 ins_cost(SVE_COST);
1020 format %{ "sve_cmp $dst, $pg, $src1, $src2\t# vector mask cmp (sve)" %}
1021 ins_encode %{
1022 BasicType bt = Matcher::vector_element_basic_type(this);
1023 __ sve_compare(as_PRegister($dst$$reg), bt, as_PRegister($pg$$reg), as_FloatRegister($src1$$reg),
1024 as_FloatRegister($src2$$reg), (int)$cond$$constant);
1025 %}
1026 ins_pipe(pipe_slow);
1027 %}
1028
1029 // vector blend
1030
1031 instruct vblend(vReg dst, vReg src1, vReg src2, pRegGov pg) %{
1032 predicate(UseSVE > 0);
1033 match(Set dst (VectorBlend (Binary src1 src2) pg));
1034 ins_cost(SVE_COST);
1035 format %{ "sve_sel $dst, $pg, $src2, $src1\t# vector blend (sve)" %}
1036 ins_encode %{
1037 Assembler::SIMD_RegVariant size =
1038 __ elemType_to_regVariant(Matcher::vector_element_basic_type(this));
1039 __ sve_sel(as_FloatRegister($dst$$reg), size, as_PRegister($pg$$reg),
1040 as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
1041 %}
1042 ins_pipe(pipe_slow);
1043 %}
1044
1045 // vector load mask
1046
1047 instruct vloadmaskB(pRegGov dst, vReg src, rFlagsReg cr) %{
1048 predicate(UseSVE > 0 &&
1049 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
1050 match(Set dst (VectorLoadMask src));
1051 effect(KILL cr);
1052 ins_cost(SVE_COST);
1053 format %{ "vloadmaskB $dst, $src\t# vector load mask (sve) (B)" %}
1054 ins_encode %{
1055 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ B,
1056 ptrue, as_FloatRegister($src$$reg), 0);
1057 %}
1058 ins_pipe(pipe_slow);
1059 %}
1060
1061 instruct vloadmask_extend(pRegGov dst, vReg src, vReg tmp, rFlagsReg cr) %{
1062 predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() != T_BYTE);
1063 match(Set dst (VectorLoadMask src));
1064 effect(TEMP tmp, KILL cr);
1065 ins_cost(3 * SVE_COST);
1066 format %{ "vloadmask $dst, $src\t# vector load mask (sve) (H/S/D)" %}
1067 ins_encode %{
1068 BasicType bt = Matcher::vector_element_basic_type(this);
1069 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1070 __ sve_vector_extend(as_FloatRegister($tmp$$reg), size, as_FloatRegister($src$$reg), __ B);
1071 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
1072 %}
1073 ins_pipe(pipe_slow);
1074 %}
1075
1076 // vector store mask
1077
1078 instruct vstoremaskB(vReg dst, pRegGov src, immI_1 size) %{
1079 predicate(UseSVE > 0);
1080 match(Set dst (VectorStoreMask src size));
1081 ins_cost(SVE_COST);
1082 format %{ "vstoremask $dst, $src\t# vector store mask (sve) (B)" %}
1083 ins_encode %{
1084 __ sve_cpy(as_FloatRegister($dst$$reg), __ B, as_PRegister($src$$reg), 1, false);
1085 %}
1086 ins_pipe(pipe_slow);
1087 %}
1088
1089 instruct vstoremask_narrow(vReg dst, pRegGov src, vReg tmp, immI_gt_1 size) %{
1090 predicate(UseSVE > 0);
1091 match(Set dst (VectorStoreMask src size));
1092 effect(TEMP_DEF dst, TEMP tmp);
1093 ins_cost(3 * SVE_COST);
1094 format %{ "vstoremask $dst, $src\t# vector store mask (sve) (H/S/D)" %}
1095 ins_encode %{
1096 Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant((int)$size$$constant);
1097 __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($src$$reg), 1, false);
1098 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
1099 as_FloatRegister($dst$$reg), size, as_FloatRegister($tmp$$reg));
1100 %}
1101 ins_pipe(pipe_slow);
1102 %}
1103
1104 // Combine LoadVector+VectorLoadMask when the vector element type is not T_BYTE
1105
1106 instruct vloadmask_loadV(pRegGov dst, indirect mem, vReg tmp, rFlagsReg cr) %{
1107 predicate(UseSVE > 0 &&
1108 n->as_Vector()->length_in_bytes() == MaxVectorSize &&
1109 type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
1110 match(Set dst (VectorLoadMask (LoadVector mem)));
1111 effect(TEMP tmp, KILL cr);
1112 ins_cost(3 * SVE_COST);
1113 format %{ "sve_ld1b $tmp, $mem\n\t"
1114 "sve_cmpne $dst, $tmp, 0\t# load vector mask (sve) (H/S/D)" %}
1115 ins_encode %{
1116 // Load mask values which are boolean type, and extend them to the
1117 // expected vector element type. Convert the vector to predicate.
1118 BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
1119 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($tmp$$reg),
1120 ptrue, T_BOOLEAN, to_vect_bt, $mem->opcode(),
1121 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1122 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ elemType_to_regVariant(to_vect_bt),
1123 ptrue, as_FloatRegister($tmp$$reg), 0);
1124 %}
1125 ins_pipe(pipe_slow);
1126 %}
1127
1128 instruct vloadmask_loadV_partial(pRegGov dst, indirect mem, vReg vtmp, pRegGov ptmp, rFlagsReg cr) %{
1129 predicate(UseSVE > 0 &&
1130 n->as_Vector()->length_in_bytes() > 16 &&
1131 n->as_Vector()->length_in_bytes() < MaxVectorSize &&
1132 type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
1133 match(Set dst (VectorLoadMask (LoadVector mem)));
1134 effect(TEMP vtmp, TEMP ptmp, KILL cr);
1135 ins_cost(6 * SVE_COST);
1136 format %{ "vloadmask_loadV $dst, $mem\t# load vector mask partial (sve) (H/S/D)" %}
1137 ins_encode %{
1138 // Load valid mask values which are boolean type, and extend them to the
1139 // expected vector element type. Convert the vector to predicate.
1140 BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
1141 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(to_vect_bt);
1142 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this));
1143 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($vtmp$$reg),
1144 as_PRegister($ptmp$$reg), T_BOOLEAN, to_vect_bt, $mem->opcode(),
1145 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1146 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($vtmp$$reg), 0);
1147 %}
1148 ins_pipe(pipe_slow);
1149 %}
1150
1151 // Combine VectorStoreMask+StoreVector when the vector element type is not T_BYTE
1152
1153 instruct storeV_vstoremask(indirect mem, pRegGov src, vReg tmp, immI_gt_1 esize) %{
1154 predicate(UseSVE > 0 &&
1155 Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
1156 match(Set mem (StoreVector mem (VectorStoreMask src esize)));
1157 effect(TEMP tmp);
1158 ins_cost(3 * SVE_COST);
1159 format %{ "sve_cpy $tmp, $src, 1\n\t"
1160 "sve_st1b $tmp, $mem\t# store vector mask (sve) (H/S/D)" %}
1161 ins_encode %{
1162 BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
1163 assert(type2aelembytes(from_vect_bt) == (int)$esize$$constant, "unsupported type.");
1164 Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
1165 __ sve_cpy(as_FloatRegister($tmp$$reg), size, as_PRegister($src$$reg), 1, false);
1166 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($tmp$$reg),
1167 ptrue, T_BOOLEAN, from_vect_bt, $mem->opcode(),
1168 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1169 %}
1170 ins_pipe(pipe_slow);
1171 %}
1172
1173 instruct storeV_vstoremask_partial(indirect mem, pRegGov src, vReg vtmp,
1174 immI_gt_1 esize, pRegGov ptmp, rFlagsReg cr) %{
1175 predicate(UseSVE > 0 &&
1176 n->as_StoreVector()->memory_size() > 16 &&
1177 type2aelembytes(n->as_StoreVector()->vect_type()->element_basic_type()) > 1 &&
1178 Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
1179 match(Set mem (StoreVector mem (VectorStoreMask src esize)));
1180 effect(TEMP vtmp, TEMP ptmp, KILL cr);
1181 format %{ "storeV_vstoremask $src, $mem\t# store vector mask partial (sve) (H/S/D)" %}
1182 ins_cost(6 * SVE_COST);
1183 ins_encode %{
1184 // Convert the valid src predicate to vector, and store the vector
1185 // elements as boolean values.
1186 BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
1187 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(from_vect_bt);
1188 __ sve_cpy(as_FloatRegister($vtmp$$reg), size, as_PRegister($src$$reg), 1, false);
1189 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
1190 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($vtmp$$reg),
1191 as_PRegister($ptmp$$reg), T_BOOLEAN, from_vect_bt, $mem->opcode(),
1192 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1193 %}
1194 ins_pipe(pipe_slow);
1195 %}
1196 dnl
1197 dnl REDUCE_I($1, $2 )
1198 dnl REDUCE_I(insn_name, op_name)
1199 define(`REDUCE_I', `
1200 instruct reduce_$1I(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
1201 ifelse($2, AddReductionVI,
1202 `predicate(UseSVE > 0 &&
1203 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);',
1204 `predicate(UseSVE > 0 &&
1205 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1206 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);')
1207 match(Set dst ($2 src1 src2));
1208 effect(TEMP_DEF dst, TEMP tmp);
1209 ins_cost(SVE_COST);
1210 format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction (sve) (may extend)" %}
1211 ins_encode %{
1212 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1213 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1214 $src1$$Register, as_FloatRegister($src2$$reg),
1215 ptrue, as_FloatRegister($tmp$$reg));
1216 %}
1217 ins_pipe(pipe_slow);
1218 %}')dnl
1219 dnl
1220 dnl
1221 dnl REDUCE_L($1, $2 )
1222 dnl REDUCE_L(insn_name, op_name)
1223 define(`REDUCE_L', `
1224 instruct reduce_$1L(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
1225 ifelse($2, AddReductionVL,
1226 `predicate(UseSVE > 0 &&
1227 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);',
1228 `predicate(UseSVE > 0 &&
1229 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1230 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);')
1231 match(Set dst ($2 src1 src2));
1232 effect(TEMP_DEF dst, TEMP tmp);
1233 ins_cost(SVE_COST);
1234 format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction (sve)" %}
1235 ins_encode %{
1236 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1237 $src1$$Register, as_FloatRegister($src2$$reg),
1238 ptrue, as_FloatRegister($tmp$$reg));
1239 %}
1240 ins_pipe(pipe_slow);
1241 %}')dnl
1242 dnl
1243 dnl REDUCE_I_PARTIAL($1, $2 )
1244 dnl REDUCE_I_PARTIAL(insn_name, op_name)
1245 define(`REDUCE_I_PARTIAL', `
1246 instruct reduce_$1I_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1247 pRegGov ptmp, rFlagsReg cr) %{
1248 ifelse($2, AddReductionVI,
1249 `predicate(UseSVE > 0 &&
1250 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);',
1251 `predicate(UseSVE > 0 &&
1252 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1253 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);')
1254 match(Set dst ($2 src1 src2));
1255 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1256 ins_cost(2 * SVE_COST);
1257 format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction partial (sve) (may extend)" %}
1258 ins_encode %{
1259 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1260 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1261 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1262 Matcher::vector_length(this, $src2));
1263 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1264 $src1$$Register, as_FloatRegister($src2$$reg),
1265 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1266 %}
1267 ins_pipe(pipe_slow);
1268 %}')dnl
1269 dnl
1270 dnl REDUCE_L_PARTIAL($1, $2 )
1271 dnl REDUCE_L_PARTIAL(insn_name, op_name)
1272 define(`REDUCE_L_PARTIAL', `
1273 instruct reduce_$1L_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1274 pRegGov ptmp, rFlagsReg cr) %{
1275 ifelse($2, AddReductionVL,
1276 `predicate(UseSVE > 0 &&
1277 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);',
1278 `predicate(UseSVE > 0 &&
1279 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1280 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);')
1281 match(Set dst ($2 src1 src2));
1282 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1283 ins_cost(2 * SVE_COST);
1284 format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction partial (sve)" %}
1285 ins_encode %{
1286 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1287 Matcher::vector_length(this, $src2));
1288 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1289 $src1$$Register, as_FloatRegister($src2$$reg),
1290 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1291 %}
1292 ins_pipe(pipe_slow);
1293 %}')dnl
1294 dnl
1295 dnl REDUCE_ADDF($1, $2, $3, $4 )
1296 dnl REDUCE_ADDF(insn_name, op_name, reg_dst, size)
1297 define(`REDUCE_ADDF', `
1298 instruct reduce_$1($3 src1_dst, vReg src2) %{
1299 predicate(UseSVE > 0 &&
1300 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1301 match(Set src1_dst ($2 src1_dst src2));
1302 ins_cost(SVE_COST);
1303 format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) ($4)" %}
1304 ins_encode %{
1305 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1306 ptrue, as_FloatRegister($src2$$reg));
1307 %}
1308 ins_pipe(pipe_slow);
1309 %}')dnl
1310 dnl
1311 dnl
1312 dnl REDUCE_ADDF_PARTIAL($1, $2, $3, $4 )
1313 dnl REDUCE_ADDF_PARTIAL(insn_name, suffix, reg_dst, size)
1314 define(`REDUCE_ADDF_PARTIAL', `
1315 instruct reduce_$1_partial($3 src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
1316 predicate(UseSVE > 0 &&
1317 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1318 match(Set src1_dst ($2 src1_dst src2));
1319 ins_cost(SVE_COST);
1320 effect(TEMP ptmp, KILL cr);
1321 format %{ "sve_reduce_$1 $src1_dst, $src1_dst, $src2\t# $1 reduction partial (sve) ($4)" %}
1322 ins_encode %{
1323 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
1324 Matcher::vector_length(this, $src2));
1325 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1326 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1327 %}
1328 ins_pipe(pipe_slow);
1329 %}')dnl
1330 dnl
1331 dnl
1332 dnl REDUCE_I_PREDICATE($1, $2 )
1333 dnl REDUCE_I_PREDICATE(insn_name, op_name)
1334 define(`REDUCE_I_PREDICATE', `
1335 instruct reduce_$1I_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
1336 ifelse($2, AddReductionVI,
1337 `predicate(UseSVE > 0 &&
1338 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);',
1339 `predicate(UseSVE > 0 &&
1340 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1341 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);')
1342 match(Set dst ($2 (Binary src1 src2) pg));
1343 effect(TEMP_DEF dst, TEMP tmp);
1344 ins_cost(SVE_COST);
1345 format %{ "sve_reduce_$1I $dst, $src1, $pg, $src2\t# $1I reduction predicated (sve) (may extend)" %}
1346 ins_encode %{
1347 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1348 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1349 $src1$$Register, as_FloatRegister($src2$$reg),
1350 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
1351 %}
1352 ins_pipe(pipe_slow);
1353 %}')dnl
1354 dnl
1355 dnl REDUCE_L_PREDICATE($1, $2 )
1356 dnl REDUCE_L_PREDICATE(insn_name, op_name)
1357 define(`REDUCE_L_PREDICATE', `
1358 instruct reduce_$1L_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
1359 ifelse($2, AddReductionVL,
1360 `predicate(UseSVE > 0 &&
1361 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);',
1362 `predicate(UseSVE > 0 &&
1363 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1364 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);')
1365 match(Set dst ($2 (Binary src1 src2) pg));
1366 effect(TEMP_DEF dst, TEMP tmp);
1367 ins_cost(SVE_COST);
1368 format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated (sve)" %}
1369 ins_encode %{
1370 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1371 $src1$$Register, as_FloatRegister($src2$$reg),
1372 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
1373 %}
1374 ins_pipe(pipe_slow);
1375 %}')dnl
1376 dnl
1377 dnl REDUCE_I_PREDICATE_PARTIAL($1, $2 )
1378 dnl REDUCE_I_PREDICATE_PARTIAL(insn_name, op_name)
1379 define(`REDUCE_I_PREDICATE_PARTIAL', `
1380 instruct reduce_$1I_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1381 pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
1382 ifelse($2, AddReductionVI,
1383 `predicate(UseSVE > 0 &&
1384 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);',
1385 `predicate(UseSVE > 0 &&
1386 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1387 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);')
1388 match(Set dst ($2 (Binary src1 src2) pg));
1389 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1390 ins_cost(3 * SVE_COST);
1391 format %{ "sve_reduce_$1I $dst, $src1, $pg, $src2\t# $1I reduction predicated partial (sve) (may extend)" %}
1392 ins_encode %{
1393 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1394 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1395 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1396 Matcher::vector_length(this, $src2));
1397 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1398 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1399 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1400 $src1$$Register, as_FloatRegister($src2$$reg),
1401 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1402 %}
1403 ins_pipe(pipe_slow);
1404 %}')dnl
1405 dnl
1406 dnl REDUCE_L_PREDICATE_PARTIAL($1, $2 )
1407 dnl REDUCE_L_PREDICATE_PARTIAL(insn_name, op_name)
1408 define(`REDUCE_L_PREDICATE_PARTIAL', `
1409 instruct reduce_$1L_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1410 pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
1411 ifelse($2, AddReductionVL,
1412 `predicate(UseSVE > 0 &&
1413 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);',
1414 `predicate(UseSVE > 0 &&
1415 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1416 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);')
1417 match(Set dst ($2 (Binary src1 src2) pg));
1418 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1419 ins_cost(3 * SVE_COST);
1420 format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated partial (sve)" %}
1421 ins_encode %{
1422 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1423 Matcher::vector_length(this, $src2));
1424 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1425 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1426 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1427 $src1$$Register, as_FloatRegister($src2$$reg),
1428 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1429 %}
1430 ins_pipe(pipe_slow);
1431 %}')dnl
1432 dnl
1433 dnl REDUCE_ADDF_PREDICATE($1, $2, $3, $4 )
1434 dnl REDUCE_ADDF_PREDICATE(insn_name, op_name, reg_dst, size)
1435 define(`REDUCE_ADDF_PREDICATE', `
1436 instruct reduce_$1_masked($3 src1_dst, vReg src2, pRegGov pg) %{
1437 predicate(UseSVE > 0 &&
1438 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1439 match(Set src1_dst ($2 (Binary src1_dst src2) pg));
1440 ins_cost(SVE_COST);
1441 format %{ "sve_reduce_$1 $src1_dst, $pg, $src2\t# $1 reduction predicated (sve)" %}
1442 ins_encode %{
1443 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1444 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1445 %}
1446 ins_pipe(pipe_slow);
1447 %}')dnl
1448 dnl
1449 dnl REDUCE_ADDF_PREDICATE_PARTIAL($1, $2, $3, $4 )
1450 dnl REDUCE_ADDF_PREDICATE_PARTIAL(insn_name, op_name, reg_dst, size)
1451 define(`REDUCE_ADDF_PREDICATE_PARTIAL', `
1452 instruct reduce_$1_masked_partial($3 src1_dst, vReg src2, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
1453 predicate(UseSVE > 0 &&
1454 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1455 match(Set src1_dst ($2 (Binary src1_dst src2) pg));
1456 effect(TEMP ptmp, KILL cr);
1457 ins_cost(SVE_COST);
1458 format %{ "sve_reduce_$1 $src1_dst, $pg, $src2\t# $1 reduction predicated partial (sve)" %}
1459 ins_encode %{
1460 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
1461 Matcher::vector_length(this, $src2));
1462 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1463 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1464 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1465 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1466 %}
1467 ins_pipe(pipe_slow);
1468 %}')dnl
1469 dnl
1470
1471 // vector add reduction
1472 REDUCE_I(add, AddReductionVI)
1473 REDUCE_L(add, AddReductionVL)
1474 REDUCE_ADDF(addF, AddReductionVF, vRegF, S)
1475 REDUCE_ADDF(addD, AddReductionVD, vRegD, D)
1476 REDUCE_I_PARTIAL(add, AddReductionVI)
1477 REDUCE_L_PARTIAL(add, AddReductionVL)
1478 REDUCE_ADDF_PARTIAL(addF, AddReductionVF, vRegF, S)
1479 REDUCE_ADDF_PARTIAL(addD, AddReductionVD, vRegD, D)
1480
1481 // vector add reduction - predicated
1482 REDUCE_I_PREDICATE(add, AddReductionVI)
1483 REDUCE_L_PREDICATE(add, AddReductionVL)
1484 REDUCE_ADDF_PREDICATE(addF, AddReductionVF, vRegF, S)
1485 REDUCE_ADDF_PREDICATE(addD, AddReductionVD, vRegD, D)
1486 REDUCE_I_PREDICATE_PARTIAL(add, AddReductionVI)
1487 REDUCE_L_PREDICATE_PARTIAL(add, AddReductionVL)
1488 REDUCE_ADDF_PREDICATE_PARTIAL(addF, AddReductionVF, vRegF, S)
1489 REDUCE_ADDF_PREDICATE_PARTIAL(addD, AddReductionVD, vRegD, D)
1490
1491 // vector and reduction
1492 REDUCE_I(and, AndReductionV)
1493 REDUCE_L(and, AndReductionV)
1494 REDUCE_I_PARTIAL(and, AndReductionV)
1495 REDUCE_L_PARTIAL(and, AndReductionV)
1496
1497 // vector and reduction - predicated
1498 REDUCE_I_PREDICATE(and, AndReductionV)
1499 REDUCE_L_PREDICATE(and, AndReductionV)
1500 REDUCE_I_PREDICATE_PARTIAL(and, AndReductionV)
1501 REDUCE_L_PREDICATE_PARTIAL(and, AndReductionV)
1502
1503 // vector or reduction
1504 REDUCE_I(or, OrReductionV)
1505 REDUCE_L(or, OrReductionV)
1506 REDUCE_I_PARTIAL(or, OrReductionV)
1507 REDUCE_L_PARTIAL(or, OrReductionV)
1508
1509 // vector or reduction - predicated
1510 REDUCE_I_PREDICATE(or, OrReductionV)
1511 REDUCE_L_PREDICATE(or, OrReductionV)
1512 REDUCE_I_PREDICATE_PARTIAL(or, OrReductionV)
1513 REDUCE_L_PREDICATE_PARTIAL(or, OrReductionV)
1514
1515 // vector xor reduction
1516 REDUCE_I(eor, XorReductionV)
1517 REDUCE_L(eor, XorReductionV)
1518 REDUCE_I_PARTIAL(eor, XorReductionV)
1519 REDUCE_L_PARTIAL(eor, XorReductionV)
1520
1521 // vector xor reduction - predicated
1522 REDUCE_I_PREDICATE(eor, XorReductionV)
1523 REDUCE_L_PREDICATE(eor, XorReductionV)
1524 REDUCE_I_PREDICATE_PARTIAL(eor, XorReductionV)
1525 REDUCE_L_PREDICATE_PARTIAL(eor, XorReductionV)
1526
1527 dnl
1528 dnl REDUCE_MAXMIN_I($1, $2 )
1529 dnl REDUCE_MAXMIN_I(insn_name, op_name)
1530 define(`REDUCE_MAXMIN_I', `
1531 instruct reduce_$1I(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
1532 predicate(UseSVE > 0 &&
1533 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1534 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1535 is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
1536 match(Set dst ($2 src1 src2));
1537 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1538 ins_cost(SVE_COST);
1539 format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction (sve)" %}
1540 ins_encode %{
1541 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1542 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1543 $src1$$Register, as_FloatRegister($src2$$reg),
1544 ptrue, as_FloatRegister($tmp$$reg));
1545 %}
1546 ins_pipe(pipe_slow);
1547 %}')dnl
1548 dnl
1549 dnl REDUCE_MAXMIN_L($1, $2 )
1550 dnl REDUCE_MAXMIN_L(insn_name, op_name)
1551 define(`REDUCE_MAXMIN_L', `
1552 instruct reduce_$1L(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
1553 predicate(UseSVE > 0 &&
1554 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1555 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1556 match(Set dst ($2 src1 src2));
1557 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1558 ins_cost(SVE_COST);
1559 format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction (sve)" %}
1560 ins_encode %{
1561 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1562 $src1$$Register, as_FloatRegister($src2$$reg),
1563 ptrue, as_FloatRegister($tmp$$reg));
1564 %}
1565 ins_pipe(pipe_slow);
1566 %}')dnl
1567 dnl
1568 dnl REDUCE_MAXMIN_I_PARTIAL($1 , $2 )
1569 dnl REDUCE_MAXMIN_I_PARTIAL(min_max, op_name)
1570 define(`REDUCE_MAXMIN_I_PARTIAL', `
1571 instruct reduce_$1I_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1572 pRegGov ptmp, rFlagsReg cr) %{
1573 predicate(UseSVE > 0 &&
1574 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1575 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1576 is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
1577 match(Set dst ($2 src1 src2));
1578 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1579 ins_cost(2 * SVE_COST);
1580 format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction partial (sve)" %}
1581 ins_encode %{
1582 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1583 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1584 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1585 Matcher::vector_length(this, $src2));
1586 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1587 $src1$$Register, as_FloatRegister($src2$$reg),
1588 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1589 %}
1590 ins_pipe(pipe_slow);
1591 %}')dnl
1592 dnl
1593 dnl REDUCE_MAXMIN_L_PARTIAL($1 , $2 )
1594 dnl REDUCE_MAXMIN_L_PARTIAL(min_max, op_name)
1595 define(`REDUCE_MAXMIN_L_PARTIAL', `
1596 instruct reduce_$1L_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1597 pRegGov ptmp, rFlagsReg cr) %{
1598 predicate(UseSVE > 0 &&
1599 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1600 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1601 match(Set dst ($2 src1 src2));
1602 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1603 ins_cost(2 * SVE_COST);
1604 format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction partial (sve)" %}
1605 ins_encode %{
1606 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1607 Matcher::vector_length(this, $src2));
1608 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1609 $src1$$Register, as_FloatRegister($src2$$reg),
1610 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1611 %}
1612 ins_pipe(pipe_slow);
1613 %}')dnl
1614 dnl
1615 dnl REDUCE_MAXMIN_I_PREDICATE($1 , $2 )
1616 dnl REDUCE_MAXMIN_I_PREDICATE(min_max, op_name)
1617 define(`REDUCE_MAXMIN_I_PREDICATE', `
1618 instruct reduce_$1I_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp,
1619 pRegGov pg, rFlagsReg cr) %{
1620 predicate(UseSVE > 0 &&
1621 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1622 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1623 is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
1624 match(Set dst ($2 (Binary src1 src2) pg));
1625 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1626 ins_cost(SVE_COST);
1627 format %{ "sve_reduce_$1I $dst, $src1, $pg, $src2\t# $1I reduction predicated (sve)" %}
1628 ins_encode %{
1629 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1630 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1631 $src1$$Register, as_FloatRegister($src2$$reg),
1632 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
1633 %}
1634 ins_pipe(pipe_slow);
1635 %}')dnl
1636 dnl
1637 dnl REDUCE_MAXMIN_L_PREDICATE($1 , $2 )
1638 dnl REDUCE_MAXMIN_L_PREDICATE(min_max, op_name)
1639 define(`REDUCE_MAXMIN_L_PREDICATE', `
1640 instruct reduce_$1L_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp,
1641 pRegGov pg, rFlagsReg cr) %{
1642 predicate(UseSVE > 0 &&
1643 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1644 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1645 match(Set dst ($2 (Binary src1 src2) pg));
1646 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1647 ins_cost(SVE_COST);
1648 format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated (sve)" %}
1649 ins_encode %{
1650 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1651 $src1$$Register, as_FloatRegister($src2$$reg),
1652 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
1653 %}
1654 ins_pipe(pipe_slow);
1655 %}')dnl
1656 dnl
1657 dnl REDUCE_MAXMIN_I_PREDICATE_PARTIAL($1 , $2 )
1658 dnl REDUCE_MAXMIN_I_PREDICATE_PARTIAL(min_max, op_name)
1659 define(`REDUCE_MAXMIN_I_PREDICATE_PARTIAL', `
1660 instruct reduce_$1I_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1661 pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
1662 predicate(UseSVE > 0 &&
1663 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1664 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1665 is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
1666 match(Set dst ($2 (Binary src1 src2) pg));
1667 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1668 ins_cost(3 * SVE_COST);
1669 format %{ "sve_reduce_$1I $dst, $src1, $pg, $src2\t# $1I reduction predicated partial (sve)" %}
1670 ins_encode %{
1671 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1672 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1673 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1674 Matcher::vector_length(this, $src2));
1675 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1676 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1677 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1678 $src1$$Register, as_FloatRegister($src2$$reg),
1679 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1680 %}
1681 ins_pipe(pipe_slow);
1682 %}')dnl
1683 dnl
1684 dnl REDUCE_MAXMIN_L_PREDICATE_PARTIAL($1 , $2 )
1685 dnl REDUCE_MAXMIN_L_PREDICATE_PARTIAL(min_max, op_name)
1686 define(`REDUCE_MAXMIN_L_PREDICATE_PARTIAL', `
1687 instruct reduce_$1L_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1688 pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
1689 predicate(UseSVE > 0 &&
1690 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1691 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1692 match(Set dst ($2 (Binary src1 src2) pg));
1693 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1694 ins_cost(3 * SVE_COST);
1695 format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated partial (sve)" %}
1696 ins_encode %{
1697 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1698 Matcher::vector_length(this, $src2));
1699 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1700 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1701 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1702 $src1$$Register, as_FloatRegister($src2$$reg),
1703 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1704 %}
1705 ins_pipe(pipe_slow);
1706 %}')dnl
1707 dnl
1708 dnl REDUCE_FMINMAX($1, $2, $3, $4, $5 )
1709 dnl REDUCE_FMINMAX(min_max, name_suffix, element_type, size, reg_src_dst)
1710 define(`REDUCE_FMINMAX', `
1711 instruct reduce_$1$2($5 dst, $5 src1, vReg src2) %{
1712 predicate(UseSVE > 0 &&
1713 n->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1714 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1715 match(Set dst (translit($1, `m', `M')ReductionV src1 src2));
1716 ins_cost(INSN_COST);
1717 effect(TEMP_DEF dst);
1718 format %{ "sve_reduce_$1$2 $dst, $src1, $src2\t# $1$2 reduction (sve)" %}
1719 ins_encode %{
1720 __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src2$$reg));
1721 __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1722 %}
1723 ins_pipe(pipe_slow);
1724 %}')dnl
1725 dnl
1726 dnl REDUCE_FMINMAX_PARTIAL($1, $2, $3, $4, $5 )
1727 dnl REDUCE_FMINMAX_PARTIAL(min_max, name_suffix, element_type, size, reg_src_dst)
1728 define(`REDUCE_FMINMAX_PARTIAL', `
1729 instruct reduce_$1$2_partial($5 dst, $5 src1, vReg src2,
1730 pRegGov ptmp, rFlagsReg cr) %{
1731 predicate(UseSVE > 0 &&
1732 n->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1733 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1734 match(Set dst (translit($1, `m', `M')ReductionV src1 src2));
1735 ins_cost(INSN_COST);
1736 effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
1737 format %{ "sve_reduce_$1$2 $dst, $src1, $src2\t# $1$2 reduction partial (sve)" %}
1738 ins_encode %{
1739 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
1740 Matcher::vector_length(this, $src2));
1741 __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1742 __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1743 %}
1744 ins_pipe(pipe_slow);
1745 %}')dnl
1746 dnl
1747 dnl REDUCE_FMINMAX_PREDICATE($1, $2, $3, $4, $5 )
1748 dnl REDUCE_FMINMAX_PREDICATE(min_max, name_suffix, element_type, size, reg_src_dst)
1749 define(`REDUCE_FMINMAX_PREDICATE', `
1750 instruct reduce_$1$2_masked($5 dst, $5 src1, vReg src2, pRegGov pg) %{
1751 predicate(UseSVE > 0 &&
1752 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1753 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1754 match(Set dst (translit($1, `m', `M')ReductionV (Binary src1 src2) pg));
1755 ins_cost(SVE_COST);
1756 format %{ "sve_reduce_$1$2 $dst, $src1, $pg, $src2\t# $1$2 reduction predicated (sve)" %}
1757 ins_encode %{
1758 __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1759 __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1760 %}
1761 ins_pipe(pipe_slow);
1762 %}')dnl
1763 dnl
1764 dnl REDUCE_FMINMAX_PREDICATE_PARTIAL($1, $2, $3, $4, $5 )
1765 dnl REDUCE_FMINMAX_PREDICATE_PARTIAL(min_max, name_suffix, element_type, size, reg_src_dst)
1766 define(`REDUCE_FMINMAX_PREDICATE_PARTIAL', `
1767 instruct reduce_$1$2_masked_partial($5 dst, $5 src1, vReg src2, pRegGov pg,
1768 pRegGov ptmp, rFlagsReg cr) %{
1769 predicate(UseSVE > 0 &&
1770 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1771 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1772 match(Set dst (translit($1, `m', `M')ReductionV (Binary src1 src2) pg));
1773 effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
1774 ins_cost(3 * SVE_COST);
1775 format %{ "sve_reduce_$1$2 $dst, $src1, $pg, $src2\t# $1$2 reduction predicated partial (sve)" %}
1776 ins_encode %{
1777 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
1778 Matcher::vector_length(this, $src2));
1779 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1780 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1781 __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4,
1782 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1783 __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1784 %}
1785 ins_pipe(pipe_slow);
1786 %}')dnl
1787 // vector max reduction
1788 REDUCE_MAXMIN_I(max, MaxReductionV)
1789 REDUCE_MAXMIN_L(max, MaxReductionV)
1790 REDUCE_MAXMIN_I_PARTIAL(max, MaxReductionV)
1791 REDUCE_MAXMIN_L_PARTIAL(max, MaxReductionV)
1792 REDUCE_FMINMAX(max, F, T_FLOAT, S, vRegF)
1793 REDUCE_FMINMAX_PARTIAL(max, F, T_FLOAT, S, vRegF)
1794 REDUCE_FMINMAX(max, D, T_DOUBLE, D, vRegD)
1795 REDUCE_FMINMAX_PARTIAL(max, D, T_DOUBLE, D, vRegD)
1796
1797 // vector max reduction - predicated
1798 REDUCE_MAXMIN_I_PREDICATE(max, MaxReductionV)
1799 REDUCE_MAXMIN_L_PREDICATE(max, MaxReductionV)
1800 REDUCE_MAXMIN_I_PREDICATE_PARTIAL(max, MaxReductionV)
1801 REDUCE_MAXMIN_L_PREDICATE_PARTIAL(max, MaxReductionV)
1802 REDUCE_FMINMAX_PREDICATE(max, F, T_FLOAT, S, vRegF)
1803 REDUCE_FMINMAX_PREDICATE(max, D, T_DOUBLE, D, vRegD)
1804 REDUCE_FMINMAX_PREDICATE_PARTIAL(max, F, T_FLOAT, S, vRegF)
1805 REDUCE_FMINMAX_PREDICATE_PARTIAL(max, D, T_DOUBLE, D, vRegD)
1806
1807 // vector min reduction
1808 REDUCE_MAXMIN_I(min, MinReductionV)
1809 REDUCE_MAXMIN_L(min, MinReductionV)
1810 REDUCE_MAXMIN_I_PARTIAL(min, MinReductionV)
1811 REDUCE_MAXMIN_L_PARTIAL(min, MinReductionV)
1812 REDUCE_FMINMAX(min, F, T_FLOAT, S, vRegF)
1813 REDUCE_FMINMAX_PARTIAL(min, F, T_FLOAT, S, vRegF)
1814 REDUCE_FMINMAX(min, D, T_DOUBLE, D, vRegD)
1815 REDUCE_FMINMAX_PARTIAL(min, D, T_DOUBLE, D, vRegD)
1816
1817 // vector min reduction - predicated
1818 REDUCE_MAXMIN_I_PREDICATE(min, MinReductionV)
1819 REDUCE_MAXMIN_L_PREDICATE(min, MinReductionV)
1820 REDUCE_MAXMIN_I_PREDICATE_PARTIAL(min, MinReductionV)
1821 REDUCE_MAXMIN_L_PREDICATE_PARTIAL(min, MinReductionV)
1822 REDUCE_FMINMAX_PREDICATE(min, F, T_FLOAT, S, vRegF)
1823 REDUCE_FMINMAX_PREDICATE(min, D, T_DOUBLE, D, vRegD)
1824 REDUCE_FMINMAX_PREDICATE_PARTIAL(min, F, T_FLOAT, S, vRegF)
1825 REDUCE_FMINMAX_PREDICATE_PARTIAL(min, D, T_DOUBLE, D, vRegD)
1826
1827 // vector Math.rint, floor, ceil
1828
1829 instruct vroundD(vReg dst, vReg src, immI rmode) %{
1830 predicate(UseSVE > 0 &&
1831 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
1832 match(Set dst (RoundDoubleModeV src rmode));
1833 format %{ "sve_frint $dst, $src, $rmode\t# vector (sve) (D)" %}
1834 ins_encode %{
1835 switch ($rmode$$constant) {
1836 case RoundDoubleModeNode::rmode_rint:
1837 __ sve_frintn(as_FloatRegister($dst$$reg), __ D,
1838 ptrue, as_FloatRegister($src$$reg));
1839 break;
1840 case RoundDoubleModeNode::rmode_floor:
1841 __ sve_frintm(as_FloatRegister($dst$$reg), __ D,
1842 ptrue, as_FloatRegister($src$$reg));
1843 break;
1844 case RoundDoubleModeNode::rmode_ceil:
1845 __ sve_frintp(as_FloatRegister($dst$$reg), __ D,
1846 ptrue, as_FloatRegister($src$$reg));
1984 VSHIFT_TRUE_PREDICATE(vlsrS, URShiftVS, H, 8, sve_lsr)
1985 VSHIFT_TRUE_PREDICATE(vlsrI, URShiftVI, S, 4, sve_lsr)
1986 VSHIFT_TRUE_PREDICATE(vlsrL, URShiftVL, D, 2, sve_lsr)
1987 VSHIFT_IMM_UNPREDICATED(vasrB_imm, RShiftVB, RShiftCntV, B, 16, sve_asr)
1988 VSHIFT_IMM_UNPREDICATED(vasrS_imm, RShiftVS, RShiftCntV, H, 8, sve_asr)
1989 VSHIFT_IMM_UNPREDICATED(vasrI_imm, RShiftVI, RShiftCntV, S, 4, sve_asr)
1990 VSHIFT_IMM_UNPREDICATED(vasrL_imm, RShiftVL, RShiftCntV, D, 2, sve_asr)
1991 VSHIFT_IMM_UNPREDICATED(vlsrB_imm, URShiftVB, RShiftCntV, B, 16, sve_lsr)
1992 VSHIFT_IMM_UNPREDICATED(vlsrS_imm, URShiftVS, RShiftCntV, H, 8, sve_lsr)
1993 VSHIFT_IMM_UNPREDICATED(vlsrI_imm, URShiftVI, RShiftCntV, S, 4, sve_lsr)
1994 VSHIFT_IMM_UNPREDICATED(vlsrL_imm, URShiftVL, RShiftCntV, D, 2, sve_lsr)
1995 VSHIFT_IMM_UNPREDICATED(vlslB_imm, LShiftVB, LShiftCntV, B, 16, sve_lsl)
1996 VSHIFT_IMM_UNPREDICATED(vlslS_imm, LShiftVS, LShiftCntV, H, 8, sve_lsl)
1997 VSHIFT_IMM_UNPREDICATED(vlslI_imm, LShiftVI, LShiftCntV, S, 4, sve_lsl)
1998 VSHIFT_IMM_UNPREDICATED(vlslL_imm, LShiftVL, LShiftCntV, D, 2, sve_lsl)
1999 VSHIFT_COUNT(vshiftcntB, B, 16, T_BYTE)
2000 VSHIFT_COUNT(vshiftcntS, H, 8, T_SHORT)
2001 VSHIFT_COUNT(vshiftcntI, S, 4, T_INT)
2002 VSHIFT_COUNT(vshiftcntL, D, 2, T_LONG)
2003
2004 // vector shift - predicated
2005 BINARY_OP_PREDICATE(vasrB, RShiftVB, B, sve_asr)
2006 BINARY_OP_PREDICATE(vasrS, RShiftVS, H, sve_asr)
2007 BINARY_OP_PREDICATE(vasrI, RShiftVI, S, sve_asr)
2008 BINARY_OP_PREDICATE(vasrL, RShiftVL, D, sve_asr)
2009 BINARY_OP_PREDICATE(vlslB, LShiftVB, B, sve_lsl)
2010 BINARY_OP_PREDICATE(vlslS, LShiftVS, H, sve_lsl)
2011 BINARY_OP_PREDICATE(vlslI, LShiftVI, S, sve_lsl)
2012 BINARY_OP_PREDICATE(vlslL, LShiftVL, D, sve_lsl)
2013 BINARY_OP_PREDICATE(vlsrB, URShiftVB, B, sve_lsr)
2014 BINARY_OP_PREDICATE(vlsrS, URShiftVS, H, sve_lsr)
2015 BINARY_OP_PREDICATE(vlsrI, URShiftVI, S, sve_lsr)
2016 BINARY_OP_PREDICATE(vlsrL, URShiftVL, D, sve_lsr)
2017 dnl
2018 dnl VSHIFT_IMM_PREDICATED($1, $2, $3, $4, $5, $6 )
2019 dnl VSHIFT_IMM_PREDICATED(insn_name, op_name, op_name2, type, size, insn)
2020 define(`VSHIFT_IMM_PREDICATED', `
2021 instruct $1_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
2022 predicate(UseSVE > 0);
2023 match(Set dst_src ($2 (Binary dst_src ($3 shift)) pg));
2024 ins_cost(SVE_COST);
2025 format %{ "$6 $dst_src, $pg, $dst_src, $shift\t# vector (sve) ($4)" %}
2026 ins_encode %{
2027 int con = (int)$shift$$constant;
2028 assert(con ifelse(index(`$1', `vlsl'), 0, `>=', `>') 0 && con < $5, "invalid shift immediate");
2029 __ $6(as_FloatRegister($dst_src$$reg), __ $4, as_PRegister($pg$$reg), con);
2030 %}
2031 ins_pipe(pipe_slow);
2032 %}')dnl
2033 dnl
2034 VSHIFT_IMM_PREDICATED(vasrB, RShiftVB, RShiftCntV, B, 8, sve_asr)
2035 VSHIFT_IMM_PREDICATED(vasrS, RShiftVS, RShiftCntV, H, 16, sve_asr)
2036 VSHIFT_IMM_PREDICATED(vasrI, RShiftVI, RShiftCntV, S, 32, sve_asr)
2037 VSHIFT_IMM_PREDICATED(vasrL, RShiftVL, RShiftCntV, D, 64, sve_asr)
2038 VSHIFT_IMM_PREDICATED(vlsrB, URShiftVB, RShiftCntV, B, 8, sve_lsr)
2039 VSHIFT_IMM_PREDICATED(vlsrS, URShiftVS, RShiftCntV, H, 16, sve_lsr)
2040 VSHIFT_IMM_PREDICATED(vlsrI, URShiftVI, RShiftCntV, S, 32, sve_lsr)
2041 VSHIFT_IMM_PREDICATED(vlsrL, URShiftVL, RShiftCntV, D, 64, sve_lsr)
2042 VSHIFT_IMM_PREDICATED(vlslB, LShiftVB, LShiftCntV, B, 8, sve_lsl)
2043 VSHIFT_IMM_PREDICATED(vlslS, LShiftVS, LShiftCntV, H, 16, sve_lsl)
2044 VSHIFT_IMM_PREDICATED(vlslI, LShiftVI, LShiftCntV, S, 32, sve_lsl)
2045 VSHIFT_IMM_PREDICATED(vlslL, LShiftVL, LShiftCntV, D, 64, sve_lsl)
2046
2047 // vector sqrt
2048 UNARY_OP_TRUE_PREDICATE(vsqrtF, SqrtVF, S, sve_fsqrt)
2049 UNARY_OP_TRUE_PREDICATE(vsqrtD, SqrtVD, D, sve_fsqrt)
2050
2051 // vector sqrt - predicated
2052 UNARY_OP_PREDICATE(vsqrtF, SqrtVF, S, sve_fsqrt)
2053 UNARY_OP_PREDICATE(vsqrtD, SqrtVD, D, sve_fsqrt)
2054
2055 // vector sub
2056 BINARY_OP_UNPREDICATE(vsubB, SubVB, B, 16, sve_sub)
2057 BINARY_OP_UNPREDICATE(vsubS, SubVS, H, 8, sve_sub)
2058 BINARY_OP_UNPREDICATE(vsubI, SubVI, S, 4, sve_sub)
2059 BINARY_OP_UNPREDICATE(vsubL, SubVL, D, 2, sve_sub)
2060 BINARY_OP_UNPREDICATE(vsubF, SubVF, S, 4, sve_fsub)
2061 BINARY_OP_UNPREDICATE(vsubD, SubVD, D, 2, sve_fsub)
2062
2063 // vector sub - predicated
2064 BINARY_OP_PREDICATE(vsubB, SubVB, B, sve_sub)
2065 BINARY_OP_PREDICATE(vsubS, SubVS, H, sve_sub)
2066 BINARY_OP_PREDICATE(vsubI, SubVI, S, sve_sub)
2067 BINARY_OP_PREDICATE(vsubL, SubVL, D, sve_sub)
2068 BINARY_OP_PREDICATE(vsubF, SubVF, S, sve_fsub)
2069 BINARY_OP_PREDICATE(vsubD, SubVD, D, sve_fsub)
2070
2071 // ------------------------------ Vector mask cast --------------------------
2072
2073 instruct vmaskcast(pRegGov dst_src) %{
2074 predicate(UseSVE > 0 &&
2075 n->bottom_type()->is_vect()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
2076 n->bottom_type()->is_vect()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
2077 match(Set dst_src (VectorMaskCast dst_src));
2078 ins_cost(0);
2079 format %{ "vmaskcast $dst_src\t# empty (sve)" %}
2080 ins_encode %{
2081 // empty
2082 %}
2083 ins_pipe(pipe_class_empty);
2084 %}
2085
2086 instruct vmaskcast_extend(pRegGov dst, pReg src)
2087 %{
2088 predicate(UseSVE > 0 &&
2089 (Matcher::vector_length_in_bytes(n) == 2 * Matcher::vector_length_in_bytes(n->in(1)) ||
2090 Matcher::vector_length_in_bytes(n) == 4 * Matcher::vector_length_in_bytes(n->in(1)) ||
2091 Matcher::vector_length_in_bytes(n) == 8 * Matcher::vector_length_in_bytes(n->in(1))));
2092 match(Set dst (VectorMaskCast src));
2093 ins_cost(SVE_COST * 3);
2094 format %{ "sve_vmaskcast_extend $dst, $src\t# extend predicate $src" %}
2095 ins_encode %{
2096 __ sve_vmaskcast_extend(as_PRegister($dst$$reg), as_PRegister($src$$reg),
2097 Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
2098 %}
2099 ins_pipe(pipe_slow);
2100 %}
2101
2102 instruct vmaskcast_narrow(pRegGov dst, pReg src)
2103 %{
2104 predicate(UseSVE > 0 &&
2105 (Matcher::vector_length_in_bytes(n) * 2 == Matcher::vector_length_in_bytes(n->in(1)) ||
2106 Matcher::vector_length_in_bytes(n) * 4 == Matcher::vector_length_in_bytes(n->in(1)) ||
2107 Matcher::vector_length_in_bytes(n) * 8 == Matcher::vector_length_in_bytes(n->in(1))));
2108 match(Set dst (VectorMaskCast src));
2109 ins_cost(SVE_COST * 3);
2110 format %{ "sve_vmaskcast_narrow $dst, $src\t# narrow predicate $src" %}
2111 ins_encode %{
2112 __ sve_vmaskcast_narrow(as_PRegister($dst$$reg), as_PRegister($src$$reg),
2113 Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
2114 %}
2115 ins_pipe(pipe_slow);
2116 %}
2117 dnl
2118
2119 // ------------------------------ Vector cast -------------------------------
2120 dnl
2121 dnl
2122 define(`VECTOR_CAST_X2X', `
2123 instruct vcvt$1to$2`'(vReg dst, vReg src)
2124 %{
2125 predicate(UseSVE > 0 &&
2126 n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
2127 match(Set dst (VectorCast$1`'2X src));
2128 ins_cost(SVE_COST);
2129 format %{ "sve_vectorcast_$5 $dst, $src\t# convert $1 to $2 vector" %}
2130 ins_encode %{
2131 __ sve_$3(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src$$reg), __ $4);
2132 %}
2133 ins_pipe(pipe_slow);
2134 %}')dnl
2135
2136 dnl
2137 dnl Start of vector cast rules
2138 dnl
2139 instruct vcvtBtoX_extend(vReg dst, vReg src)
2140 %{
2141 predicate(UseSVE > 0);
2142 match(Set dst (VectorCastB2X src));
2143 ins_cost(2 * SVE_COST);
2144 format %{ "sve_vectorcast_b2x $dst, $src\t# convert B to X vector (extend)" %}
2145 ins_encode %{
2146 BasicType to_bt = Matcher::vector_element_basic_type(this);
2147 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2148 __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ B);
2149 if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
2150 __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
2151 }
2152 %}
2153 ins_pipe(pipe_slow);
2154 %}
2155
2156 instruct vcvtStoB(vReg dst, vReg src, vReg tmp)
2157 %{
2158 predicate(UseSVE > 0 &&
2159 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2160 match(Set dst (VectorCastS2X src));
2161 effect(TEMP tmp);
2162 ins_cost(2 * SVE_COST);
2163 format %{ "sve_vectorcast_s2b $dst, $src\t# convert H to B vector" %}
2164 ins_encode %{
2165 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
2166 as_FloatRegister($src$$reg), __ H, as_FloatRegister($tmp$$reg));
2167 %}
2168 ins_pipe(pipe_slow);
2169 %}
2170
2171 instruct vcvtStoX_extend(vReg dst, vReg src)
2172 %{
2173 predicate(UseSVE > 0 &&
2174 type2aelembytes(Matcher::vector_element_basic_type(n)) > 2);
2175 match(Set dst (VectorCastS2X src));
2176 ins_cost(2 * SVE_COST);
2177 format %{ "sve_vectorcast_s2x $dst, $src\t# convert H to X vector (extend)" %}
2178 ins_encode %{
2179 BasicType to_bt = Matcher::vector_element_basic_type(this);
2180 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2181 __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ H);
2182 if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
2183 __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
2184 }
2185 %}
2186 ins_pipe(pipe_slow);
2187 %}
2188
2189 instruct vcvtItoB(vReg dst, vReg src, vReg tmp)
2190 %{
2191 predicate(UseSVE > 0 &&
2192 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2193 match(Set dst (VectorCastI2X src));
2194 effect(TEMP_DEF dst, TEMP tmp);
2195 ins_cost(3 * SVE_COST);
2196 format %{ "sve_vectorcast_i2b $dst, $src\t# convert I to B vector" %}
2197 ins_encode %{
2198 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
2199 as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
2200 %}
2201 ins_pipe(pipe_slow);
2202 %}
2203
2204 instruct vcvtItoS(vReg dst, vReg src, vReg tmp)
2205 %{
2206 predicate(UseSVE > 0 &&
2207 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
2208 match(Set dst (VectorCastI2X src));
2209 effect(TEMP tmp);
2210 ins_cost(2 * SVE_COST);
2211 format %{ "sve_vectorcast_i2s $dst, $src\t# convert I to H vector" %}
2212 ins_encode %{
2213 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ H,
2214 as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
2215 %}
2216 ins_pipe(pipe_slow);
2217 %}
2218
2219 instruct vcvtItoL(vReg dst, vReg src)
2220 %{
2221 predicate(UseSVE > 0 &&
2222 n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2223 match(Set dst (VectorCastI2X src));
2224 ins_cost(SVE_COST);
2225 format %{ "sve_vectorcast_i2l $dst, $src\t# convert I to L vector" %}
2226 ins_encode %{
2227 __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
2228 %}
2229 ins_pipe(pipe_slow);
2230 %}
2231 dnl
2232 dnl vcvtItoF
2233 VECTOR_CAST_X2X(I, F, scvtf, S, i2f)
2234
2235 instruct vcvtItoD(vReg dst, vReg src)
2236 %{
2237 predicate(UseSVE > 0 &&
2238 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
2239 match(Set dst (VectorCastI2X src));
2240 ins_cost(2 * SVE_COST);
2241 format %{ "sve_vectorcast_i2d $dst, $src\t# convert I to D vector" %}
2242 ins_encode %{
2243 __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
2244 __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ D);
2245 %}
2246 ins_pipe(pipe_slow);
2247 %}
2248
2249 instruct vcvtLtoX_narrow(vReg dst, vReg src, vReg tmp)
2250 %{
2251 predicate(UseSVE > 0 && is_integral_type(Matcher::vector_element_basic_type(n)));
2252 match(Set dst (VectorCastL2X src));
2253 effect(TEMP_DEF dst, TEMP tmp);
2254 ins_cost(2 * SVE_COST);
2255 format %{ "sve_vectorcast_l2x $dst, $src\t# convert L to B/H/S vector (narrow)" %}
2256 ins_encode %{
2257 BasicType to_bt = Matcher::vector_element_basic_type(this);
2258 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2259 __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
2260 as_FloatRegister($src$$reg), __ D, as_FloatRegister($tmp$$reg));
2261 %}
2262 ins_pipe(pipe_slow);
2263 %}
2264
2265 instruct vcvtLtoF(vReg dst, vReg src, vReg tmp)
2266 %{
2267 predicate(UseSVE > 0 &&
2268 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2269 match(Set dst (VectorCastL2X src));
2270 effect(TEMP_DEF dst, TEMP tmp);
2271 ins_cost(3 * SVE_COST);
2272 format %{ "sve_vectorcast_l2f $dst, $src\t# convert L to F vector" %}
2273 ins_encode %{
2274 __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
2275 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
2276 as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
2277
2278 %}
2279 ins_pipe(pipe_slow);
2280 %}
2281 dnl
2282 dnl vcvtLtoD
2283 VECTOR_CAST_X2X(L, D, scvtf, D, l2d)
2284
2285 instruct vcvtFtoX_narrow(vReg dst, vReg src, vReg tmp)
2286 %{
2287 predicate(UseSVE > 0 &&
2288 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2289 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT));
2290 match(Set dst (VectorCastF2X src));
2291 effect(TEMP_DEF dst, TEMP tmp);
2292 ins_cost(3 * SVE_COST);
2293 format %{ "sve_vectorcast_f2x $dst, $src\t# convert F to B/H vector" %}
2294 ins_encode %{
2295 BasicType to_bt = Matcher::vector_element_basic_type(this);
2296 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2297 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
2298 __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
2299 as_FloatRegister($dst$$reg), __ S, as_FloatRegister($tmp$$reg));
2300 %}
2301 ins_pipe(pipe_slow);
2302 %}
2303
2304 instruct vcvtFtoX_extend(vReg dst, vReg src)
2305 %{
2306 predicate(UseSVE > 0 &&
2307 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2308 n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
2309 match(Set dst (VectorCastF2X src));
2310 ins_cost(SVE_COST);
2311 format %{ "sve_vectorcast_f2x $dst, $src\t# convert F to I/L vector" %}
2312 ins_encode %{
2313 BasicType to_bt = Matcher::vector_element_basic_type(this);
2314 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
2315 if (to_bt == T_LONG) {
2316 __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($dst$$reg), __ S);
2317 }
2318 %}
2319 ins_pipe(pipe_slow);
2320 %}
2321
2322 instruct vcvtFtoD(vReg dst, vReg src)
2323 %{
2324 predicate(UseSVE > 0 &&
2325 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
2326 match(Set dst (VectorCastF2X src));
2327 ins_cost(2 * SVE_COST);
2328 format %{ "sve_vectorcast_f2d $dst, $dst\t# convert F to D vector" %}
2329 ins_encode %{
2330 __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
2331 __ sve_fcvt(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);
2332 %}
2333 ins_pipe(pipe_slow);
2334 %}
2335
2336 instruct vcvtDtoX_narrow(vReg dst, vReg src, vReg tmp)
2337 %{
2338 predicate(UseSVE > 0 &&
2339 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2340 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2341 n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2342 match(Set dst (VectorCastD2X src));
2343 effect(TEMP_DEF dst, TEMP tmp);
2344 ins_cost(3 * SVE_COST);
2345 format %{ "sve_vectorcast_d2x $dst, $src\t# convert D to X vector (narrow)" %}
2346 ins_encode %{
2347 BasicType to_bt = Matcher::vector_element_basic_type(this);
2348 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2349 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
2350 __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
2351 as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
2352 %}
2353 ins_pipe(pipe_slow);
2354 %}
2355 dnl
2356 dnl vcvtDtoL
2357 VECTOR_CAST_X2X(D, L, fcvtzs, D, d2l)
2358
2359 instruct vcvtDtoF(vReg dst, vReg src, vReg tmp)
2360 %{
2361 predicate(UseSVE > 0 &&
2362 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2363 match(Set dst (VectorCastD2X src));
2364 effect(TEMP_DEF dst, TEMP tmp);
2365 ins_cost(3 * SVE_COST);
2366 format %{ "sve_vectorcast_d2f $dst, S, $dst\t# convert D to F vector" %}
2367 ins_encode %{
2368 __ sve_fcvt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
2369 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
2370 as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
2371 %}
2372 ins_pipe(pipe_slow);
2373 %}
2374
2375 dnl
2376 dnl
2377 // ------------------------------ Vector extract ---------------------------------
2378 define(`VECTOR_EXTRACT_SXT', `
2379 instruct extract$1`'($2 dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
2380 %{
2381 predicate(UseSVE > 0);
2382 match(Set dst (Extract$1 src idx));
2383 effect(TEMP pgtmp, KILL cr);
2384 ins_cost(2 * SVE_COST);
2385 format %{ "sve_extract $dst, $3, $pgtmp, $src, $idx\n\t"
2386 "sbfmw $dst, $dst, 0U, $5\t# extract from vector($1)" %}
2387 ins_encode %{
2388 __ sve_extract(as_$4($dst$$reg), __ $3, as_PRegister($pgtmp$$reg),
2389 as_FloatRegister($src$$reg), (int)($idx$$constant));
2390 __ sbfmw(as_$4($dst$$reg), as_$4($dst$$reg), 0U, $5);
2391 %}
2392 ins_pipe(pipe_slow);
2393 %}')dnl
2394 dnl $1 $2 $3 $4 $5
2395 VECTOR_EXTRACT_SXT(B, iRegINoSp, B, Register, 7U)
2396 VECTOR_EXTRACT_SXT(S, iRegINoSp, H, Register, 15U)
2397
2398 dnl
2399 define(`VECTOR_EXTRACT', `
2400 instruct extract$1`'($2 dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
2401 %{
2402 predicate(UseSVE > 0);
2403 match(Set dst (Extract$1 src idx));
2404 effect(TEMP pgtmp, KILL cr);
2405 ins_cost(2 * SVE_COST);
2406 format %{ "sve_extract $dst, $3, $pgtmp, $src, $idx\t# extract from vector($1)" %}
2407 ins_encode %{
2408 __ sve_extract(as_$4($dst$$reg), __ $3, as_PRegister($pgtmp$$reg),
2409 as_FloatRegister($src$$reg), (int)($idx$$constant));
2410 %}
2411 ins_pipe(pipe_slow);
2412 %}')dnl
2413 dnl $1 $2 $3 $4
2414 VECTOR_EXTRACT(I, iRegINoSp, S, Register)
2415 VECTOR_EXTRACT(L, iRegLNoSp, D, Register)
2416 VECTOR_EXTRACT(F, vRegF, S, FloatRegister)
2417 VECTOR_EXTRACT(D, vRegD, D, FloatRegister)
2418
2419 // ------------------------------- VectorTest ----------------------------------
2420
2421 instruct vtest_alltrue(iRegINoSp dst, pRegGov src1, pRegGov src2, pReg ptmp, rFlagsReg cr)
2422 %{
2423 predicate(UseSVE > 0 &&
2424 n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2425 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
2426 match(Set dst (VectorTest src1 src2));
2427 effect(TEMP ptmp, KILL cr);
2428 ins_cost(SVE_COST);
2429 format %{ "sve_eors $ptmp, $src1, $src2\t# $src2 is all true mask\n"
2430 "csetw $dst, EQ\t# VectorTest (sve) - alltrue" %}
2431 ins_encode %{
2432 __ sve_eors(as_PRegister($ptmp$$reg), ptrue,
2433 as_PRegister($src1$$reg), as_PRegister($src2$$reg));
2434 __ csetw(as_Register($dst$$reg), Assembler::EQ);
2435 %}
2436 ins_pipe(pipe_slow);
2437 %}
2438
2439 instruct vtest_anytrue(iRegINoSp dst, pRegGov src1, pRegGov src2, rFlagsReg cr)
2440 %{
2441 predicate(UseSVE > 0 &&
2442 n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2443 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
2444 match(Set dst (VectorTest src1 src2));
2445 effect(KILL cr);
2446 ins_cost(SVE_COST);
2447 format %{ "sve_ptest $src1\n\t"
2448 "csetw $dst, NE\t# VectorTest (sve) - anytrue" %}
2449 ins_encode %{
2450 // "src2" is not used for sve.
2451 __ sve_ptest(ptrue, as_PRegister($src1$$reg));
2452 __ csetw(as_Register($dst$$reg), Assembler::NE);
2453 %}
2454 ins_pipe(pipe_slow);
2455 %}
2456 dnl
2457 dnl
2458 dnl VTEST_PARTIAL($1, $2, $3, $4 )
2459 dnl VTEST_PARTIAL(op_name, pred, inst, cond)
2460 define(`VTEST_PARTIAL', `
2461 instruct vtest_$1_partial`'(iRegINoSp dst, pRegGov src1, pRegGov src2, pRegGov ptmp, rFlagsReg cr)
2462 %{
2463 predicate(UseSVE > 0 &&
2464 n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2465 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::$2);
2466 match(Set dst (VectorTest src1 src2));
2467 effect(TEMP ptmp, KILL cr);
2468 ins_cost(SVE_COST);
2469 format %{ "vtest_$1_partial $dst, $src1, $src2\t# VectorTest partial (sve) - $1" %}
2470 ins_encode %{
2471 BasicType bt = Matcher::vector_element_basic_type(this, $src1);
2472 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2473 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size,
2474 Matcher::vector_length(this, $src1));
2475 __ $3(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2476 as_PRegister($src1$$reg), as_PRegister($src2$$reg));
2477 __ csetw(as_Register($dst$$reg), Assembler::$4);
2478 %}
2479 ins_pipe(pipe_slow);
2480 %}')dnl
2481 dnl
2482 VTEST_PARTIAL(alltrue, overflow, sve_eors, EQ)
2483 VTEST_PARTIAL(anytrue, ne, sve_ands, NE)
2484
2485 // ------------------------------ Vector insert ---------------------------------
2486
2487 instruct insertI_small(vReg dst, vReg src, iRegIorL2I val, immI idx, pRegGov pgtmp, rFlagsReg cr)
2488 %{
2489 predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
2490 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2491 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2492 n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2493 match(Set dst (VectorInsert (Binary src val) idx));
2494 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
2495 ins_cost(4 * SVE_COST);
2496 format %{ "sve_index $dst, -16, 1\t# (B/H/S)\n\t"
2497 "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2498 "sve_orr $dst, $src, $src\n\t"
2499 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
2500 ins_encode %{
2501 BasicType bt = Matcher::vector_element_basic_type(this, $src);
2502 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2503 __ sve_index(as_FloatRegister($dst$$reg), size, -16, 1);
2504 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
2505 as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2506 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2507 __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
2508 %}
2509 ins_pipe(pipe_slow);
2510 %}
2511
2512 instruct insertF_small(vReg dst, vReg src, vRegF val, immI idx, pRegGov pgtmp, rFlagsReg cr)
2513 %{
2514 predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
2515 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2516 match(Set dst (VectorInsert (Binary src val) idx));
2517 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
2518 ins_cost(4 * SVE_COST);
2519 format %{ "sve_index $dst, S, -16, 1\n\t"
2520 "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2521 "sve_orr $dst, $src, $src\n\t"
2522 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
2523 ins_encode %{
2524 __ sve_index(as_FloatRegister($dst$$reg), __ S, -16, 1);
2525 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
2526 as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2527 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2528 __ sve_cpy(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
2529 %}
2530 ins_pipe(pipe_slow);
2531 %}
2532
2533 instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr)
2534 %{
2535 predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
2536 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2537 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2538 n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2539 match(Set dst (VectorInsert (Binary src val) idx));
2540 effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
2541 ins_cost(5 * SVE_COST);
2542 format %{ "sve_index $tmp1, 0, 1\t# (B/H/S)\n\t"
2543 "sve_dup $dst, $idx\t# (B/H/S)\n\t"
2544 "sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
2545 "sve_orr $dst, $src, $src\n\t"
2546 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
2547 ins_encode %{
2548 BasicType bt = Matcher::vector_element_basic_type(this, $src);
2549 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2550 __ sve_index(as_FloatRegister($tmp1$$reg), size, 0, 1);
2551 __ sve_dup(as_FloatRegister($dst$$reg), size, (int)($idx$$constant));
2552 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
2553 as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
2554 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2555 __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
2556 %}
2557 ins_pipe(pipe_slow);
2558 %}
2559 dnl
2560 dnl
2561 define(`VECTOR_INSERT_D', `
2562 instruct insert$1`'(vReg dst, vReg src, $2 val, immI idx, pRegGov pgtmp, rFlagsReg cr)
2563 %{
2564 predicate(UseSVE > 0 &&
2565 n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($1));
2566 match(Set dst (VectorInsert (Binary src val) idx));
2567 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
2568 ins_cost(4 * SVE_COST);
2569 format %{ "sve_index $dst, $3, -16, 1\n\t"
2570 "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2571 "sve_orr $dst, $src, $src\n\t"
2572 "sve_cpy $dst, $pgtmp, $val\t# insert into vector ($1)" %}
2573 ins_encode %{
2574 __ sve_index(as_FloatRegister($dst$$reg), __ $3, -16, 1);
2575 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ $3, ptrue,
2576 as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2577 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2578 __ sve_cpy(as_FloatRegister($dst$$reg), __ $3, as_PRegister($pgtmp$$reg), as_$4($val$$reg));
2579 %}
2580 ins_pipe(pipe_slow);
2581 %}')dnl
2582 dnl $1 $2 $3 $4
2583 VECTOR_INSERT_D(L, iRegL, D, Register)
2584 VECTOR_INSERT_D(D, vRegD, D, FloatRegister)
2585
2586 instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr)
2587 %{
2588 predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
2589 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2590 match(Set dst (VectorInsert (Binary src val) idx));
2591 effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
2592 ins_cost(5 * SVE_COST);
2593 format %{ "sve_index $tmp1, S, 0, 1\n\t"
2594 "sve_dup $dst, S, $idx\n\t"
2595 "sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
2596 "sve_orr $dst, $src, $src\n\t"
2597 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
2598 ins_encode %{
2599 __ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
2600 __ sve_dup(as_FloatRegister($dst$$reg), __ S, (int)($idx$$constant));
2601 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
2602 as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
2603 __ sve_orr(as_FloatRegister($dst$$reg),
2604 as_FloatRegister($src$$reg),
2605 as_FloatRegister($src$$reg));
2606 __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
2607 as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
2608 %}
2609 ins_pipe(pipe_slow);
2610 %}
2611
2612 // ------------------------------ Vector shuffle -------------------------------
2613
2614 instruct loadshuffle(vReg dst, vReg src) %{
2615 predicate(UseSVE > 0);
2616 match(Set dst (VectorLoadShuffle src));
2617 ins_cost(SVE_COST);
2618 format %{ "sve_loadshuffle $dst, $src\t# vector load shuffle (B/H/S/D)" %}
2619 ins_encode %{
2620 BasicType bt = Matcher::vector_element_basic_type(this);
2621 if (bt == T_BYTE) {
2622 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
2623 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
2624 as_FloatRegister($src$$reg));
2625 }
2626 } else {
2627 __ sve_vector_extend(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
2628 as_FloatRegister($src$$reg), __ B);
2629 }
2630 %}
2631 ins_pipe(pipe_slow);
2632 %}
2633
2634 // ------------------------------ Vector rearrange -------------------------------
2635
2636 instruct rearrange(vReg dst, vReg src, vReg shuffle)
2637 %{
2638 predicate(UseSVE > 0);
2639 match(Set dst (VectorRearrange src shuffle));
2640 ins_cost(SVE_COST);
2641 format %{ "sve_tbl $dst, $src, $shuffle\t# vector rearrange" %}
2642 ins_encode %{
2643 BasicType bt = Matcher::vector_element_basic_type(this, $src);
2644 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2645 __ sve_tbl(as_FloatRegister($dst$$reg), size,
2646 as_FloatRegister($src$$reg), as_FloatRegister($shuffle$$reg));
2647 %}
2648 ins_pipe(pipe_slow);
2649 %}
2650
2651 // ------------------------------ Vector Load Gather ---------------------------------
2652
2653 instruct gatherI(vReg dst, indirect mem, vReg idx) %{
2654 predicate(UseSVE > 0 &&
2655 n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
2656 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2657 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2658 match(Set dst (LoadVectorGather mem idx));
2659 ins_cost(SVE_COST);
2660 format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (S)" %}
2661 ins_encode %{
2662 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), ptrue,
2663 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2664 %}
2665 ins_pipe(pipe_slow);
2666 %}
2667
2668 instruct gatherL(vReg dst, indirect mem, vReg idx) %{
2669 predicate(UseSVE > 0 &&
2670 n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
2671 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2672 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2673 match(Set dst (LoadVectorGather mem idx));
2674 ins_cost(2 * SVE_COST);
2675 format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (D)" %}
2676 ins_encode %{
2677 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2678 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), ptrue, as_Register($mem$$base),
2679 as_FloatRegister($idx$$reg));
2680 %}
2681 ins_pipe(pipe_slow);
2682 %}
2683
2684 // ------------------------------ Vector Load Gather Partial-------------------------------
2685
2686 instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2687 predicate(UseSVE > 0 &&
2688 n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
2689 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2690 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2691 match(Set dst (LoadVectorGather mem idx));
2692 effect(TEMP ptmp, KILL cr);
2693 ins_cost(2 * SVE_COST + INSN_COST);
2694 format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (S)" %}
2695 ins_encode %{
2696 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this));
2697 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
2698 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2699 %}
2700 ins_pipe(pipe_slow);
2701 %}
2702
2703 instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2704 predicate(UseSVE > 0 &&
2705 n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
2706 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2707 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2708 match(Set dst (LoadVectorGather mem idx));
2709 effect(TEMP ptmp, KILL cr);
2710 ins_cost(3 * SVE_COST + INSN_COST);
2711 format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (D)" %}
2712 ins_encode %{
2713 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2714 Matcher::vector_length(this));
2715 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2716 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
2717 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2718 %}
2719 ins_pipe(pipe_slow);
2720 %}
2721
2722 // ------------------------------ Vector Load Gather Predicated -------------------------------
2723
2724 instruct gatherI_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
2725 predicate(UseSVE > 0 &&
2726 n->as_LoadVector()->memory_size() == MaxVectorSize &&
2727 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2728 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2729 match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
2730 ins_cost(SVE_COST);
2731 format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (S)" %}
2732 ins_encode %{
2733 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
2734 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2735 %}
2736 ins_pipe(pipe_slow);
2737 %}
2738
2739 instruct gatherL_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
2740 predicate(UseSVE > 0 &&
2741 n->as_LoadVector()->memory_size() == MaxVectorSize &&
2742 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2743 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2744 match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
2745 ins_cost(2 * SVE_COST);
2746 format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (D)" %}
2747 ins_encode %{
2748 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2749 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
2750 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2751 %}
2752 ins_pipe(pipe_slow);
2753 %}
2754
2755 // ------------------------------ Vector Load Gather Predicated Partial -------------------------------
2756
2757 instruct gatherI_masked_partial(vReg dst, indirect mem, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2758 predicate(UseSVE > 0 &&
2759 n->as_LoadVector()->memory_size() < MaxVectorSize &&
2760 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2761 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2762 match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
2763 effect(TEMP ptmp, KILL cr);
2764 ins_cost(3 * SVE_COST);
2765 format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated partial (S)" %}
2766 ins_encode %{
2767 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2768 Matcher::vector_length(this));
2769 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2770 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2771 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
2772 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2773 %}
2774 ins_pipe(pipe_slow);
2775 %}
2776
2777 instruct gatherL_masked_partial(vReg dst, indirect mem, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2778 predicate(UseSVE > 0 &&
2779 n->as_LoadVector()->memory_size() < MaxVectorSize &&
2780 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2781 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2782 match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
2783 effect(TEMP ptmp, KILL cr);
2784 ins_cost(4 * SVE_COST);
2785 format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated partial (D)" %}
2786 ins_encode %{
2787 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this));
2788 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2789 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2790 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2791 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
2792 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2793 %}
2794 ins_pipe(pipe_slow);
2795 %}
2796
2797 // ------------------------------ Vector Store Scatter -------------------------------
2798
2799 instruct scatterI(indirect mem, vReg src, vReg idx) %{
2800 predicate(UseSVE > 0 &&
2801 n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
2802 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2803 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2804 match(Set mem (StoreVectorScatter mem (Binary src idx)));
2805 ins_cost(SVE_COST);
2806 format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (S)" %}
2807 ins_encode %{
2808 __ sve_st1w_scatter(as_FloatRegister($src$$reg), ptrue,
2809 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2810 %}
2811 ins_pipe(pipe_slow);
2812 %}
2813
2814 instruct scatterL(indirect mem, vReg src, vReg idx) %{
2815 predicate(UseSVE > 0 &&
2816 n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
2817 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2818 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2819 match(Set mem (StoreVectorScatter mem (Binary src idx)));
2820 ins_cost(2 * SVE_COST);
2821 format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (D)" %}
2822 ins_encode %{
2823 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2824 __ sve_st1d_scatter(as_FloatRegister($src$$reg), ptrue,
2825 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2826 %}
2827 ins_pipe(pipe_slow);
2828 %}
2829
2830 // ------------------------------ Vector Store Scatter Partial -------------------------------
2831
2832 instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2833 predicate(UseSVE > 0 &&
2834 n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
2835 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2836 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2837 match(Set mem (StoreVectorScatter mem (Binary src idx)));
2838 effect(TEMP ptmp, KILL cr);
2839 ins_cost(2 * SVE_COST + INSN_COST);
2840 format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (S)" %}
2841 ins_encode %{
2842 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2843 Matcher::vector_length(this, $src));
2844 __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
2845 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2846 %}
2847 ins_pipe(pipe_slow);
2848 %}
2849
2850 instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2851 predicate(UseSVE > 0 &&
2852 n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
2853 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2854 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2855 match(Set mem (StoreVectorScatter mem (Binary src idx)));
2856 effect(TEMP ptmp, KILL cr);
2857 ins_cost(3 * SVE_COST + INSN_COST);
2858 format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (D)" %}
2859 ins_encode %{
2860 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2861 Matcher::vector_length(this, $src));
2862 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2863 __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
2864 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2865 %}
2866 ins_pipe(pipe_slow);
2867 %}
2868
2869 // ------------------------------ Vector Store Scatter Predicated -------------------------------
2870
2871 instruct scatterI_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
2872 predicate(UseSVE > 0 &&
2873 n->as_StoreVector()->memory_size() == MaxVectorSize &&
2874 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2875 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2876 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
2877 ins_cost(SVE_COST);
2878 format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicate (S)" %}
2879 ins_encode %{
2880 __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
2881 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2882 %}
2883 ins_pipe(pipe_slow);
2884 %}
2885
2886 instruct scatterL_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
2887 predicate(UseSVE > 0 &&
2888 n->as_StoreVector()->memory_size() == MaxVectorSize &&
2889 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2890 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2891 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
2892 ins_cost(2 * SVE_COST);
2893 format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated (D)" %}
2894 ins_encode %{
2895 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2896 __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
2897 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2898 %}
2899 ins_pipe(pipe_slow);
2900 %}
2901
2902 // ------------------------------ Vector Store Scatter Predicated Partial -------------------------------
2903
2904 instruct scatterI_masked_partial(indirect mem, vReg src, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2905 predicate(UseSVE > 0 &&
2906 n->as_StoreVector()->memory_size() < MaxVectorSize &&
2907 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2908 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2909 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
2910 effect(TEMP ptmp, KILL cr);
2911 ins_cost(3 * SVE_COST);
2912 format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated partial (S)" %}
2913 ins_encode %{
2914 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2915 Matcher::vector_length(this, $src));
2916 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2917 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2918 __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
2919 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2920 %}
2921 ins_pipe(pipe_slow);
2922 %}
2923
2924 instruct scatterL_masked_partial(indirect mem, vReg src, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2925 predicate(UseSVE > 0 &&
2926 n->as_StoreVector()->memory_size() < MaxVectorSize &&
2927 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2928 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2929 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
2930 effect(TEMP ptmp, KILL cr);
2931 ins_cost(4 * SVE_COST);
2932 format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated partial (D)" %}
2933 ins_encode %{
2934 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2935 Matcher::vector_length(this, $src));
2936 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2937 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2938 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2939 __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
2940 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2941 %}
2942 ins_pipe(pipe_slow);
2943 %}
2944
2945 // ------------------------------ Vector Load Const -------------------------------
2946
2947 instruct loadconB(vReg dst, immI0 src) %{
2948 predicate(UseSVE > 0 &&
2949 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2950 match(Set dst (VectorLoadConst src));
2951 ins_cost(SVE_COST);
2952 format %{ "sve_index $dst, 0, 1\t# generate iota indices" %}
2953 ins_encode %{
2954 __ sve_index(as_FloatRegister($dst$$reg), __ B, 0, 1);
2955 %}
2956 ins_pipe(pipe_slow);
2957 %}
2958
2959 // Intrisics for String.indexOf(char)
2960
2961 dnl
2962 define(`STRING_INDEXOF_CHAR', `
2963 instruct string$1_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
2964 iRegI_R0 result, vReg ztmp1, vReg ztmp2,
2965 pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
2966 %{
2967 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
2968 predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::$1));
2969 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
2970
2971 format %{ "String$2 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
2972
2973 ins_encode %{
2974 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
2975 as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
2976 as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), $3 /* isL */);
2977 %}
2978 ins_pipe(pipe_class_memory);
2979 %}')dnl
2980 dnl $1 $2 $3
2981 STRING_INDEXOF_CHAR(L, Latin1, true)
2982 STRING_INDEXOF_CHAR(U, UTF16, false)
2983
2984 // ---------------------------- Vector mask reductions ---------------------------
2985 instruct vmask_truecount(iRegINoSp dst, pReg src) %{
2986 predicate(UseSVE > 0 &&
2987 n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
2988 match(Set dst (VectorMaskTrueCount src));
2989 ins_cost(SVE_COST);
2990 format %{ "vmask_truecount $dst, $src\t# vector mask truecount (sve)" %}
2991 ins_encode %{
2992 BasicType bt = Matcher::vector_element_basic_type(this, $src);
2993 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2994 __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($src$$reg));
2995 %}
2996 ins_pipe(pipe_slow);
2997 %}
2998
2999 instruct vmask_firsttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
3000 predicate(UseSVE > 0 &&
3001 n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3002 match(Set dst (VectorMaskFirstTrue src));
3003 effect(TEMP ptmp);
3004 ins_cost(2 * SVE_COST);
3005 format %{ "vmask_firsttrue $dst, $src\t# vector mask firsttrue (sve)" %}
3006 ins_encode %{
3007 BasicType bt = Matcher::vector_element_basic_type(this, $src);
3008 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3009 __ sve_brkb(as_PRegister($ptmp$$reg), ptrue, as_PRegister($src$$reg), false);
3010 __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($ptmp$$reg));
3011 %}
3012 ins_pipe(pipe_slow);
3013 %}
3014
3015 instruct vmask_lasttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
3016 predicate(UseSVE > 0 &&
3017 n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3018 match(Set dst (VectorMaskLastTrue src));
3019 effect(TEMP ptmp);
3020 ins_cost(3 * SVE_COST);
3021 format %{ "vmask_lasttrue $dst, $src\t# vector mask lasttrue (sve)" %}
3022 ins_encode %{
3023 BasicType bt = Matcher::vector_element_basic_type(this, $src);
3024 __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($src$$reg), as_PRegister($ptmp$$reg));
3025 %}
3026 ins_pipe(pipe_slow);
3027 %}
3028
3029 instruct vmask_truecount_partial(iRegINoSp dst, pReg src, pRegGov pgtmp, rFlagsReg cr) %{
3030 predicate(UseSVE > 0 &&
3031 n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3032 match(Set dst (VectorMaskTrueCount src));
3033 effect(TEMP pgtmp, KILL cr);
3034 ins_cost(2 * SVE_COST);
3035 format %{ "vmask_truecount_partial $dst, $src\t# vector mask truecount partial (sve)" %}
3036 ins_encode %{
3037 BasicType bt = Matcher::vector_element_basic_type(this, $src);
3038 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3039 __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), size, Matcher::vector_length(this, $src));
3040 __ sve_cntp($dst$$Register, size, as_PRegister($pgtmp$$reg), as_PRegister($src$$reg));
3041 %}
3042 ins_pipe(pipe_slow);
3043 %}
3044
3045 instruct vmask_firsttrue_partial(iRegINoSp dst, pReg src, pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
3046 predicate(UseSVE > 0 &&
3047 n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3048 match(Set dst (VectorMaskFirstTrue src));
3049 effect(TEMP pgtmp, TEMP ptmp, KILL cr);
3050 ins_cost(3 * SVE_COST);
3051 format %{ "vmask_firsttrue_partial $dst, $src\t# vector mask firsttrue partial (sve)" %}
3052 ins_encode %{
3053 BasicType bt = Matcher::vector_element_basic_type(this, $src);
3054 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3055 __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), size,
3056 Matcher::vector_length(this, $src));
3057 __ sve_brkb(as_PRegister($ptmp$$reg), as_PRegister($pgtmp$$reg), as_PRegister($src$$reg), false);
3058 __ sve_cntp($dst$$Register, size, as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg));
3059 %}
3060 ins_pipe(pipe_slow);
3061 %}
3062
3063 instruct vmask_lasttrue_partial(iRegINoSp dst, pReg src, pReg ptmp, rFlagsReg cr) %{
3064 predicate(UseSVE > 0 &&
3065 n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3066 match(Set dst (VectorMaskLastTrue src));
3067 effect(TEMP ptmp, KILL cr);
3068 ins_cost(5 * SVE_COST);
3069 format %{ "vmask_lasttrue_partial $dst, $src\t# vector mask lasttrue partial (sve)" %}
3070 ins_encode %{
3071 BasicType bt = Matcher::vector_element_basic_type(this, $src);
3072 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3073 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
3074 __ sve_and(as_PRegister($ptmp$$reg), ptrue, as_PRegister($ptmp$$reg), as_PRegister($src$$reg));
3075 __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg));
3076 %}
3077 ins_pipe(pipe_slow);
3078 %}dnl
|