1 //
2 // Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2020, 2022, Arm Limited. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 dnl Generate the warning
27 // This file is automatically generated by running "m4 aarch64_sve_ad.m4". Do not edit ----
28 dnl
29
30 // AArch64 SVE Architecture Description File
31
32 dnl
33 define(`TYPE2DATATYPE',
34 `ifelse($1, `B', `BYTE',
35 $1, `S', `SHORT',
36 $1, `I', `INT',
37 $1, `L', `LONG',
38 $1, `F', `FLOAT',
39 $1, `D', `DOUBLE',
40 `error($1)')')dnl
41 dnl
42 dnl OPERAND_VMEMORYA_IMMEDIATE_OFFSET($1, $2, $3 $4 )
43 dnl OPERAND_VMEMORYA_IMMEDIATE_OFFSET(imm_type_abbr, imm_type, imm_len, scale)
44 define(`OPERAND_VMEMORYA_IMMEDIATE_OFFSET', `
45 operand vmemA_imm$1Offset$3()
46 %{
47 // (esize / msize) = $4
48 predicate(Address::offset_ok_for_sve_immed(n->get_$2(), $3,
49 Matcher::scalable_vector_reg_size(T_BYTE)ifelse($4, `1', `', ` / $4')));
50 match(Con$1);
51
52 op_cost(0);
53 format %{ %}
54 interface(CONST_INTER);
55 %}')dnl
56
57 // 4 bit signed offset -- for predicated load/store
58 OPERAND_VMEMORYA_IMMEDIATE_OFFSET(I, int, 4, 1)
59 OPERAND_VMEMORYA_IMMEDIATE_OFFSET(L, long, 4, 1)
60 dnl
61 dnl OPERAND_VMEMORYA_INDIRECT_OFFSET($1, $2 )
62 dnl OPERAND_VMEMORYA_INDIRECT_OFFSET(imm_type_abbr, imm_len)
63 define(`OPERAND_VMEMORYA_INDIRECT_OFFSET', `
64 operand vmemA_indOff$1$2$3(iRegP reg, vmemA_imm$1Offset$2 off)
65 %{
66 constraint(ALLOC_IN_RC(ptr_reg));
67 match(AddP reg off);
68 op_cost(0);
69 format %{ "[$reg, $off]" %}
70 interface(MEMORY_INTER) %{
71 base($reg);
72 `index'(0xffffffff);
73 scale(0x0);
74 disp($off);
75 %}
76 %}')dnl
77 OPERAND_VMEMORYA_INDIRECT_OFFSET(I, 4)
78 OPERAND_VMEMORYA_INDIRECT_OFFSET(L, 4)
79
80 // The indOff of vmemA is valid only when the vector element (load to/store from)
81 // size equals to memory element (load from/store to) size.
82 opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);
83
84 source_hpp %{
85 bool op_sve_supported(int opcode, int vlen, BasicType bt);
86 bool masked_op_sve_supported(int opcode, int vlen, BasicType bt);
87 %}
88
89 source %{
90
91 typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
92 PRegister Pg, const Address &adr);
93
94 // Predicated load/store, with optional ptrue to all elements of given predicate register.
95 static void loadStoreA_predicated(C2_MacroAssembler masm, bool is_store, FloatRegister reg,
96 PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
97 int opcode, Register base, int index, int size, int disp) {
98 sve_mem_insn_predicate insn;
99 int mesize = type2aelembytes(mem_elem_bt);
100 if (index == -1) {
101 assert(size == 0, "unsupported address mode: scale size = %d", size);
102 switch(mesize) {
103 case 1:
104 insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
105 break;
106 case 2:
107 insn = is_store ? &C2_MacroAssembler::sve_st1h : &C2_MacroAssembler::sve_ld1h;
108 break;
109 case 4:
110 insn = is_store ? &C2_MacroAssembler::sve_st1w : &C2_MacroAssembler::sve_ld1w;
111 break;
112 case 8:
113 insn = is_store ? &C2_MacroAssembler::sve_st1d : &C2_MacroAssembler::sve_ld1d;
114 break;
115 default:
116 assert(false, "unsupported");
117 ShouldNotReachHere();
118 }
119 int imm4 = disp / mesize / Matcher::scalable_vector_reg_size(vector_elem_bt);
120 (masm.*insn)(reg, Assembler::elemType_to_regVariant(vector_elem_bt), pg, Address(base, imm4));
121 } else {
122 assert(false, "unimplemented");
123 ShouldNotReachHere();
124 }
125 }
126
127 bool op_sve_supported(int opcode, int vlen, BasicType bt) {
128 int length_in_bytes = vlen * type2aelembytes(bt);
129 switch (opcode) {
130 case Op_MulAddVS2VI:
131 // No multiply reduction instructions
132 case Op_MulReductionVD:
133 case Op_MulReductionVF:
134 case Op_MulReductionVI:
135 case Op_MulReductionVL:
136 // Others
137 case Op_ExtractC:
138 case Op_ExtractUB:
139 return false;
140 // Vector API specific
141 case Op_VectorLoadShuffle:
142 case Op_VectorRearrange:
143 return vlen >= 4 && length_in_bytes <= MaxVectorSize;
144 case Op_LoadVector:
145 case Op_StoreVector:
146 return Matcher::vector_size_supported(bt, vlen);
147 default:
148 break;
149 }
150 // By default, we only support vector operations with no less than 8 bytes and 2 elements.
151 return 8 <= length_in_bytes && length_in_bytes <= MaxVectorSize && vlen >= 2;
152 }
153
154 bool masked_op_sve_supported(int opcode, int vlen, BasicType bt) {
155 if (opcode == Op_VectorRearrange) {
156 return false;
157 }
158 return op_sve_supported(opcode, vlen, bt);
159 }
160 %}
161
162 definitions %{
163 int_def SVE_COST (200, 200);
164 %}
165
166 dnl
167 dnl ELEMENT_SHORT_CHART($1, $2)
168 dnl ELEMENT_SHORT_CHART(etype, node)
169 define(`ELEMENT_SHORT_CHAR',`ifelse(`$1', `T_SHORT',
170 `($2->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
171 ($2->bottom_type()->is_vect()->element_basic_type() == T_CHAR))',
172 `($2->bottom_type()->is_vect()->element_basic_type() == $1)')')dnl
173 dnl
174
175 // All SVE instructions
176
177 // vector load/store
178
179 // Unpredicated vector load/store
180 instruct loadV(vReg dst, vmemA mem) %{
181 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() >= 16 &&
182 n->as_LoadVector()->memory_size() == MaxVectorSize);
183 match(Set dst (LoadVector mem));
184 ins_cost(4 * SVE_COST);
185 format %{ "sve_ldr $dst, $mem\t# vector (sve)" %}
186 ins_encode %{
187 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
188 BasicType bt = Matcher::vector_element_basic_type(this);
189 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg, ptrue,
190 bt, bt, $mem->opcode(),
191 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
192 %}
193 ins_pipe(pipe_slow);
194 %}
195
196 instruct storeV(vReg src, vmemA mem) %{
197 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() >= 16 &&
198 n->as_StoreVector()->memory_size() == MaxVectorSize);
199 match(Set mem (StoreVector mem src));
200 ins_cost(4 * SVE_COST);
201 format %{ "sve_str $mem, $src\t# vector (sve)" %}
202 ins_encode %{
203 FloatRegister src_reg = as_FloatRegister($src$$reg);
204 BasicType bt = Matcher::vector_element_basic_type(this, $src);
205 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg, ptrue,
206 bt, bt, $mem->opcode(),
207 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
208 %}
209 ins_pipe(pipe_slow);
210 %}dnl
211
212 dnl
213 define(`VLoadStore', `
214 // ifelse(load, $3, Load, Store) Vector ($6 bits)
215 instruct $3V$4_vreg`'(vReg $7, vmem$4 mem)
216 %{
217 predicate(UseSVE > 0 && `n->as_'ifelse(load, $3, Load, Store)Vector()->memory_size() == $4);
218 match(Set ifelse(load, $3, dst (LoadVector mem), mem (StoreVector mem src)));
219 ins_cost(4 * INSN_COST);
220 format %{ "$1 ifelse(load, $3, `$dst,$mem', `$mem,$src')\t# vector ($6 bits)" %}
221 ins_encode( `aarch64_enc_'ifelse(load, $3, ldr, str)v$2($7, mem) );
222 ins_pipe(v$3`_reg_mem'ifelse(eval($4 * 8), 128, 128, 64));
223 %}')dnl
224 dnl $1 $2 $3 $4 $5 $6 $7
225 VLoadStore(ldrh, H, load, 2, D, 16, dst)
226 VLoadStore(strh, H, store, 2, D, 16, src)
227 VLoadStore(ldrs, S, load, 4, D, 32, dst)
228 VLoadStore(strs, S, store, 4, D, 32, src)
229 VLoadStore(ldrd, D, load, 8, D, 64, dst)
230 VLoadStore(strd, D, store, 8, D, 64, src)
231 VLoadStore(ldrq, Q, load, 16, X, 128, dst)
232 VLoadStore(strq, Q, store, 16, X, 128, src)
233
234 // Predicated vector load/store, based on the vector length of the node.
235 // Only load/store values in the range of the memory_size. This is needed
236 // when the memory_size is lower than the hardware supported max vector size.
237 // And this might happen for Vector API mask vector load/store.
238 instruct loadV_partial(vReg dst, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
239 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() > 16 &&
240 n->as_LoadVector()->memory_size() < MaxVectorSize);
241 match(Set dst (LoadVector mem));
242 effect(TEMP pgtmp, KILL cr);
243 ins_cost(6 * SVE_COST);
244 format %{ "sve_ptrue $pgtmp, vector_length\n\t"
245 "sve_ldr $dst, $pgtmp, $mem\t# load vector partial" %}
246 ins_encode %{
247 BasicType bt = Matcher::vector_element_basic_type(this);
248 __ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
249 Matcher::vector_length(this));
250 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
251 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg,
252 as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
253 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
254 %}
255 ins_pipe(pipe_slow);
256 %}
257
258 instruct storeV_partial(vReg src, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
259 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() > 16 &&
260 n->as_StoreVector()->memory_size() < MaxVectorSize);
261 match(Set mem (StoreVector mem src));
262 effect(TEMP pgtmp, KILL cr);
263 ins_cost(5 * SVE_COST);
264 format %{ "sve_ptrue $pgtmp, vector_length\n\t"
265 "sve_str $src, $pgtmp, $mem\t# store vector partial" %}
266 ins_encode %{
267 BasicType bt = Matcher::vector_element_basic_type(this, $src);
268 __ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
269 Matcher::vector_length(this, $src));
270 FloatRegister src_reg = as_FloatRegister($src$$reg);
271 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg,
272 as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
273 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
274 %}
275 ins_pipe(pipe_slow);
276 %}
277
278 // vector load/store - predicated
279
280 instruct loadV_masked(vReg dst, vmemA mem, pRegGov pg) %{
281 predicate(UseSVE > 0);
282 match(Set dst (LoadVectorMasked mem pg));
283 ins_cost(4 * SVE_COST);
284 format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated (sve)" %}
285 ins_encode %{
286 BasicType bt = Matcher::vector_element_basic_type(this);
287 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
288 as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
289 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
290 %}
291 ins_pipe(pipe_slow);
292 %}
293
294 instruct storeV_masked(vReg src, vmemA mem, pRegGov pg) %{
295 predicate(UseSVE > 0);
296 match(Set mem (StoreVectorMasked mem (Binary src pg)));
297 ins_cost(4 * SVE_COST);
298 format %{ "sve_str $mem, $pg, $src\t# store vector predicated (sve)" %}
299 ins_encode %{
300 BasicType bt = Matcher::vector_element_basic_type(this, $src);
301 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
302 as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
303 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
304 %}
305 ins_pipe(pipe_slow);
306 %}
307
308 dnl
309 dnl MASK_LOGICAL_OP($1, $2, $3 )
310 dnl MASK_LOGICAL_OP(insn_name, op_name, insn)
311 define(`MASK_LOGICAL_OP', `
312 instruct vmask_$1(pRegGov pd, pRegGov pn, pRegGov pm) %{
313 predicate(UseSVE > 0);
314 match(Set pd ($2 pn pm));
315 ins_cost(SVE_COST);
316 format %{ "$3 $pd, $pn, $pm\t# predicate (sve)" %}
317 ins_encode %{
318 __ $3(as_PRegister($pd$$reg), ptrue,
319 as_PRegister($pn$$reg), as_PRegister($pm$$reg));
320 %}
321 ins_pipe(pipe_slow);
322 %}')dnl
323 dnl
324 // mask logical and/or/xor
325 MASK_LOGICAL_OP(and, AndVMask, sve_and)
326 MASK_LOGICAL_OP(or, OrVMask, sve_orr)
327 MASK_LOGICAL_OP(xor, XorVMask, sve_eor)
328
329 dnl
330 dnl MASK_LOGICAL_AND_NOT($1, $2 )
331 dnl MASK_LOGICAL_AND_NOT(type, size)
332 define(`MASK_LOGICAL_AND_NOT', `
333 instruct vmask_and_not$1(pRegGov pd, pRegGov pn, pRegGov pm, imm$1_M1 m1) %{
334 predicate(UseSVE > 0);
335 match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
336 ins_cost(SVE_COST);
337 format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) ($2)" %}
338 ins_encode %{
339 __ sve_bic(as_PRegister($pd$$reg), ptrue,
340 as_PRegister($pn$$reg), as_PRegister($pm$$reg));
341 %}
342 ins_pipe(pipe_slow);
343 %}')dnl
344 dnl
345 // mask logical and_not
346 MASK_LOGICAL_AND_NOT(I, B/H/S)
347 MASK_LOGICAL_AND_NOT(L, D)
348
349 // vector reinterpret
350
351 instruct reinterpret(vReg dst) %{
352 predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() ==
353 n->in(1)->bottom_type()->is_vect()->length_in_bytes()); // src == dst
354 match(Set dst (VectorReinterpret dst));
355 ins_cost(0);
356 format %{ "# reinterpret $dst\t# do nothing" %}
357 ins_encode %{
358 // empty
359 %}
360 ins_pipe(pipe_class_empty);
361 %}
362
363 instruct reinterpretResize(vReg dst, vReg src, pRegGov pgtmp, rFlagsReg cr) %{
364 predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() !=
365 n->in(1)->bottom_type()->is_vect()->length_in_bytes()); // src != dst
366 match(Set dst (VectorReinterpret src));
367 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
368 ins_cost(3 * SVE_COST);
369 format %{ "reinterpretResize $dst, $src\t# vector (sve)" %}
370 ins_encode %{
371 uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
372 uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
373 uint length_in_bytes_resize = length_in_bytes_src < length_in_bytes_dst ?
374 length_in_bytes_src : length_in_bytes_dst;
375 assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
376 "invalid vector length");
377 __ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ B, length_in_bytes_resize);
378 __ sve_dup(as_FloatRegister($dst$$reg), __ B, 0);
379 __ sve_sel(as_FloatRegister($dst$$reg), __ B, as_PRegister($pgtmp$$reg),
380 as_FloatRegister($src$$reg), as_FloatRegister($dst$$reg));
381 %}
382 ins_pipe(pipe_slow);
383 %}
384
385 // vector mask reinterpret
386
387 instruct vmask_reinterpret_same_esize(pRegGov dst_src) %{
388 predicate(UseSVE > 0 &&
389 n->as_Vector()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
390 n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
391 match(Set dst_src (VectorReinterpret dst_src));
392 ins_cost(0);
393 format %{ "# vmask_reinterpret $dst_src\t# do nothing" %}
394 ins_encode %{
395 // empty
396 %}
397 ins_pipe(pipe_class_empty);
398 %}
399
400 instruct vmask_reinterpret_diff_esize(pRegGov dst, pRegGov src, vReg tmp, rFlagsReg cr) %{
401 predicate(UseSVE > 0 &&
402 n->as_Vector()->length() != n->in(1)->bottom_type()->is_vect()->length() &&
403 n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
404 match(Set dst (VectorReinterpret src));
405 effect(TEMP tmp, KILL cr);
406 ins_cost(2 * SVE_COST);
407 format %{ "# vmask_reinterpret $dst, $src\t# vector (sve)" %}
408 ins_encode %{
409 BasicType from_bt = Matcher::vector_element_basic_type(this, $src);
410 Assembler::SIMD_RegVariant from_size = __ elemType_to_regVariant(from_bt);
411 BasicType to_bt = Matcher::vector_element_basic_type(this);
412 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
413 __ sve_cpy(as_FloatRegister($tmp$$reg), from_size, as_PRegister($src$$reg), -1, false);
414 __ sve_cmp(Assembler::EQ, as_PRegister($dst$$reg), to_size, ptrue, as_FloatRegister($tmp$$reg), -1);
415 %}
416 ins_pipe(pipe_slow);
417 %}
418 dnl
419 dnl UNARY_OP_TRUE_PREDICATE($1, $2, $3, $4 )
420 dnl UNARY_OP_TRUE_PREDICATE(insn_name, op_name, size, insn)
421 define(`UNARY_OP_TRUE_PREDICATE', `
422 instruct $1(vReg dst, vReg src) %{
423 predicate(UseSVE > 0 &&
424 !n->as_Vector()->is_predicated_vector());
425 match(Set dst ($2 src));
426 ins_cost(SVE_COST);
427 format %{ "$4 $dst, $src\t# vector (sve) ($3)" %}
428 ins_encode %{dnl
429 ifelse($1, `vnegI', `
430 BasicType bt = Matcher::vector_element_basic_type(this);', `')
431 __ $4(as_FloatRegister($dst$$reg), ifelse($1, `vnegI', `__ elemType_to_regVariant(bt)', `__ $3'),
432 ptrue, as_FloatRegister($src$$reg));
433 %}
434 ins_pipe(pipe_slow);
435 %}')dnl
436 dnl
437
438 // vector abs
439 UNARY_OP_TRUE_PREDICATE(vabsB, AbsVB, B, sve_abs)
440 UNARY_OP_TRUE_PREDICATE(vabsS, AbsVS, H, sve_abs)
441 UNARY_OP_TRUE_PREDICATE(vabsI, AbsVI, S, sve_abs)
442 UNARY_OP_TRUE_PREDICATE(vabsL, AbsVL, D, sve_abs)
443 UNARY_OP_TRUE_PREDICATE(vabsF, AbsVF, S, sve_fabs)
444 UNARY_OP_TRUE_PREDICATE(vabsD, AbsVD, D, sve_fabs)
445
446 dnl UNARY_OP_PREDICATE($1, $2, $3, $4 )
447 dnl UNARY_OP_PREDICATE(insn_name, op_name, size, insn)
448 define(`UNARY_OP_PREDICATE', `
449 instruct $1_masked(vReg dst_src, pRegGov pg) %{
450 predicate(UseSVE > 0);
451 match(Set dst_src ($2 dst_src pg));
452 ins_cost(SVE_COST);
453 format %{ "$4 $dst_src, $pg, $dst_src\t# vector (sve) ($3)" %}
454 ins_encode %{dnl
455 ifelse($1, `vnegI', `
456 BasicType bt = Matcher::vector_element_basic_type(this);', `')
457 __ $4(as_FloatRegister($dst_src$$reg), ifelse($1, `vnegI', `__ elemType_to_regVariant(bt)', `__ $3'),
458 as_PRegister($pg$$reg),
459 as_FloatRegister($dst_src$$reg));
460 %}
461 ins_pipe(pipe_slow);
462 %}')dnl
463 // vector abs - predicated
464 UNARY_OP_PREDICATE(vabsB, AbsVB, B, sve_abs)
465 UNARY_OP_PREDICATE(vabsS, AbsVS, H, sve_abs)
466 UNARY_OP_PREDICATE(vabsI, AbsVI, S, sve_abs)
467 UNARY_OP_PREDICATE(vabsL, AbsVL, D, sve_abs)
468 UNARY_OP_PREDICATE(vabsF, AbsVF, S, sve_fabs)
469 UNARY_OP_PREDICATE(vabsD, AbsVD, D, sve_fabs)
470
471 dnl
472 dnl BINARY_OP_UNPREDICATE($1, $2 $3, $4 $5 )
473 dnl BINARY_OP_UNPREDICATE(insn_name, op_name, size, min_vec_len, insn)
474 define(`BINARY_OP_UNPREDICATE', `
475 instruct $1(vReg dst, vReg src1, vReg src2) %{
476 predicate(UseSVE > 0);
477 match(Set dst ($2 src1 src2));
478 ins_cost(SVE_COST);
479 format %{ "$5 $dst, $src1, $src2\t # vector (sve) ($3)" %}
480 ins_encode %{
481 __ $5(as_FloatRegister($dst$$reg), __ $3,
482 as_FloatRegister($src1$$reg),
483 as_FloatRegister($src2$$reg));
484 %}
485 ins_pipe(pipe_slow);
486 %}')dnl
487 dnl
488 dnl
489 dnl BINARY_OP_PREDICATE($1, $2, $3, $4 )
490 dnl BINARY_OP_PREDICATE(insn_name, op_name, size, insn)
491 define(`BINARY_OP_PREDICATE', `
492 instruct $1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
493 predicate(UseSVE > 0);
494 match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
495 ins_cost(SVE_COST);
496 format %{ "$4 $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) ($3)" %}
497 ins_encode %{
498 __ $4(as_FloatRegister($dst_src1$$reg), __ $3,
499 as_PRegister($pg$$reg),
500 as_FloatRegister($src2$$reg));
501 %}
502 ins_pipe(pipe_slow);
503 %}')dnl
504 dnl
505 // vector add
506 BINARY_OP_UNPREDICATE(vaddB, AddVB, B, 16, sve_add)
507 BINARY_OP_UNPREDICATE(vaddS, AddVS, H, 8, sve_add)
508 BINARY_OP_UNPREDICATE(vaddI, AddVI, S, 4, sve_add)
509 BINARY_OP_UNPREDICATE(vaddL, AddVL, D, 2, sve_add)
510 BINARY_OP_UNPREDICATE(vaddF, AddVF, S, 4, sve_fadd)
511 BINARY_OP_UNPREDICATE(vaddD, AddVD, D, 2, sve_fadd)
512
513 // vector add - predicated
514 BINARY_OP_PREDICATE(vaddB, AddVB, B, sve_add)
515 BINARY_OP_PREDICATE(vaddS, AddVS, H, sve_add)
516 BINARY_OP_PREDICATE(vaddI, AddVI, S, sve_add)
517 BINARY_OP_PREDICATE(vaddL, AddVL, D, sve_add)
518 BINARY_OP_PREDICATE(vaddF, AddVF, S, sve_fadd)
519 BINARY_OP_PREDICATE(vaddD, AddVD, D, sve_fadd)
520 dnl
521 dnl ADD_IMM($1, $2, $3 )
522 dnl ADD_IMM(name_suffix, size, imm_type)
523 define(`ADD_IMM', `
524 instruct vaddImm$1(vReg dst_src, $3 con) %{
525 predicate(UseSVE > 0);
526 match(Set dst_src (AddV$1 dst_src (Replicate$1 con)));
527 ins_cost(SVE_COST);
528 format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) ($2)" %}
529 ins_encode %{
530 int32_t val = $con$$constant;
531 if (val > 0){
532 __ sve_add(as_FloatRegister($dst_src$$reg), __ $2, val);
533 } else if (val < 0){
534 __ sve_sub(as_FloatRegister($dst_src$$reg), __ $2, -val);
535 }
536 %}
537 ins_pipe(pipe_slow);
538 %}')dnl
539
540 // vector add reg imm (unpredicated)
541 ADD_IMM(B, B, immBAddSubV)
542 ADD_IMM(S, H, immIAddSubV)
543 ADD_IMM(I, S, immIAddSubV)
544 ADD_IMM(L, D, immLAddSubV)
545 dnl
546 dnl BITWISE_OP_IMM($1, $2 $3, $4 $5 )
547 dnl BITWISE_OP_IMM(insn_name, op_name1, size, type, op_name2)
548 define(`BITWISE_OP_IMM', `
549 instruct $1(vReg dst_src, imm$4Log con) %{
550 predicate(UseSVE > 0);
551 match(Set dst_src ($2 dst_src (Replicate$4 con)));
552 ins_cost(SVE_COST);
553 format %{ "$5 $dst_src, $dst_src, $con\t # vector (sve) ($3)" %}
554 ins_encode %{
555 __ $5(as_FloatRegister($dst_src$$reg), __ $3,
556 (uint64_t)($con$$constant));
557 %}
558 ins_pipe(pipe_slow);
559 %}')dnl
560
561 // vector binary op reg imm (unpredicated)
562 BITWISE_OP_IMM(vandB, AndV, B, B, sve_and)
563 BITWISE_OP_IMM(vandH, AndV, H, S, sve_and)
564 BITWISE_OP_IMM(vandS, AndV, S, I, sve_and)
565 BITWISE_OP_IMM(vandD, AndV, D, L, sve_and)
566 BITWISE_OP_IMM(vorB, OrV, B, B, sve_orr)
567 BITWISE_OP_IMM(vorH, OrV, H, S, sve_orr)
568 BITWISE_OP_IMM(vorS, OrV, S, I, sve_orr)
569 BITWISE_OP_IMM(vorD, OrV, D, L, sve_orr)
570 BITWISE_OP_IMM(vxorB, XorV, B, B, sve_eor)
571 BITWISE_OP_IMM(vxorH, XorV, H, S, sve_eor)
572 BITWISE_OP_IMM(vxorS, XorV, S, I, sve_eor)
573 BITWISE_OP_IMM(vxorD, XorV, D, L, sve_eor)
574 dnl
575 dnl
576 dnl BINARY_OP_UNSIZED($1, $2, $3 )
577 dnl BINARY_OP_UNSIZED(insn_name, op_name, insn)
578 define(`BINARY_OP_UNSIZED', `
579 instruct $1(vReg dst, vReg src1, vReg src2) %{
580 predicate(UseSVE > 0);
581 match(Set dst ($2 src1 src2));
582 ins_cost(SVE_COST);
583 format %{ "$3 $dst, $src1, $src2\t# vector (sve)" %}
584 ins_encode %{
585 __ $3(as_FloatRegister($dst$$reg),
586 as_FloatRegister($src1$$reg),
587 as_FloatRegister($src2$$reg));
588 %}
589 ins_pipe(pipe_slow);
590 %}')dnl
591 dnl
592 // vector and
593 BINARY_OP_UNSIZED(vand, AndV, sve_and)
594
595 // vector or
596 BINARY_OP_UNSIZED(vor, OrV, sve_orr)
597
598 // vector xor
599 BINARY_OP_UNSIZED(vxor, XorV, sve_eor)
600
601 dnl BINARY_LOGIC_OP_PREDICATE($1, $2, $3 )
602 dnl BINARY_LOGIC_OP_PREDICATE(insn_name, op_name, insn)
603 define(`BINARY_LOGIC_OP_PREDICATE', `
604 instruct $1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
605 predicate(UseSVE > 0);
606 match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
607 ins_cost(SVE_COST);
608 format %{ "$3 $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
609 ins_encode %{
610 BasicType bt = Matcher::vector_element_basic_type(this);
611 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
612 __ $3(as_FloatRegister($dst_src1$$reg), size,
613 as_PRegister($pg$$reg),
614 as_FloatRegister($src2$$reg));
615 %}
616 ins_pipe(pipe_slow);
617 %}')dnl
618 dnl
619 // vector and - predicated
620 BINARY_LOGIC_OP_PREDICATE(vand, AndV, sve_and)
621
622 // vector or - predicated
623 BINARY_LOGIC_OP_PREDICATE(vor, OrV, sve_orr)
624
625 // vector xor - predicated
626 BINARY_LOGIC_OP_PREDICATE(vxor, XorV, sve_eor)
627
628 // vector not
629 dnl
630 define(`MATCH_RULE', `ifelse($1, I,
631 `match(Set dst (XorV src (ReplicateB m1)));
632 match(Set dst (XorV src (ReplicateS m1)));
633 match(Set dst (XorV src (ReplicateI m1)));',
634 `match(Set dst (XorV src (ReplicateL m1)));')')dnl
635 dnl
636 define(`VECTOR_NOT', `
637 instruct vnot$1`'(vReg dst, vReg src, imm$1_M1 m1) %{
638 predicate(UseSVE > 0);
639 MATCH_RULE($1)
640 ins_cost(SVE_COST);
641 format %{ "sve_not $dst, $src\t# vector (sve) $2" %}
642 ins_encode %{
643 __ sve_not(as_FloatRegister($dst$$reg), __ D,
644 ptrue, as_FloatRegister($src$$reg));
645 %}
646 ins_pipe(pipe_slow);
647 %}')dnl
648 dnl $1,$2
649 VECTOR_NOT(I, B/H/S)
650 VECTOR_NOT(L, D)
651 undefine(MATCH_RULE)
652 dnl
653 // vector not - predicated
654 dnl
655 define(`MATCH_RULE', `ifelse($1, I,
656 `match(Set dst (XorV (Binary src (ReplicateB m1)) pg));
657 match(Set dst (XorV (Binary src (ReplicateS m1)) pg));
658 match(Set dst (XorV (Binary src (ReplicateI m1)) pg));',
659 `match(Set dst (XorV (Binary src (ReplicateL m1)) pg));')')dnl
660 dnl
661 define(`VECTOR_NOT_PREDICATE', `
662 instruct vnot$1_masked`'(vReg dst, vReg src, imm$1_M1 m1, pRegGov pg) %{
663 predicate(UseSVE > 0);
664 MATCH_RULE($1)
665 ins_cost(SVE_COST);
666 format %{ "sve_not $dst, $pg, $src\t# vector (sve) $2" %}
667 ins_encode %{
668 BasicType bt = Matcher::vector_element_basic_type(this);
669 __ sve_not(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
670 as_PRegister($pg$$reg), as_FloatRegister($src$$reg));
671 %}
672 ins_pipe(pipe_slow);
673 %}')dnl
674 dnl $1, $2
675 VECTOR_NOT_PREDICATE(I, B/H/S)
676 VECTOR_NOT_PREDICATE(L, D)
677 undefine(MATCH_RULE)
678 dnl
679 // vector and_not
680 dnl
681 define(`MATCH_RULE', `ifelse($1, I,
682 `match(Set dst (AndV src1 (XorV src2 (ReplicateB m1))));
683 match(Set dst (AndV src1 (XorV src2 (ReplicateS m1))));
684 match(Set dst (AndV src1 (XorV src2 (ReplicateI m1))));',
685 `match(Set dst (AndV src1 (XorV src2 (ReplicateL m1))));')')dnl
686 dnl
687 define(`VECTOR_AND_NOT', `
688 instruct vand_not$1`'(vReg dst, vReg src1, vReg src2, imm$1_M1 m1) %{
689 predicate(UseSVE > 0);
690 MATCH_RULE($1)
691 ins_cost(SVE_COST);
692 format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) $2" %}
693 ins_encode %{
694 __ sve_bic(as_FloatRegister($dst$$reg),
695 as_FloatRegister($src1$$reg),
696 as_FloatRegister($src2$$reg));
697 %}
698 ins_pipe(pipe_slow);
699 %}')dnl
700 dnl $1,$2
701 VECTOR_AND_NOT(I, B/H/S)
702 VECTOR_AND_NOT(L, D)
703 undefine(MATCH_RULE)
704 dnl
705 // vector and_not - predicated
706 dnl
707 define(`MATCH_RULE', `ifelse($1, I,
708 `match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateB m1))) pg));
709 match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateS m1))) pg));
710 match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateI m1))) pg));',
711 `match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateL m1))) pg));')')dnl
712 dnl
713 define(`VECTOR_AND_NOT_PREDICATE', `
714 instruct vand_not$1_masked`'(vReg dst_src1, vReg src2, imm$1_M1 m1, pRegGov pg) %{
715 predicate(UseSVE > 0);
716 MATCH_RULE($1)
717 ins_cost(SVE_COST);
718 format %{ "sve_bic $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) $2" %}
719 ins_encode %{
720 BasicType bt = Matcher::vector_element_basic_type(this);
721 __ sve_bic(as_FloatRegister($dst_src1$$reg), __ elemType_to_regVariant(bt),
722 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
723 %}
724 ins_pipe(pipe_slow);
725 %}')dnl
726 dnl $1, $2
727 VECTOR_AND_NOT_PREDICATE(I, B/H/S)
728 VECTOR_AND_NOT_PREDICATE(L, D)
729 undefine(MATCH_RULE)
730 dnl
731 dnl VDIVF($1, $2 , $3 )
732 dnl VDIVF(name_suffix, size, min_vec_len)
733 define(`VDIVF', `
734 instruct vdiv$1(vReg dst_src1, vReg src2) %{
735 predicate(UseSVE > 0);
736 match(Set dst_src1 (DivV$1 dst_src1 src2));
737 ins_cost(SVE_COST);
738 format %{ "sve_fdiv $dst_src1, $dst_src1, $src2\t# vector (sve) ($2)" %}
739 ins_encode %{
740 __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ $2,
741 ptrue, as_FloatRegister($src2$$reg));
742 %}
743 ins_pipe(pipe_slow);
744 %}')dnl
745 dnl
746 // vector float div
747 VDIVF(F, S, 4)
748 VDIVF(D, D, 2)
749
750 // vector float div - predicated
751 BINARY_OP_PREDICATE(vfdivF, DivVF, S, sve_fdiv)
752 BINARY_OP_PREDICATE(vfdivD, DivVD, D, sve_fdiv)
753
754 dnl
755 dnl VMINMAX($1 , $2, $3 , $4 )
756 dnl VMINMAX(op_name, op, finsn, insn)
757 define(`VMINMAX', `
758 instruct v$1(vReg dst_src1, vReg src2) %{
759 predicate(UseSVE > 0);
760 match(Set dst_src1 ($2 dst_src1 src2));
761 ins_cost(SVE_COST);
762 format %{ "sve_$1 $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
763 ins_encode %{
764 BasicType bt = Matcher::vector_element_basic_type(this);
765 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
766 if (is_floating_point_type(bt)) {
767 __ $3(as_FloatRegister($dst_src1$$reg), size,
768 ptrue, as_FloatRegister($src2$$reg));
769 } else {
770 assert(is_integral_type(bt), "unsupported type");
771 __ $4(as_FloatRegister($dst_src1$$reg), size,
772 ptrue, as_FloatRegister($src2$$reg));
773 }
774 %}
775 ins_pipe(pipe_slow);
776 %}')dnl
777 dnl
778 // vector min/max
779 VMINMAX(min, MinV, sve_fmin, sve_smin)
780 VMINMAX(max, MaxV, sve_fmax, sve_smax)
781
782 dnl
783 dnl VMINMAX_PREDICATE($1 , $2, $3 , $4 )
784 dnl VMINMAX_PREDICATE(op_name, op, finsn, insn)
785 define(`VMINMAX_PREDICATE', `
786 instruct v$1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
787 predicate(UseSVE > 0);
788 match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
789 ins_cost(SVE_COST);
790 format %{ "sve_$1 $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %}
791 ins_encode %{
792 BasicType bt = Matcher::vector_element_basic_type(this);
793 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
794 if (is_floating_point_type(bt)) {
795 __ $3(as_FloatRegister($dst_src1$$reg), size,
796 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
797 } else {
798 assert(is_integral_type(bt), "unsupported type");
799 __ $4(as_FloatRegister($dst_src1$$reg), size,
800 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
801 }
802 %}
803 ins_pipe(pipe_slow);
804 %}')dnl
805 dnl
806 // vector min/max - predicated
807 VMINMAX_PREDICATE(min, MinV, sve_fmin, sve_smin)
808 VMINMAX_PREDICATE(max, MaxV, sve_fmax, sve_smax)
809
810 dnl
811 dnl VFMLA($1 $2 )
812 dnl VFMLA(name_suffix, size)
813 define(`VFMLA', `
814 // dst_src1 = dst_src1 + src2 * src3
815 instruct vfmla$1(vReg dst_src1, vReg src2, vReg src3) %{
816 predicate(UseFMA && UseSVE > 0);
817 match(Set dst_src1 (FmaV$1 dst_src1 (Binary src2 src3)));
818 ins_cost(SVE_COST);
819 format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
820 ins_encode %{
821 __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ $2,
822 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
823 %}
824 ins_pipe(pipe_slow);
825 %}')dnl
826 dnl
827 // vector fmla
828 VFMLA(F, S)
829 VFMLA(D, D)
830
831 dnl
832 dnl VFMAD_PREDICATE($1 $2 )
833 dnl VFMAD_PREDICATE(name_suffix, size)
834 define(`VFMAD_PREDICATE', `
835 // dst_src1 = dst_src1 * src2 + src3
836 instruct vfmad$1_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
837 predicate(UseFMA && UseSVE > 0);
838 match(Set dst_src1 (FmaV$1 (Binary dst_src1 src2) (Binary src3 pg)));
839 ins_cost(SVE_COST);
840 format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) ($2)" %}
841 ins_encode %{
842 __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ $2, as_PRegister($pg$$reg),
843 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
844 %}
845 ins_pipe(pipe_slow);
846 %}')dnl
847 dnl
848 // vector fmad - predicated
849 VFMAD_PREDICATE(F, S)
850 VFMAD_PREDICATE(D, D)
851
852 dnl
853 dnl VFMLS1($1 $2 )
854 dnl VFMLS1(name_suffix, size)
855 define(`VFMLS1', `
856 // dst_src1 = dst_src1 + -src2 * src3
857 // The NegV$1 must not be predicated.
858 instruct vfmls`$1'1(vReg dst_src1, vReg src2, vReg src3) %{
859 predicate(UseFMA && UseSVE > 0 &&
860 !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
861 match(Set dst_src1 (FmaV$1 dst_src1 (Binary (NegV$1 src2) src3)));
862 ins_cost(SVE_COST);
863 format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
864 ins_encode %{
865 __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ $2,
866 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
867 %}
868 ins_pipe(pipe_slow);
869 %}')dnl
870 dnl
871 dnl VFMLS2($1 $2 )
872 dnl VFMLS2(name_suffix, size)
873 define(`VFMLS2', `
874 // dst_src1 = dst_src1 + src2 * -src3
875 // The NegV$1 must not be predicated.
876 instruct vfmls`$1'2(vReg dst_src1, vReg src2, vReg src3) %{
877 predicate(UseFMA && UseSVE > 0 &&
878 !n->in(2)->in(2)->as_Vector()->is_predicated_vector());
879 match(Set dst_src1 (FmaV$1 dst_src1 (Binary src2 (NegV$1 src3))));
880 ins_cost(SVE_COST);
881 format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
882 ins_encode %{
883 __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ $2,
884 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
885 %}
886 ins_pipe(pipe_slow);
887 %}')dnl
888 dnl
889 // vector fmls
890 VFMLS1(F, S)
891 VFMLS2(F, S)
892 VFMLS1(D, D)
893 VFMLS2(D, D)
894
895 dnl
896 dnl VFMSB_PREDICATE($1 $2 )
897 dnl VFMSB_PREDICATE(name_suffix, size)
898 define(`VFMSB_PREDICATE', `
899 // dst_src1 = dst_src1 * -src2 + src3
900 // The NegV$1 must not be predicated.
901 instruct vfmsb$1_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
902 predicate(UseFMA && UseSVE > 0 &&
903 !n->in(1)->in(2)->as_Vector()->is_predicated_vector());
904 match(Set dst_src1 (FmaV$1 (Binary dst_src1 (NegV$1 src2)) (Binary src3 pg)));
905 ins_cost(SVE_COST);
906 format %{ "sve_fmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) ($2)" %}
907 ins_encode %{
908 __ sve_fmsb(as_FloatRegister($dst_src1$$reg), __ $2, as_PRegister($pg$$reg),
909 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
910 %}
911 ins_pipe(pipe_slow);
912 %}')dnl
913 dnl
914 // vector fmsb - predicated
915 VFMSB_PREDICATE(F, S)
916 VFMSB_PREDICATE(D, D)
917
918 dnl
919 dnl VFNMLA1($1 $2 )
920 dnl VFNMLA1(name_suffix, size)
921 define(`VFNMLA1', `
922 // dst_src1 = -dst_src1 + -src2 * src3
923 // The NegV$1 must not be predicated.
924 instruct vfnmla`$1'1(vReg dst_src1, vReg src2, vReg src3) %{
925 predicate(UseFMA && UseSVE > 0 &&
926 !n->in(1)->as_Vector()->is_predicated_vector() &&
927 !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
928 match(Set dst_src1 (FmaV$1 (NegV$1 dst_src1) (Binary (NegV$1 src2) src3)));
929 ins_cost(SVE_COST);
930 format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
931 ins_encode %{
932 __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ $2,
933 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
934 %}
935 ins_pipe(pipe_slow);
936 %}')dnl
937 dnl
938 dnl VFNMLA2($1 $2 )
939 dnl VFNMLA2(name_suffix, size)
940 define(`VFNMLA2', `
941 // dst_src1 = -dst_src1 + src2 * -src3
942 // The NegV$1 must not be predicated.
943 instruct vfnmla`$1'2(vReg dst_src1, vReg src2, vReg src3) %{
944 predicate(UseFMA && UseSVE > 0 &&
945 !n->in(1)->as_Vector()->is_predicated_vector() &&
946 !n->in(2)->in(2)->as_Vector()->is_predicated_vector());
947 match(Set dst_src1 (FmaV$1 (NegV$1 dst_src1) (Binary src2 (NegV$1 src3))));
948 ins_cost(SVE_COST);
949 format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
950 ins_encode %{
951 __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ $2,
952 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
953 %}
954 ins_pipe(pipe_slow);
955 %}')dnl
956 dnl
957 // vector fnmla
958 VFNMLA1(F, S)
959 VFNMLA2(F, S)
960 VFNMLA1(D, D)
961 VFNMLA2(D, D)
962
963 dnl
964 dnl VFNMAD_PREDICATE($1 $2 )
965 dnl VFNMAD_PREDICATE(name_suffix, size)
966 define(`VFNMAD_PREDICATE', `
967 // dst_src1 = -src3 + dst_src1 * -src2
968 // The NegV$1 must not be predicated.
969 instruct vfnmad$1_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
970 predicate(UseFMA && UseSVE > 0 &&
971 !n->in(1)->in(2)->as_Vector()->is_predicated_vector() &&
972 !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
973 match(Set dst_src1 (FmaV$1 (Binary dst_src1 (NegV$1 src2)) (Binary (NegV$1 src3) pg)));
974 ins_cost(SVE_COST);
975 format %{ "sve_fnmad $dst_src1, $pg, $src2, $src3\t # vector (sve) ($2)" %}
976 ins_encode %{
977 __ sve_fnmad(as_FloatRegister($dst_src1$$reg), __ $2, as_PRegister($pg$$reg),
978 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
979 %}
980 ins_pipe(pipe_slow);
981 %}')dnl
982 dnl
983 // vector fnmad - predicated
984 VFNMAD_PREDICATE(F, S)
985 VFNMAD_PREDICATE(D, D)
986
987 dnl
988 dnl VFNMLS($1 $2 )
989 dnl VFNMLS(name_suffix, size)
990 define(`VFNMLS', `
991 // dst_src1 = -dst_src1 + src2 * src3
992 // The NegV$1 must not be predicated.
993 instruct vfnmls$1(vReg dst_src1, vReg src2, vReg src3) %{
994 predicate(UseFMA && UseSVE > 0 &&
995 !n->in(1)->as_Vector()->is_predicated_vector());
996 match(Set dst_src1 (FmaV$1 (NegV$1 dst_src1) (Binary src2 src3)));
997 ins_cost(SVE_COST);
998 format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
999 ins_encode %{
1000 __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ $2,
1001 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1002 %}
1003 ins_pipe(pipe_slow);
1004 %}')dnl
1005 dnl
1006 // vector fnmls
1007 VFNMLS(F, S)
1008 VFNMLS(D, D)
1009
1010 dnl
1011 dnl VFNMSB_PREDICATE($1 $2 )
1012 dnl VFNMSB_PREDICATE(name_suffix, size)
1013 define(`VFNMSB_PREDICATE', `
1014 // dst_src1 = -src3 + dst_src1 * src2
1015 // The NegV$1 must not be predicated.
1016 instruct vfnmsb$1_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
1017 predicate(UseFMA && UseSVE > 0 &&
1018 !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
1019 match(Set dst_src1 (FmaV$1 (Binary dst_src1 src2) (Binary (NegV$1 src3) pg)));
1020 ins_cost(SVE_COST);
1021 format %{ "sve_fnmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) ($2)" %}
1022 ins_encode %{
1023 __ sve_fnmsb(as_FloatRegister($dst_src1$$reg), __ $2, as_PRegister($pg$$reg),
1024 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1025 %}
1026 ins_pipe(pipe_slow);
1027 %}')dnl
1028 dnl
1029 // vector fnmsb - predicated
1030 VFNMSB_PREDICATE(F, S)
1031 VFNMSB_PREDICATE(D, D)
1032
1033 dnl
1034 dnl VMLA($1 $2 )
1035 dnl VMLA(name_suffix, size)
1036 define(`VMLA', `
1037 // dst_src1 = dst_src1 + src2 * src3
1038 instruct vmla$1(vReg dst_src1, vReg src2, vReg src3)
1039 %{
1040 predicate(UseSVE > 0);
1041 match(Set dst_src1 (AddV$1 dst_src1 (MulV$1 src2 src3)));
1042 ins_cost(SVE_COST);
1043 format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) ($2)" %}
1044 ins_encode %{
1045 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ $2,
1046 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1047 %}
1048 ins_pipe(pipe_slow);
1049 %}')dnl
1050 dnl
1051 // vector mla
1052 VMLA(B, B)
1053 VMLA(S, H)
1054 VMLA(I, S)
1055 VMLA(L, D)
1056
1057 dnl
1058 dnl VMLA_PREDICATE($1 $2 )
1059 dnl VMLA_PREDICATE(name_suffix, size)
1060 define(`VMLA_PREDICATE', `
1061 // dst_src1 = dst_src1 + src2 * src3
1062 instruct vmla$1_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1063 %{
1064 predicate(UseSVE > 0);
1065 match(Set dst_src1 (AddV$1 (Binary dst_src1 (MulV$1 src2 src3)) pg));
1066 ins_cost(SVE_COST);
1067 format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) ($2)" %}
1068 ins_encode %{
1069 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ $2, as_PRegister($pg$$reg),
1070 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1071 %}
1072 ins_pipe(pipe_slow);
1073 %}')dnl
1074 dnl
1075 // vector mla - predicated
1076 VMLA_PREDICATE(B, B)
1077 VMLA_PREDICATE(S, H)
1078 VMLA_PREDICATE(I, S)
1079 VMLA_PREDICATE(L, D)
1080
1081 dnl
1082 dnl VMLS($1 $2 )
1083 dnl VMLS(name_suffix, size)
1084 define(`VMLS', `
1085 // dst_src1 = dst_src1 - src2 * src3
1086 instruct vmls$1(vReg dst_src1, vReg src2, vReg src3)
1087 %{
1088 predicate(UseSVE > 0);
1089 match(Set dst_src1 (SubV$1 dst_src1 (MulV$1 src2 src3)));
1090 ins_cost(SVE_COST);
1091 format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) ($2)" %}
1092 ins_encode %{
1093 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ $2,
1094 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1095 %}
1096 ins_pipe(pipe_slow);
1097 %}')dnl
1098 dnl
1099 // vector mls
1100 VMLS(B, B)
1101 VMLS(S, H)
1102 VMLS(I, S)
1103 VMLS(L, D)
1104
1105 dnl
1106 dnl VMLS_PREDICATE($1 $2 )
1107 dnl VMLS_PREDICATE(name_suffix, size)
1108 define(`VMLS_PREDICATE', `
1109 // dst_src1 = dst_src1 - src2 * src3
1110 instruct vmls$1_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
1111 %{
1112 predicate(UseSVE > 0);
1113 match(Set dst_src1 (SubV$1 (Binary dst_src1 (MulV$1 src2 src3)) pg));
1114 ins_cost(SVE_COST);
1115 format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) ($2)" %}
1116 ins_encode %{
1117 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ $2, as_PRegister($pg$$reg),
1118 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1119 %}
1120 ins_pipe(pipe_slow);
1121 %}')dnl
1122 dnl
1123 // vector mls - predicated
1124 VMLS_PREDICATE(B, B)
1125 VMLS_PREDICATE(S, H)
1126 VMLS_PREDICATE(I, S)
1127 VMLS_PREDICATE(L, D)
1128
1129 dnl
1130 dnl BINARY_OP_TRUE_PREDICATE($1, $2, $3, $4, $5 )
1131 dnl BINARY_OP_TRUE_PREDICATE(insn_name, op_name, size, min_vec_len, insn)
1132 define(`BINARY_OP_TRUE_PREDICATE', `
1133 instruct $1(vReg dst_src1, vReg src2) %{
1134 predicate(UseSVE > 0);
1135 match(Set dst_src1 ($2 dst_src1 src2));
1136 ins_cost(SVE_COST);
1137 format %{ "$5 $dst_src1, $dst_src1, $src2\t # vector (sve) ($3)" %}
1138 ins_encode %{
1139 __ $5(as_FloatRegister($dst_src1$$reg), __ $3,
1140 ptrue, as_FloatRegister($src2$$reg));
1141 %}
1142 ins_pipe(pipe_slow);
1143 %}')dnl
1144 dnl
1145 // vector mul
1146 BINARY_OP_TRUE_PREDICATE(vmulB, MulVB, B, 16, sve_mul)
1147 BINARY_OP_TRUE_PREDICATE(vmulS, MulVS, H, 8, sve_mul)
1148 BINARY_OP_TRUE_PREDICATE(vmulI, MulVI, S, 4, sve_mul)
1149 BINARY_OP_TRUE_PREDICATE(vmulL, MulVL, D, 2, sve_mul)
1150 BINARY_OP_UNPREDICATE(vmulF, MulVF, S, 4, sve_fmul)
1151 BINARY_OP_UNPREDICATE(vmulD, MulVD, D, 2, sve_fmul)
1152
1153 // vector mul - predicated
1154 BINARY_OP_PREDICATE(vmulB, MulVB, B, sve_mul)
1155 BINARY_OP_PREDICATE(vmulS, MulVS, H, sve_mul)
1156 BINARY_OP_PREDICATE(vmulI, MulVI, S, sve_mul)
1157 BINARY_OP_PREDICATE(vmulL, MulVL, D, sve_mul)
1158 BINARY_OP_PREDICATE(vmulF, MulVF, S, sve_fmul)
1159 BINARY_OP_PREDICATE(vmulD, MulVD, D, sve_fmul)
1160
1161 // vector neg
1162 UNARY_OP_TRUE_PREDICATE(vnegI, NegVI, B/H/S, sve_neg)
1163 UNARY_OP_TRUE_PREDICATE(vnegL, NegVL, D, sve_neg)
1164 UNARY_OP_TRUE_PREDICATE(vnegF, NegVF, S, sve_fneg)
1165 UNARY_OP_TRUE_PREDICATE(vnegD, NegVD, D, sve_fneg)
1166
1167 // vector neg - predicated
1168 UNARY_OP_PREDICATE(vnegI, NegVI, B/H/S, sve_neg)
1169 UNARY_OP_PREDICATE(vnegL, NegVL, D, sve_neg)
1170 UNARY_OP_PREDICATE(vnegF, NegVF, S, sve_fneg)
1171 UNARY_OP_PREDICATE(vnegD, NegVD, D, sve_fneg)
1172
1173 // popcount vector
1174
1175 instruct vpopcountI(vReg dst, vReg src) %{
1176 predicate(UseSVE > 0);
1177 match(Set dst (PopCountVI src));
1178 format %{ "sve_cnt $dst, $src\t# vector (sve) (S)\n\t" %}
1179 ins_encode %{
1180 __ sve_cnt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg));
1181 %}
1182 ins_pipe(pipe_slow);
1183 %}
1184
1185 // vector blend
1186
1187 instruct vblend(vReg dst, vReg src1, vReg src2, pRegGov pg) %{
1188 predicate(UseSVE > 0);
1189 match(Set dst (VectorBlend (Binary src1 src2) pg));
1190 ins_cost(SVE_COST);
1191 format %{ "sve_sel $dst, $pg, $src2, $src1\t# vector blend (sve)" %}
1192 ins_encode %{
1193 Assembler::SIMD_RegVariant size =
1194 __ elemType_to_regVariant(Matcher::vector_element_basic_type(this));
1195 __ sve_sel(as_FloatRegister($dst$$reg), size, as_PRegister($pg$$reg),
1196 as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
1197 %}
1198 ins_pipe(pipe_slow);
1199 %}
1200
1201 // vector store mask
1202
1203 instruct vstoremaskB(vReg dst, pRegGov src, immI_1 size) %{
1204 predicate(UseSVE > 0);
1205 match(Set dst (VectorStoreMask src size));
1206 ins_cost(SVE_COST);
1207 format %{ "vstoremask $dst, $src\t# vector store mask (sve) (B)" %}
1208 ins_encode %{
1209 __ sve_cpy(as_FloatRegister($dst$$reg), __ B, as_PRegister($src$$reg), 1, false);
1210 %}
1211 ins_pipe(pipe_slow);
1212 %}
1213
1214 instruct vstoremask_narrow(vReg dst, pRegGov src, vReg tmp, immI_gt_1 size) %{
1215 predicate(UseSVE > 0);
1216 match(Set dst (VectorStoreMask src size));
1217 effect(TEMP_DEF dst, TEMP tmp);
1218 ins_cost(3 * SVE_COST);
1219 format %{ "vstoremask $dst, $src\t# vector store mask (sve) (H/S/D)" %}
1220 ins_encode %{
1221 Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant((int)$size$$constant);
1222 __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($src$$reg), 1, false);
1223 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
1224 as_FloatRegister($dst$$reg), size, as_FloatRegister($tmp$$reg));
1225 %}
1226 ins_pipe(pipe_slow);
1227 %}
1228
1229 // Combine LoadVector+VectorLoadMask when the vector element type is not T_BYTE
1230
1231 instruct vloadmask_loadV(pRegGov dst, indirect mem, vReg tmp, rFlagsReg cr) %{
1232 predicate(UseSVE > 0 &&
1233 n->as_Vector()->length_in_bytes() == MaxVectorSize &&
1234 type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
1235 match(Set dst (VectorLoadMask (LoadVector mem)));
1236 effect(TEMP tmp, KILL cr);
1237 ins_cost(3 * SVE_COST);
1238 format %{ "sve_ld1b $tmp, $mem\n\t"
1239 "sve_cmpne $dst, $tmp, 0\t# load vector mask (sve) (H/S/D)" %}
1240 ins_encode %{
1241 // Load mask values which are boolean type, and extend them to the
1242 // expected vector element type. Convert the vector to predicate.
1243 BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
1244 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($tmp$$reg),
1245 ptrue, T_BOOLEAN, to_vect_bt, $mem->opcode(),
1246 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1247 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ elemType_to_regVariant(to_vect_bt),
1248 ptrue, as_FloatRegister($tmp$$reg), 0);
1249 %}
1250 ins_pipe(pipe_slow);
1251 %}
1252
1253 instruct vloadmask_loadV_partial(pRegGov dst, indirect mem, vReg vtmp, pRegGov ptmp, rFlagsReg cr) %{
1254 predicate(UseSVE > 0 &&
1255 n->as_Vector()->length_in_bytes() > 16 &&
1256 n->as_Vector()->length_in_bytes() < MaxVectorSize &&
1257 type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
1258 match(Set dst (VectorLoadMask (LoadVector mem)));
1259 effect(TEMP vtmp, TEMP ptmp, KILL cr);
1260 ins_cost(6 * SVE_COST);
1261 format %{ "vloadmask_loadV $dst, $mem\t# load vector mask partial (sve) (H/S/D)" %}
1262 ins_encode %{
1263 // Load valid mask values which are boolean type, and extend them to the
1264 // expected vector element type. Convert the vector to predicate.
1265 BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
1266 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(to_vect_bt);
1267 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this));
1268 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($vtmp$$reg),
1269 as_PRegister($ptmp$$reg), T_BOOLEAN, to_vect_bt, $mem->opcode(),
1270 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1271 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($vtmp$$reg), 0);
1272 %}
1273 ins_pipe(pipe_slow);
1274 %}
1275
1276 // Combine VectorStoreMask+StoreVector when the vector element type is not T_BYTE
1277
1278 instruct storeV_vstoremask(indirect mem, pRegGov src, vReg tmp, immI_gt_1 esize) %{
1279 predicate(UseSVE > 0 &&
1280 Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
1281 match(Set mem (StoreVector mem (VectorStoreMask src esize)));
1282 effect(TEMP tmp);
1283 ins_cost(3 * SVE_COST);
1284 format %{ "sve_cpy $tmp, $src, 1\n\t"
1285 "sve_st1b $tmp, $mem\t# store vector mask (sve) (H/S/D)" %}
1286 ins_encode %{
1287 BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
1288 assert(type2aelembytes(from_vect_bt) == (int)$esize$$constant, "unsupported type.");
1289 Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
1290 __ sve_cpy(as_FloatRegister($tmp$$reg), size, as_PRegister($src$$reg), 1, false);
1291 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($tmp$$reg),
1292 ptrue, T_BOOLEAN, from_vect_bt, $mem->opcode(),
1293 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1294 %}
1295 ins_pipe(pipe_slow);
1296 %}
1297
1298 instruct storeV_vstoremask_partial(indirect mem, pRegGov src, vReg vtmp,
1299 immI_gt_1 esize, pRegGov ptmp, rFlagsReg cr) %{
1300 predicate(UseSVE > 0 &&
1301 n->as_StoreVector()->memory_size() > 16 &&
1302 type2aelembytes(n->as_StoreVector()->vect_type()->element_basic_type()) > 1 &&
1303 Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
1304 match(Set mem (StoreVector mem (VectorStoreMask src esize)));
1305 effect(TEMP vtmp, TEMP ptmp, KILL cr);
1306 format %{ "storeV_vstoremask $src, $mem\t# store vector mask partial (sve) (H/S/D)" %}
1307 ins_cost(6 * SVE_COST);
1308 ins_encode %{
1309 // Convert the valid src predicate to vector, and store the vector
1310 // elements as boolean values.
1311 BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
1312 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(from_vect_bt);
1313 __ sve_cpy(as_FloatRegister($vtmp$$reg), size, as_PRegister($src$$reg), 1, false);
1314 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
1315 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($vtmp$$reg),
1316 as_PRegister($ptmp$$reg), T_BOOLEAN, from_vect_bt, $mem->opcode(),
1317 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1318 %}
1319 ins_pipe(pipe_slow);
1320 %}
1321 dnl
1322 dnl REDUCE_I($1, $2 )
1323 dnl REDUCE_I(insn_name, op_name)
1324 define(`REDUCE_I', `
1325 instruct reduce_$1I(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
1326 ifelse($2, AddReductionVI,
1327 `predicate(UseSVE > 0 &&
1328 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);',
1329 `predicate(UseSVE > 0 &&
1330 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1331 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);')
1332 match(Set dst ($2 src1 src2));
1333 effect(TEMP_DEF dst, TEMP tmp);
1334 ins_cost(SVE_COST);
1335 format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction (sve) (may extend)" %}
1336 ins_encode %{
1337 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1338 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1339 $src1$$Register, as_FloatRegister($src2$$reg),
1340 ptrue, as_FloatRegister($tmp$$reg));
1341 %}
1342 ins_pipe(pipe_slow);
1343 %}')dnl
1344 dnl
1345 dnl
1346 dnl REDUCE_L($1, $2 )
1347 dnl REDUCE_L(insn_name, op_name)
1348 define(`REDUCE_L', `
1349 instruct reduce_$1L(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
1350 ifelse($2, AddReductionVL,
1351 `predicate(UseSVE > 0 &&
1352 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);',
1353 `predicate(UseSVE > 0 &&
1354 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1355 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);')
1356 match(Set dst ($2 src1 src2));
1357 effect(TEMP_DEF dst, TEMP tmp);
1358 ins_cost(SVE_COST);
1359 format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction (sve)" %}
1360 ins_encode %{
1361 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1362 $src1$$Register, as_FloatRegister($src2$$reg),
1363 ptrue, as_FloatRegister($tmp$$reg));
1364 %}
1365 ins_pipe(pipe_slow);
1366 %}')dnl
1367 dnl
1368 dnl REDUCE_I_PARTIAL($1, $2 )
1369 dnl REDUCE_I_PARTIAL(insn_name, op_name)
1370 define(`REDUCE_I_PARTIAL', `
1371 instruct reduce_$1I_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1372 pRegGov ptmp, rFlagsReg cr) %{
1373 ifelse($2, AddReductionVI,
1374 `predicate(UseSVE > 0 &&
1375 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);',
1376 `predicate(UseSVE > 0 &&
1377 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1378 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);')
1379 match(Set dst ($2 src1 src2));
1380 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1381 ins_cost(2 * SVE_COST);
1382 format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction partial (sve) (may extend)" %}
1383 ins_encode %{
1384 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1385 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1386 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
1387 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1388 $src1$$Register, as_FloatRegister($src2$$reg),
1389 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1390 %}
1391 ins_pipe(pipe_slow);
1392 %}')dnl
1393 dnl
1394 dnl REDUCE_L_PARTIAL($1, $2 )
1395 dnl REDUCE_L_PARTIAL(insn_name, op_name)
1396 define(`REDUCE_L_PARTIAL', `
1397 instruct reduce_$1L_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1398 pRegGov ptmp, rFlagsReg cr) %{
1399 ifelse($2, AddReductionVL,
1400 `predicate(UseSVE > 0 &&
1401 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);',
1402 `predicate(UseSVE > 0 &&
1403 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1404 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);')
1405 match(Set dst ($2 src1 src2));
1406 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1407 ins_cost(2 * SVE_COST);
1408 format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction partial (sve)" %}
1409 ins_encode %{
1410 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
1411 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1412 $src1$$Register, as_FloatRegister($src2$$reg),
1413 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1414 %}
1415 ins_pipe(pipe_slow);
1416 %}')dnl
1417 dnl
1418 dnl REDUCE_ADDF($1, $2, $3, $4 )
1419 dnl REDUCE_ADDF(insn_name, op_name, reg_dst, size)
1420 define(`REDUCE_ADDF', `
1421 instruct reduce_$1($3 src1_dst, vReg src2) %{
1422 predicate(UseSVE > 0 &&
1423 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1424 match(Set src1_dst ($2 src1_dst src2));
1425 ins_cost(SVE_COST);
1426 format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) ($4)" %}
1427 ins_encode %{
1428 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1429 ptrue, as_FloatRegister($src2$$reg));
1430 %}
1431 ins_pipe(pipe_slow);
1432 %}')dnl
1433 dnl
1434 dnl
1435 dnl REDUCE_ADDF_PARTIAL($1, $2, $3, $4 )
1436 dnl REDUCE_ADDF_PARTIAL(insn_name, suffix, reg_dst, size)
1437 define(`REDUCE_ADDF_PARTIAL', `
1438 instruct reduce_$1_partial($3 src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
1439 predicate(UseSVE > 0 &&
1440 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1441 match(Set src1_dst ($2 src1_dst src2));
1442 ins_cost(SVE_COST);
1443 effect(TEMP ptmp, KILL cr);
1444 format %{ "sve_reduce_$1 $src1_dst, $src1_dst, $src2\t# $1 reduction partial (sve) ($4)" %}
1445 ins_encode %{
1446 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ $4, Matcher::vector_length(this, $src2));
1447 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1448 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1449 %}
1450 ins_pipe(pipe_slow);
1451 %}')dnl
1452 dnl
1453 dnl
1454 dnl REDUCE_I_PREDICATE($1, $2 )
1455 dnl REDUCE_I_PREDICATE(insn_name, op_name)
1456 define(`REDUCE_I_PREDICATE', `
1457 instruct reduce_$1I_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
1458 ifelse($2, AddReductionVI,
1459 `predicate(UseSVE > 0);',
1460 `predicate(UseSVE > 0 &&
1461 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG);')
1462 match(Set dst ($2 (Binary src1 src2) pg));
1463 effect(TEMP_DEF dst, TEMP tmp);
1464 ins_cost(SVE_COST);
1465 format %{ "sve_reduce_$1I $dst, $src1, $pg, $src2\t# $1I reduction predicated (sve) (may extend)" %}
1466 ins_encode %{
1467 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1468 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1469 $src1$$Register, as_FloatRegister($src2$$reg),
1470 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
1471 %}
1472 ins_pipe(pipe_slow);
1473 %}')dnl
1474 dnl
1475 dnl REDUCE_L_PREDICATE($1, $2 )
1476 dnl REDUCE_L_PREDICATE(insn_name, op_name)
1477 define(`REDUCE_L_PREDICATE', `
1478 instruct reduce_$1L_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
1479 ifelse($2, AddReductionVL,
1480 `predicate(UseSVE > 0);',
1481 `predicate(UseSVE > 0 &&
1482 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);')
1483 match(Set dst ($2 (Binary src1 src2) pg));
1484 effect(TEMP_DEF dst, TEMP tmp);
1485 ins_cost(SVE_COST);
1486 format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated (sve)" %}
1487 ins_encode %{
1488 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1489 $src1$$Register, as_FloatRegister($src2$$reg),
1490 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
1491 %}
1492 ins_pipe(pipe_slow);
1493 %}')dnl
1494 dnl
1495 dnl REDUCE_ADDF_PREDICATE($1, $2, $3, $4 )
1496 dnl REDUCE_ADDF_PREDICATE(insn_name, op_name, reg_dst, size)
1497 define(`REDUCE_ADDF_PREDICATE', `
1498 instruct reduce_$1_masked($3 src1_dst, vReg src2, pRegGov pg) %{
1499 predicate(UseSVE > 0);
1500 match(Set src1_dst ($2 (Binary src1_dst src2) pg));
1501 ins_cost(SVE_COST);
1502 format %{ "sve_reduce_$1 $src1_dst, $pg, $src2\t# $1 reduction predicated (sve)" %}
1503 ins_encode %{
1504 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1505 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1506 %}
1507 ins_pipe(pipe_slow);
1508 %}')dnl
1509 dnl
1510
1511 // vector add reduction
1512 REDUCE_I(add, AddReductionVI)
1513 REDUCE_L(add, AddReductionVL)
1514 REDUCE_ADDF(addF, AddReductionVF, vRegF, S)
1515 REDUCE_ADDF(addD, AddReductionVD, vRegD, D)
1516 REDUCE_I_PARTIAL(add, AddReductionVI)
1517 REDUCE_L_PARTIAL(add, AddReductionVL)
1518 REDUCE_ADDF_PARTIAL(addF, AddReductionVF, vRegF, S)
1519 REDUCE_ADDF_PARTIAL(addD, AddReductionVD, vRegD, D)
1520
1521 // vector add reduction - predicated
1522 REDUCE_I_PREDICATE(add, AddReductionVI)
1523 REDUCE_L_PREDICATE(add, AddReductionVL)
1524 REDUCE_ADDF_PREDICATE(addF, AddReductionVF, vRegF, S)
1525 REDUCE_ADDF_PREDICATE(addD, AddReductionVD, vRegD, D)
1526
1527 // vector and reduction
1528 REDUCE_I(and, AndReductionV)
1529 REDUCE_L(and, AndReductionV)
1530 REDUCE_I_PARTIAL(and, AndReductionV)
1531 REDUCE_L_PARTIAL(and, AndReductionV)
1532
1533 // vector and reduction - predicated
1534 REDUCE_I_PREDICATE(and, AndReductionV)
1535 REDUCE_L_PREDICATE(and, AndReductionV)
1536
1537 // vector or reduction
1538 REDUCE_I(or, OrReductionV)
1539 REDUCE_L(or, OrReductionV)
1540 REDUCE_I_PARTIAL(or, OrReductionV)
1541 REDUCE_L_PARTIAL(or, OrReductionV)
1542
1543 // vector or reduction - predicated
1544 REDUCE_I_PREDICATE(or, OrReductionV)
1545 REDUCE_L_PREDICATE(or, OrReductionV)
1546
1547 // vector xor reduction
1548 REDUCE_I(eor, XorReductionV)
1549 REDUCE_L(eor, XorReductionV)
1550 REDUCE_I_PARTIAL(eor, XorReductionV)
1551 REDUCE_L_PARTIAL(eor, XorReductionV)
1552
1553 // vector xor reduction - predicated
1554 REDUCE_I_PREDICATE(eor, XorReductionV)
1555 REDUCE_L_PREDICATE(eor, XorReductionV)
1556
1557 dnl
1558 dnl REDUCE_MAXMIN_I($1, $2 )
1559 dnl REDUCE_MAXMIN_I(insn_name, op_name)
1560 define(`REDUCE_MAXMIN_I', `
1561 instruct reduce_$1I(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
1562 predicate(UseSVE > 0 &&
1563 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1564 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1565 is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
1566 match(Set dst ($2 src1 src2));
1567 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1568 ins_cost(SVE_COST);
1569 format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction (sve)" %}
1570 ins_encode %{
1571 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1572 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1573 $src1$$Register, as_FloatRegister($src2$$reg),
1574 ptrue, as_FloatRegister($tmp$$reg));
1575 %}
1576 ins_pipe(pipe_slow);
1577 %}')dnl
1578 dnl
1579 dnl REDUCE_MAXMIN_L($1, $2 )
1580 dnl REDUCE_MAXMIN_L(insn_name, op_name)
1581 define(`REDUCE_MAXMIN_L', `
1582 instruct reduce_$1L(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
1583 predicate(UseSVE > 0 &&
1584 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1585 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1586 match(Set dst ($2 src1 src2));
1587 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1588 ins_cost(SVE_COST);
1589 format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction (sve)" %}
1590 ins_encode %{
1591 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1592 $src1$$Register, as_FloatRegister($src2$$reg),
1593 ptrue, as_FloatRegister($tmp$$reg));
1594 %}
1595 ins_pipe(pipe_slow);
1596 %}')dnl
1597 dnl
1598 dnl REDUCE_MAXMIN_I_PARTIAL($1 , $2 )
1599 dnl REDUCE_MAXMIN_I_PARTIAL(min_max, op_name)
1600 define(`REDUCE_MAXMIN_I_PARTIAL', `
1601 instruct reduce_$1I_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1602 pRegGov ptmp, rFlagsReg cr) %{
1603 predicate(UseSVE > 0 &&
1604 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1605 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1606 is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
1607 match(Set dst ($2 src1 src2));
1608 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1609 ins_cost(2 * SVE_COST);
1610 format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction partial (sve)" %}
1611 ins_encode %{
1612 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1613 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1614 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
1615 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1616 $src1$$Register, as_FloatRegister($src2$$reg),
1617 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1618 %}
1619 ins_pipe(pipe_slow);
1620 %}')dnl
1621 dnl
1622 dnl REDUCE_MAXMIN_L_PARTIAL($1 , $2 )
1623 dnl REDUCE_MAXMIN_L_PARTIAL(min_max, op_name)
1624 define(`REDUCE_MAXMIN_L_PARTIAL', `
1625 instruct reduce_$1L_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1626 pRegGov ptmp, rFlagsReg cr) %{
1627 predicate(UseSVE > 0 &&
1628 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1629 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1630 match(Set dst ($2 src1 src2));
1631 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1632 ins_cost(2 * SVE_COST);
1633 format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction partial (sve)" %}
1634 ins_encode %{
1635 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
1636 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1637 $src1$$Register, as_FloatRegister($src2$$reg),
1638 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1639 %}
1640 ins_pipe(pipe_slow);
1641 %}')dnl
1642 dnl
1643 dnl REDUCE_MAXMIN_I_PREDICATE($1 , $2 )
1644 dnl REDUCE_MAXMIN_I_PREDICATE(min_max, op_name)
1645 define(`REDUCE_MAXMIN_I_PREDICATE', `
1646 instruct reduce_$1I_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp,
1647 pRegGov pg, rFlagsReg cr) %{
1648 predicate(UseSVE > 0 &&
1649 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1650 is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
1651 match(Set dst ($2 (Binary src1 src2) pg));
1652 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1653 ins_cost(SVE_COST);
1654 format %{ "sve_reduce_$1I $dst, $src1, $pg, $src2\t# $1I reduction predicated (sve)" %}
1655 ins_encode %{
1656 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1657 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1658 $src1$$Register, as_FloatRegister($src2$$reg),
1659 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
1660 %}
1661 ins_pipe(pipe_slow);
1662 %}')dnl
1663 dnl
1664 dnl REDUCE_MAXMIN_L_PREDICATE($1 , $2 )
1665 dnl REDUCE_MAXMIN_L_PREDICATE(min_max, op_name)
1666 define(`REDUCE_MAXMIN_L_PREDICATE', `
1667 instruct reduce_$1L_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp,
1668 pRegGov pg, rFlagsReg cr) %{
1669 predicate(UseSVE > 0 &&
1670 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1671 match(Set dst ($2 (Binary src1 src2) pg));
1672 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1673 ins_cost(SVE_COST);
1674 format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated (sve)" %}
1675 ins_encode %{
1676 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1677 $src1$$Register, as_FloatRegister($src2$$reg),
1678 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
1679 %}
1680 ins_pipe(pipe_slow);
1681 %}')dnl
1682 dnl
1683 dnl REDUCE_FMINMAX($1, $2, $3, $4, $5 )
1684 dnl REDUCE_FMINMAX(min_max, name_suffix, element_type, size, reg_src_dst)
1685 define(`REDUCE_FMINMAX', `
1686 instruct reduce_$1$2($5 dst, $5 src1, vReg src2) %{
1687 predicate(UseSVE > 0 &&
1688 n->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1689 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1690 match(Set dst (translit($1, `m', `M')ReductionV src1 src2));
1691 ins_cost(INSN_COST);
1692 effect(TEMP_DEF dst);
1693 format %{ "sve_reduce_$1$2 $dst, $src1, $src2\t# $1$2 reduction (sve)" %}
1694 ins_encode %{
1695 __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src2$$reg));
1696 __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1697 %}
1698 ins_pipe(pipe_slow);
1699 %}')dnl
1700 dnl
1701 dnl REDUCE_FMINMAX_PARTIAL($1, $2, $3, $4, $5 )
1702 dnl REDUCE_FMINMAX_PARTIAL(min_max, name_suffix, element_type, size, reg_src_dst)
1703 define(`REDUCE_FMINMAX_PARTIAL', `
1704 instruct reduce_$1$2_partial($5 dst, $5 src1, vReg src2,
1705 pRegGov ptmp, rFlagsReg cr) %{
1706 predicate(UseSVE > 0 &&
1707 n->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1708 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1709 match(Set dst (translit($1, `m', `M')ReductionV src1 src2));
1710 ins_cost(INSN_COST);
1711 effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
1712 format %{ "sve_reduce_$1$2 $dst, $src1, $src2\t# $1$2 reduction partial (sve)" %}
1713 ins_encode %{
1714 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ $4, Matcher::vector_length(this, $src2));
1715 __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1716 __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1717 %}
1718 ins_pipe(pipe_slow);
1719 %}')dnl
1720 dnl
1721 dnl REDUCE_FMINMAX_PREDICATE($1, $2, $3, $4, $5 )
1722 dnl REDUCE_FMINMAX_PREDICATE(min_max, name_suffix, element_type, size, reg_src_dst)
1723 define(`REDUCE_FMINMAX_PREDICATE', `
1724 instruct reduce_$1$2_masked($5 dst, $5 src1, vReg src2, pRegGov pg) %{
1725 predicate(UseSVE > 0 &&
1726 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == $3);
1727 match(Set dst (translit($1, `m', `M')ReductionV (Binary src1 src2) pg));
1728 ins_cost(SVE_COST);
1729 effect(TEMP_DEF dst);
1730 format %{ "sve_reduce_$1$2 $dst, $src1, $pg, $src2\t# $1$2 reduction predicated (sve)" %}
1731 ins_encode %{
1732 __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1733 __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1734 %}
1735 ins_pipe(pipe_slow);
1736 %}')dnl
1737 dnl
1738 // vector max reduction
1739 REDUCE_MAXMIN_I(max, MaxReductionV)
1740 REDUCE_MAXMIN_L(max, MaxReductionV)
1741 REDUCE_MAXMIN_I_PARTIAL(max, MaxReductionV)
1742 REDUCE_MAXMIN_L_PARTIAL(max, MaxReductionV)
1743 REDUCE_FMINMAX(max, F, T_FLOAT, S, vRegF)
1744 REDUCE_FMINMAX_PARTIAL(max, F, T_FLOAT, S, vRegF)
1745 REDUCE_FMINMAX(max, D, T_DOUBLE, D, vRegD)
1746 REDUCE_FMINMAX_PARTIAL(max, D, T_DOUBLE, D, vRegD)
1747
1748 // vector max reduction - predicated
1749 REDUCE_MAXMIN_I_PREDICATE(max, MaxReductionV)
1750 REDUCE_MAXMIN_L_PREDICATE(max, MaxReductionV)
1751 REDUCE_FMINMAX_PREDICATE(max, F, T_FLOAT, S, vRegF)
1752 REDUCE_FMINMAX_PREDICATE(max, D, T_DOUBLE, D, vRegD)
1753
1754 // vector min reduction
1755 REDUCE_MAXMIN_I(min, MinReductionV)
1756 REDUCE_MAXMIN_L(min, MinReductionV)
1757 REDUCE_MAXMIN_I_PARTIAL(min, MinReductionV)
1758 REDUCE_MAXMIN_L_PARTIAL(min, MinReductionV)
1759 REDUCE_FMINMAX(min, F, T_FLOAT, S, vRegF)
1760 REDUCE_FMINMAX_PARTIAL(min, F, T_FLOAT, S, vRegF)
1761 REDUCE_FMINMAX(min, D, T_DOUBLE, D, vRegD)
1762 REDUCE_FMINMAX_PARTIAL(min, D, T_DOUBLE, D, vRegD)
1763
1764 // vector min reduction - predicated
1765 REDUCE_MAXMIN_I_PREDICATE(min, MinReductionV)
1766 REDUCE_MAXMIN_L_PREDICATE(min, MinReductionV)
1767 REDUCE_FMINMAX_PREDICATE(min, F, T_FLOAT, S, vRegF)
1768 REDUCE_FMINMAX_PREDICATE(min, D, T_DOUBLE, D, vRegD)
1769
1770 // vector Math.rint, floor, ceil
1771
1772 instruct vroundD(vReg dst, vReg src, immI rmode) %{
1773 predicate(UseSVE > 0 &&
1774 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
1775 match(Set dst (RoundDoubleModeV src rmode));
1776 format %{ "sve_frint $dst, $src, $rmode\t# vector (sve) (D)" %}
1777 ins_encode %{
1778 switch ($rmode$$constant) {
1779 case RoundDoubleModeNode::rmode_rint:
1780 __ sve_frintn(as_FloatRegister($dst$$reg), __ D,
1781 ptrue, as_FloatRegister($src$$reg));
1782 break;
1783 case RoundDoubleModeNode::rmode_floor:
1784 __ sve_frintm(as_FloatRegister($dst$$reg), __ D,
1785 ptrue, as_FloatRegister($src$$reg));
1786 break;
1787 case RoundDoubleModeNode::rmode_ceil:
1788 __ sve_frintp(as_FloatRegister($dst$$reg), __ D,
1789 ptrue, as_FloatRegister($src$$reg));
1790 break;
1791 }
1792 %}
1793 ins_pipe(pipe_slow);
1794 %}
1795 define(`VECTOR_JAVA_FROUND', `
1796 instruct vround$1to$3($7 dst, $7 src, $7 tmp1, $7 tmp2, $7 tmp3, pRegGov ptmp)
1797 %{
1798 predicate(UseSVE > 0);
1799 match(Set dst (RoundV$1 src));
1800 effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP ptmp);
1801 format %{ "sve_vround $dst, $4, $src\t# round $1 to $3 vector" %}
1802 ins_encode %{
1803 BasicType bt = Matcher::vector_element_basic_type(this);
1804 int vlen = Matcher::vector_length_in_bytes(this);
1805 if (vlen > 16) {
1806 __ vector_round_sve(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
1807 as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg),
1808 as_PRegister($ptmp$$reg), __ $4);
1809 } else {
1810 __ vector_round_neon(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
1811 as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg),
1812 as_FloatRegister($tmp3$$reg),
1813 __ esize2arrangement(type2aelembytes(bt),
1814 /*isQ*/ vlen == 16));
1815 }
1816 %}
1817 ins_pipe(pipe_class_default);
1818 %}')dnl $1 $2 $3 $4 $5 $6 $7
1819 VECTOR_JAVA_FROUND(F, 8F, I, S, 8, INT, vReg)
1820 VECTOR_JAVA_FROUND(D, 4D, L, D, 4, LONG, vReg)
1821 dnl
1822 dnl REPLICATE($1, $2, $3, $4, $5 )
1823 dnl REPLICATE(insn_name, op_name, reg_src, size, min_vec_len)
1824 define(`REPLICATE', `
1825 instruct $1(vReg dst, $3 src) %{
1826 predicate(UseSVE > 0);
1827 match(Set dst ($2 src));
1828 ins_cost(SVE_COST);
1829 format %{ "sve_dup $dst, $src\t# vector (sve) ($4)" %}
1830 ins_encode %{
1831 __ sve_dup(as_FloatRegister($dst$$reg), __ $4, as_Register($src$$reg));
1832 %}
1833 ins_pipe(pipe_slow);
1834 %}')dnl
1835 dnl
1836 dnl REPLICATE_IMM8($1, $2, $3, $4, $5 )
1837 dnl REPLICATE_IMM8(insn_name, op_name, imm_type, size, min_vec_len)
1838 define(`REPLICATE_IMM8', `
1839 instruct $1(vReg dst, $3 con) %{
1840 predicate(UseSVE > 0);
1841 match(Set dst ($2 con));
1842 ins_cost(SVE_COST);
1843 format %{ "sve_dup $dst, $con\t# vector (sve) ($4)" %}
1844 ins_encode %{
1845 __ sve_dup(as_FloatRegister($dst$$reg), __ $4, $con$$constant);
1846 %}
1847 ins_pipe(pipe_slow);
1848 %}')dnl
1849 dnl
1850 dnl FREPLICATE($1, $2, $3, $4)
1851 dnl FREPLICATE(insn_name, op_name, reg_src, size)
1852 define(`FREPLICATE', `
1853 instruct $1(vReg dst, $3 src) %{
1854 predicate(UseSVE > 0);
1855 match(Set dst ($2 src));
1856 ins_cost(SVE_COST);
1857 format %{ "sve_cpy $dst, $src\t# vector (sve) ($4)" %}
1858 ins_encode %{
1859 __ sve_cpy(as_FloatRegister($dst$$reg), __ $4,
1860 ptrue, as_FloatRegister($src$$reg));
1861 %}
1862 ins_pipe(pipe_slow);
1863 %}')dnl
1864
1865 // vector replicate
1866 REPLICATE(replicateB, ReplicateB, iRegIorL2I, B, 16)
1867 REPLICATE(replicateS, ReplicateS, iRegIorL2I, H, 8)
1868 REPLICATE(replicateI, ReplicateI, iRegIorL2I, S, 4)
1869 REPLICATE(replicateL, ReplicateL, iRegL, D, 2)
1870 REPLICATE_IMM8(replicateB_imm8, ReplicateB, immI8, B, 16)
1871 REPLICATE_IMM8(replicateS_imm8, ReplicateS, immI8_shift8, H, 8)
1872 REPLICATE_IMM8(replicateI_imm8, ReplicateI, immI8_shift8, S, 4)
1873 REPLICATE_IMM8(replicateL_imm8, ReplicateL, immL8_shift8, D, 2)
1874 FREPLICATE(replicateF, ReplicateF, vRegF, S, 4)
1875 FREPLICATE(replicateD, ReplicateD, vRegD, D, 2)
1876 dnl
1877 dnl VSHIFT_TRUE_PREDICATE($1, $2, $3, $4, $5 )
1878 dnl VSHIFT_TRUE_PREDICATE(insn_name, op_name, size, min_vec_len, insn)
1879 define(`VSHIFT_TRUE_PREDICATE', `
1880 instruct $1(vReg dst, vReg shift) %{
1881 predicate(UseSVE > 0);
1882 match(Set dst ($2 dst shift));
1883 ins_cost(SVE_COST);
1884 format %{ "$5 $dst, $dst, $shift\t# vector (sve) ($3)" %}
1885 ins_encode %{
1886 __ $5(as_FloatRegister($dst$$reg), __ $3,
1887 ptrue, as_FloatRegister($shift$$reg));
1888 %}
1889 ins_pipe(pipe_slow);
1890 %}')dnl
1891 dnl
1892 dnl VSHIFT_IMM_UNPREDICATED($1, $2, $3, $4, $5, $6 )
1893 dnl VSHIFT_IMM_UNPREDICATED(insn_name, op_name, op_name2, size, min_vec_len, insn)
1894 define(`VSHIFT_IMM_UNPREDICATED', `
1895 instruct $1(vReg dst, vReg src, immI shift) %{
1896 predicate(UseSVE > 0);
1897 match(Set dst ($2 src ($3 shift)));
1898 ins_cost(SVE_COST);
1899 format %{ "$6 $dst, $src, $shift\t# vector (sve) ($4)" %}
1900 ins_encode %{
1901 int con = (int)$shift$$constant;dnl
1902 ifelse(eval(index(`$1', `vasr') == 0 || index(`$1', `vlsr') == 0), 1, `
1903 if (con == 0) {
1904 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
1905 as_FloatRegister($src$$reg));
1906 return;
1907 }')dnl
1908 ifelse(eval(index(`$1', `vasr') == 0), 1, `ifelse(eval(index(`$4', `B') == 0), 1, `
1909 if (con >= 8) con = 7;')ifelse(eval(index(`$4', `H') == 0), 1, `
1910 if (con >= 16) con = 15;')')dnl
1911 ifelse(eval(index(`$1', `vlsl') == 0 || index(`$1', `vlsr') == 0), 1, `ifelse(eval(index(`$4', `B') == 0), 1, `
1912 if (con >= 8) {
1913 __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
1914 as_FloatRegister($src$$reg));
1915 return;
1916 }')ifelse(eval(index(`$4', `H') == 0), 1, `
1917 if (con >= 16) {
1918 __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
1919 as_FloatRegister($src$$reg));
1920 return;
1921 }')')
1922 __ $6(as_FloatRegister($dst$$reg), __ $4,
1923 as_FloatRegister($src$$reg), con);
1924 %}
1925 ins_pipe(pipe_slow);
1926 %}')dnl
1927 dnl
1928 dnl VSHIFT_COUNT($1, $2, $3, $4 )
1929 dnl VSHIFT_COUNT(insn_name, size, min_vec_len, type)
1930 define(`VSHIFT_COUNT', `
1931 instruct $1(vReg dst, iRegIorL2I cnt) %{
1932 predicate(UseSVE > 0 &&
1933 ELEMENT_SHORT_CHAR($4, n));
1934 match(Set dst (LShiftCntV cnt));
1935 match(Set dst (RShiftCntV cnt));
1936 format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) ($2)" %}
1937 ins_encode %{
1938 __ sve_dup(as_FloatRegister($dst$$reg), __ $2, as_Register($cnt$$reg));
1939 %}
1940 ins_pipe(pipe_slow);
1941 %}')dnl
1942
1943 // vector shift
1944 VSHIFT_TRUE_PREDICATE(vasrB, RShiftVB, B, 16, sve_asr)
1945 VSHIFT_TRUE_PREDICATE(vasrS, RShiftVS, H, 8, sve_asr)
1946 VSHIFT_TRUE_PREDICATE(vasrI, RShiftVI, S, 4, sve_asr)
1947 VSHIFT_TRUE_PREDICATE(vasrL, RShiftVL, D, 2, sve_asr)
1948 VSHIFT_TRUE_PREDICATE(vlslB, LShiftVB, B, 16, sve_lsl)
1949 VSHIFT_TRUE_PREDICATE(vlslS, LShiftVS, H, 8, sve_lsl)
1950 VSHIFT_TRUE_PREDICATE(vlslI, LShiftVI, S, 4, sve_lsl)
1951 VSHIFT_TRUE_PREDICATE(vlslL, LShiftVL, D, 2, sve_lsl)
1952 VSHIFT_TRUE_PREDICATE(vlsrB, URShiftVB, B, 16, sve_lsr)
1953 VSHIFT_TRUE_PREDICATE(vlsrS, URShiftVS, H, 8, sve_lsr)
1954 VSHIFT_TRUE_PREDICATE(vlsrI, URShiftVI, S, 4, sve_lsr)
1955 VSHIFT_TRUE_PREDICATE(vlsrL, URShiftVL, D, 2, sve_lsr)
1956 VSHIFT_IMM_UNPREDICATED(vasrB_imm, RShiftVB, RShiftCntV, B, 16, sve_asr)
1957 VSHIFT_IMM_UNPREDICATED(vasrS_imm, RShiftVS, RShiftCntV, H, 8, sve_asr)
1958 VSHIFT_IMM_UNPREDICATED(vasrI_imm, RShiftVI, RShiftCntV, S, 4, sve_asr)
1959 VSHIFT_IMM_UNPREDICATED(vasrL_imm, RShiftVL, RShiftCntV, D, 2, sve_asr)
1960 VSHIFT_IMM_UNPREDICATED(vlsrB_imm, URShiftVB, RShiftCntV, B, 16, sve_lsr)
1961 VSHIFT_IMM_UNPREDICATED(vlsrS_imm, URShiftVS, RShiftCntV, H, 8, sve_lsr)
1962 VSHIFT_IMM_UNPREDICATED(vlsrI_imm, URShiftVI, RShiftCntV, S, 4, sve_lsr)
1963 VSHIFT_IMM_UNPREDICATED(vlsrL_imm, URShiftVL, RShiftCntV, D, 2, sve_lsr)
1964 VSHIFT_IMM_UNPREDICATED(vlslB_imm, LShiftVB, LShiftCntV, B, 16, sve_lsl)
1965 VSHIFT_IMM_UNPREDICATED(vlslS_imm, LShiftVS, LShiftCntV, H, 8, sve_lsl)
1966 VSHIFT_IMM_UNPREDICATED(vlslI_imm, LShiftVI, LShiftCntV, S, 4, sve_lsl)
1967 VSHIFT_IMM_UNPREDICATED(vlslL_imm, LShiftVL, LShiftCntV, D, 2, sve_lsl)
1968 VSHIFT_COUNT(vshiftcntB, B, 16, T_BYTE)
1969 VSHIFT_COUNT(vshiftcntS, H, 8, T_SHORT)
1970 VSHIFT_COUNT(vshiftcntI, S, 4, T_INT)
1971 VSHIFT_COUNT(vshiftcntL, D, 2, T_LONG)
1972
1973 // vector shift - predicated
1974 BINARY_OP_PREDICATE(vasrB, RShiftVB, B, sve_asr)
1975 BINARY_OP_PREDICATE(vasrS, RShiftVS, H, sve_asr)
1976 BINARY_OP_PREDICATE(vasrI, RShiftVI, S, sve_asr)
1977 BINARY_OP_PREDICATE(vasrL, RShiftVL, D, sve_asr)
1978 BINARY_OP_PREDICATE(vlslB, LShiftVB, B, sve_lsl)
1979 BINARY_OP_PREDICATE(vlslS, LShiftVS, H, sve_lsl)
1980 BINARY_OP_PREDICATE(vlslI, LShiftVI, S, sve_lsl)
1981 BINARY_OP_PREDICATE(vlslL, LShiftVL, D, sve_lsl)
1982 BINARY_OP_PREDICATE(vlsrB, URShiftVB, B, sve_lsr)
1983 BINARY_OP_PREDICATE(vlsrS, URShiftVS, H, sve_lsr)
1984 BINARY_OP_PREDICATE(vlsrI, URShiftVI, S, sve_lsr)
1985 BINARY_OP_PREDICATE(vlsrL, URShiftVL, D, sve_lsr)
1986 dnl
1987 dnl VSHIFT_IMM_PREDICATED($1, $2, $3, $4, $5, $6 )
1988 dnl VSHIFT_IMM_PREDICATED(insn_name, op_name, op_name2, type, size, insn)
1989 define(`VSHIFT_IMM_PREDICATED', `
1990 instruct $1_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
1991 predicate(UseSVE > 0);
1992 match(Set dst_src ($2 (Binary dst_src ($3 shift)) pg));
1993 ins_cost(SVE_COST);
1994 format %{ "$6 $dst_src, $pg, $dst_src, $shift\t# vector (sve) ($4)" %}
1995 ins_encode %{
1996 int con = (int)$shift$$constant;
1997 assert(con ifelse(index(`$1', `vlsl'), 0, `>=', `>') 0 && con < $5, "invalid shift immediate");
1998 __ $6(as_FloatRegister($dst_src$$reg), __ $4, as_PRegister($pg$$reg), con);
1999 %}
2000 ins_pipe(pipe_slow);
2001 %}')dnl
2002 dnl
2003 VSHIFT_IMM_PREDICATED(vasrB, RShiftVB, RShiftCntV, B, 8, sve_asr)
2004 VSHIFT_IMM_PREDICATED(vasrS, RShiftVS, RShiftCntV, H, 16, sve_asr)
2005 VSHIFT_IMM_PREDICATED(vasrI, RShiftVI, RShiftCntV, S, 32, sve_asr)
2006 VSHIFT_IMM_PREDICATED(vasrL, RShiftVL, RShiftCntV, D, 64, sve_asr)
2007 VSHIFT_IMM_PREDICATED(vlsrB, URShiftVB, RShiftCntV, B, 8, sve_lsr)
2008 VSHIFT_IMM_PREDICATED(vlsrS, URShiftVS, RShiftCntV, H, 16, sve_lsr)
2009 VSHIFT_IMM_PREDICATED(vlsrI, URShiftVI, RShiftCntV, S, 32, sve_lsr)
2010 VSHIFT_IMM_PREDICATED(vlsrL, URShiftVL, RShiftCntV, D, 64, sve_lsr)
2011 VSHIFT_IMM_PREDICATED(vlslB, LShiftVB, LShiftCntV, B, 8, sve_lsl)
2012 VSHIFT_IMM_PREDICATED(vlslS, LShiftVS, LShiftCntV, H, 16, sve_lsl)
2013 VSHIFT_IMM_PREDICATED(vlslI, LShiftVI, LShiftCntV, S, 32, sve_lsl)
2014 VSHIFT_IMM_PREDICATED(vlslL, LShiftVL, LShiftCntV, D, 64, sve_lsl)
2015
2016 // vector sqrt
2017 UNARY_OP_TRUE_PREDICATE(vsqrtF, SqrtVF, S, sve_fsqrt)
2018 UNARY_OP_TRUE_PREDICATE(vsqrtD, SqrtVD, D, sve_fsqrt)
2019
2020 // vector sqrt - predicated
2021 UNARY_OP_PREDICATE(vsqrtF, SqrtVF, S, sve_fsqrt)
2022 UNARY_OP_PREDICATE(vsqrtD, SqrtVD, D, sve_fsqrt)
2023
2024 // vector sub
2025 BINARY_OP_UNPREDICATE(vsubB, SubVB, B, 16, sve_sub)
2026 BINARY_OP_UNPREDICATE(vsubS, SubVS, H, 8, sve_sub)
2027 BINARY_OP_UNPREDICATE(vsubI, SubVI, S, 4, sve_sub)
2028 BINARY_OP_UNPREDICATE(vsubL, SubVL, D, 2, sve_sub)
2029 BINARY_OP_UNPREDICATE(vsubF, SubVF, S, 4, sve_fsub)
2030 BINARY_OP_UNPREDICATE(vsubD, SubVD, D, 2, sve_fsub)
2031
2032 // vector sub - predicated
2033 BINARY_OP_PREDICATE(vsubB, SubVB, B, sve_sub)
2034 BINARY_OP_PREDICATE(vsubS, SubVS, H, sve_sub)
2035 BINARY_OP_PREDICATE(vsubI, SubVI, S, sve_sub)
2036 BINARY_OP_PREDICATE(vsubL, SubVL, D, sve_sub)
2037 BINARY_OP_PREDICATE(vsubF, SubVF, S, sve_fsub)
2038 BINARY_OP_PREDICATE(vsubD, SubVD, D, sve_fsub)
2039
2040 // ------------------------------ Vector mask cast --------------------------
2041
2042 instruct vmaskcast(pRegGov dst_src) %{
2043 predicate(UseSVE > 0 &&
2044 n->bottom_type()->is_vect()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
2045 n->bottom_type()->is_vect()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
2046 match(Set dst_src (VectorMaskCast dst_src));
2047 ins_cost(0);
2048 format %{ "vmaskcast $dst_src\t# empty (sve)" %}
2049 ins_encode %{
2050 // empty
2051 %}
2052 ins_pipe(pipe_class_empty);
2053 %}
2054
2055 instruct vmaskcast_extend(pRegGov dst, pReg src)
2056 %{
2057 predicate(UseSVE > 0 &&
2058 (Matcher::vector_length_in_bytes(n) == 2 * Matcher::vector_length_in_bytes(n->in(1)) ||
2059 Matcher::vector_length_in_bytes(n) == 4 * Matcher::vector_length_in_bytes(n->in(1)) ||
2060 Matcher::vector_length_in_bytes(n) == 8 * Matcher::vector_length_in_bytes(n->in(1))));
2061 match(Set dst (VectorMaskCast src));
2062 ins_cost(SVE_COST * 3);
2063 format %{ "sve_vmaskcast_extend $dst, $src\t# extend predicate $src" %}
2064 ins_encode %{
2065 __ sve_vmaskcast_extend(as_PRegister($dst$$reg), as_PRegister($src$$reg),
2066 Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
2067 %}
2068 ins_pipe(pipe_slow);
2069 %}
2070
2071 instruct vmaskcast_narrow(pRegGov dst, pReg src)
2072 %{
2073 predicate(UseSVE > 0 &&
2074 (Matcher::vector_length_in_bytes(n) * 2 == Matcher::vector_length_in_bytes(n->in(1)) ||
2075 Matcher::vector_length_in_bytes(n) * 4 == Matcher::vector_length_in_bytes(n->in(1)) ||
2076 Matcher::vector_length_in_bytes(n) * 8 == Matcher::vector_length_in_bytes(n->in(1))));
2077 match(Set dst (VectorMaskCast src));
2078 ins_cost(SVE_COST * 3);
2079 format %{ "sve_vmaskcast_narrow $dst, $src\t# narrow predicate $src" %}
2080 ins_encode %{
2081 __ sve_vmaskcast_narrow(as_PRegister($dst$$reg), as_PRegister($src$$reg),
2082 Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
2083 %}
2084 ins_pipe(pipe_slow);
2085 %}
2086 dnl
2087
2088 // ------------------------------ Vector cast -------------------------------
2089 dnl
2090 dnl
2091 define(`VECTOR_CAST_X2X', `
2092 instruct vcvt$1to$2`'(vReg dst, vReg src)
2093 %{
2094 predicate(UseSVE > 0 &&
2095 n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
2096 match(Set dst (VectorCast$1`'2X src));
2097 ins_cost(SVE_COST);
2098 format %{ "sve_vectorcast_$5 $dst, $src\t# convert $1 to $2 vector" %}
2099 ins_encode %{
2100 __ sve_$3(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src$$reg), __ $4);
2101 %}
2102 ins_pipe(pipe_slow);
2103 %}')dnl
2104
2105 dnl
2106 dnl Start of vector cast rules
2107 dnl
2108 instruct vcvtBtoX_extend(vReg dst, vReg src)
2109 %{
2110 predicate(UseSVE > 0);
2111 match(Set dst (VectorCastB2X src));
2112 ins_cost(2 * SVE_COST);
2113 format %{ "sve_vectorcast_b2x $dst, $src\t# convert B to X vector (extend)" %}
2114 ins_encode %{
2115 BasicType to_bt = Matcher::vector_element_basic_type(this);
2116 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2117 __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ B);
2118 if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
2119 __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
2120 }
2121 %}
2122 ins_pipe(pipe_slow);
2123 %}
2124
2125 instruct vcvtStoB(vReg dst, vReg src, vReg tmp)
2126 %{
2127 predicate(UseSVE > 0 &&
2128 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2129 match(Set dst (VectorCastS2X src));
2130 effect(TEMP tmp);
2131 ins_cost(2 * SVE_COST);
2132 format %{ "sve_vectorcast_s2b $dst, $src\t# convert H to B vector" %}
2133 ins_encode %{
2134 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
2135 as_FloatRegister($src$$reg), __ H, as_FloatRegister($tmp$$reg));
2136 %}
2137 ins_pipe(pipe_slow);
2138 %}
2139
2140 instruct vcvtStoX_extend(vReg dst, vReg src)
2141 %{
2142 predicate(UseSVE > 0 &&
2143 type2aelembytes(Matcher::vector_element_basic_type(n)) > 2);
2144 match(Set dst (VectorCastS2X src));
2145 ins_cost(2 * SVE_COST);
2146 format %{ "sve_vectorcast_s2x $dst, $src\t# convert H to X vector (extend)" %}
2147 ins_encode %{
2148 BasicType to_bt = Matcher::vector_element_basic_type(this);
2149 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2150 __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ H);
2151 if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
2152 __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
2153 }
2154 %}
2155 ins_pipe(pipe_slow);
2156 %}
2157
2158 instruct vcvtItoB(vReg dst, vReg src, vReg tmp)
2159 %{
2160 predicate(UseSVE > 0 &&
2161 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2162 match(Set dst (VectorCastI2X src));
2163 effect(TEMP_DEF dst, TEMP tmp);
2164 ins_cost(3 * SVE_COST);
2165 format %{ "sve_vectorcast_i2b $dst, $src\t# convert I to B vector" %}
2166 ins_encode %{
2167 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
2168 as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
2169 %}
2170 ins_pipe(pipe_slow);
2171 %}
2172
2173 instruct vcvtItoS(vReg dst, vReg src, vReg tmp)
2174 %{
2175 predicate(UseSVE > 0 &&
2176 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
2177 match(Set dst (VectorCastI2X src));
2178 effect(TEMP tmp);
2179 ins_cost(2 * SVE_COST);
2180 format %{ "sve_vectorcast_i2s $dst, $src\t# convert I to H vector" %}
2181 ins_encode %{
2182 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ H,
2183 as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
2184 %}
2185 ins_pipe(pipe_slow);
2186 %}
2187
2188 instruct vcvtItoL(vReg dst, vReg src)
2189 %{
2190 predicate(UseSVE > 0 &&
2191 n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2192 match(Set dst (VectorCastI2X src));
2193 ins_cost(SVE_COST);
2194 format %{ "sve_vectorcast_i2l $dst, $src\t# convert I to L vector" %}
2195 ins_encode %{
2196 __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
2197 %}
2198 ins_pipe(pipe_slow);
2199 %}
2200 dnl
2201 dnl vcvtItoF
2202 VECTOR_CAST_X2X(I, F, scvtf, S, i2f)
2203
2204 instruct vcvtItoD(vReg dst, vReg src)
2205 %{
2206 predicate(UseSVE > 0 &&
2207 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
2208 match(Set dst (VectorCastI2X src));
2209 ins_cost(2 * SVE_COST);
2210 format %{ "sve_vectorcast_i2d $dst, $src\t# convert I to D vector" %}
2211 ins_encode %{
2212 __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
2213 __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ D);
2214 %}
2215 ins_pipe(pipe_slow);
2216 %}
2217
2218 instruct vcvtLtoX_narrow(vReg dst, vReg src, vReg tmp)
2219 %{
2220 predicate(UseSVE > 0 && is_integral_type(Matcher::vector_element_basic_type(n)));
2221 match(Set dst (VectorCastL2X src));
2222 effect(TEMP_DEF dst, TEMP tmp);
2223 ins_cost(2 * SVE_COST);
2224 format %{ "sve_vectorcast_l2x $dst, $src\t# convert L to B/H/S vector (narrow)" %}
2225 ins_encode %{
2226 BasicType to_bt = Matcher::vector_element_basic_type(this);
2227 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2228 __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
2229 as_FloatRegister($src$$reg), __ D, as_FloatRegister($tmp$$reg));
2230 %}
2231 ins_pipe(pipe_slow);
2232 %}
2233
2234 instruct vcvtLtoF(vReg dst, vReg src, vReg tmp)
2235 %{
2236 predicate(UseSVE > 0 &&
2237 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2238 match(Set dst (VectorCastL2X src));
2239 effect(TEMP_DEF dst, TEMP tmp);
2240 ins_cost(3 * SVE_COST);
2241 format %{ "sve_vectorcast_l2f $dst, $src\t# convert L to F vector" %}
2242 ins_encode %{
2243 __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
2244 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
2245 as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
2246
2247 %}
2248 ins_pipe(pipe_slow);
2249 %}
2250 dnl
2251 dnl vcvtLtoD
2252 VECTOR_CAST_X2X(L, D, scvtf, D, l2d)
2253
2254 instruct vcvtFtoX_narrow(vReg dst, vReg src, vReg tmp)
2255 %{
2256 predicate(UseSVE > 0 &&
2257 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2258 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT));
2259 match(Set dst (VectorCastF2X src));
2260 effect(TEMP_DEF dst, TEMP tmp);
2261 ins_cost(3 * SVE_COST);
2262 format %{ "sve_vectorcast_f2x $dst, $src\t# convert F to B/H vector" %}
2263 ins_encode %{
2264 BasicType to_bt = Matcher::vector_element_basic_type(this);
2265 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2266 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
2267 __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
2268 as_FloatRegister($dst$$reg), __ S, as_FloatRegister($tmp$$reg));
2269 %}
2270 ins_pipe(pipe_slow);
2271 %}
2272
2273 instruct vcvtFtoI(vReg dst, vReg src)
2274 %{
2275 predicate(UseSVE > 0 &&
2276 (n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2277 match(Set dst (VectorCastF2X src));
2278 ins_cost(SVE_COST);
2279 format %{ "sve_vectorcast_f2x $dst, $src\t# convert F to I vector" %}
2280 ins_encode %{
2281 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
2282 %}
2283 ins_pipe(pipe_slow);
2284 %}
2285
2286 instruct vcvtFtoL(vReg dst, vReg src)
2287 %{
2288 predicate(UseSVE > 0 &&
2289 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
2290 match(Set dst (VectorCastF2X src));
2291 ins_cost(SVE_COST * 2);
2292 format %{ "sve_vectorcast_f2x $dst, $src\t# convert F to L vector" %}
2293 ins_encode %{
2294 __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
2295 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);
2296 %}
2297 ins_pipe(pipe_slow);
2298 %}
2299
2300 instruct vcvtFtoD(vReg dst, vReg src)
2301 %{
2302 predicate(UseSVE > 0 &&
2303 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
2304 match(Set dst (VectorCastF2X src));
2305 ins_cost(2 * SVE_COST);
2306 format %{ "sve_vectorcast_f2d $dst, $dst\t# convert F to D vector" %}
2307 ins_encode %{
2308 __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
2309 __ sve_fcvt(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);
2310 %}
2311 ins_pipe(pipe_slow);
2312 %}
2313
2314 instruct vcvtDtoX_narrow(vReg dst, vReg src, vReg tmp)
2315 %{
2316 predicate(UseSVE > 0 &&
2317 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2318 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2319 n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2320 match(Set dst (VectorCastD2X src));
2321 effect(TEMP_DEF dst, TEMP tmp);
2322 ins_cost(3 * SVE_COST);
2323 format %{ "sve_vectorcast_d2x $dst, $src\t# convert D to X vector (narrow)" %}
2324 ins_encode %{
2325 BasicType to_bt = Matcher::vector_element_basic_type(this);
2326 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2327 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
2328 __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
2329 as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
2330 %}
2331 ins_pipe(pipe_slow);
2332 %}
2333 dnl
2334 dnl vcvtDtoL
2335 VECTOR_CAST_X2X(D, L, fcvtzs, D, d2l)
2336
2337 instruct vcvtDtoF(vReg dst, vReg src, vReg tmp)
2338 %{
2339 predicate(UseSVE > 0 &&
2340 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2341 match(Set dst (VectorCastD2X src));
2342 effect(TEMP_DEF dst, TEMP tmp);
2343 ins_cost(3 * SVE_COST);
2344 format %{ "sve_vectorcast_d2f $dst, S, $dst\t# convert D to F vector" %}
2345 ins_encode %{
2346 __ sve_fcvt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
2347 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
2348 as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
2349 %}
2350 ins_pipe(pipe_slow);
2351 %}
2352
2353 dnl
2354 dnl
2355 // ------------------------------ Vector extract ---------------------------------
2356 dnl
2357 define(`VECTOR_EXTRACT_I', `
2358 instruct extract$1`'($3 dst, vReg src, immI idx, vReg vtmp)
2359 %{
2360 predicate(UseSVE > 0 && n->in(2)->get_int() >= $2);
2361 match(Set dst (Extract$1 src idx));
2362 effect(TEMP vtmp);
2363 ins_cost(2 * SVE_COST);
2364 format %{ "sve_extract_integral $dst, $4, $src, $idx\t# extract from vector($1)" %}
2365 ins_encode %{
2366 __ sve_extract_integral(as_Register($dst$$reg), __ $4, as_FloatRegister($src$$reg),
2367 (int)($idx$$constant), /* is_signed */ ifelse($1, L, false, true), as_FloatRegister($vtmp$$reg));
2368 %}
2369 ins_pipe(pipe_slow);
2370 %}')dnl
2371 dnl $1 $2 $3 $4
2372 VECTOR_EXTRACT_I(B, 16, iRegINoSp, B)
2373 VECTOR_EXTRACT_I(S, 8, iRegINoSp, H)
2374 VECTOR_EXTRACT_I(I, 4, iRegINoSp, S)
2375 VECTOR_EXTRACT_I(L, 2, iRegLNoSp, D)
2376 dnl
2377 define(`VECTOR_EXTRACT_I_LT', `
2378 instruct extract$1_LT$2`'($3 dst, vReg src, immI idx)
2379 %{
2380 predicate(UseSVE > 0 && n->in(2)->get_int() < $2);
2381 match(Set dst (Extract$1 src idx));
2382 ins_cost(INSN_COST);
2383 format %{ "ifelse($4, D, umov, smov) $dst, $4, $src, $idx\t# extract from vector($1)" %}
2384 ins_encode %{
2385 __ ifelse($4, D, umov, smov)(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ $4, $idx$$constant);
2386 %}
2387 ins_pipe(pipe_class_default);
2388 %}')dnl
2389 dnl $1 $2 $3 $4
2390 VECTOR_EXTRACT_I_LT(B, 16, iRegINoSp, B)
2391 VECTOR_EXTRACT_I_LT(S, 8, iRegINoSp, H)
2392 VECTOR_EXTRACT_I_LT(I, 4, iRegINoSp, S)
2393 VECTOR_EXTRACT_I_LT(L, 2, iRegLNoSp, D)
2394
2395 instruct extractF(vRegF dst, vReg src, immI idx)
2396 %{
2397 predicate(UseSVE > 0);
2398 match(Set dst (ExtractF src idx));
2399 ins_cost(2 * SVE_COST);
2400 format %{ "sve_extract_f $dst, S, $src, $idx\t# extract from vector(F)" %}
2401 ins_encode %{
2402 if ((as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg)) && ($idx$$constant == 0)) {
2403 /* empty */
2404 } else if ($idx$$constant == 0) {
2405 __ fmovs(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
2406 } else if ($idx$$constant < 4) {
2407 __ ins(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($src$$reg), 0, (int)($idx$$constant));
2408 } else {
2409 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2410 __ sve_ext(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), $idx$$constant << 2);
2411 }
2412 %}
2413 ins_pipe(pipe_slow);
2414 %}
2415
2416 instruct extractD(vRegD dst, vReg src, immI idx)
2417 %{
2418 predicate(UseSVE > 0);
2419 match(Set dst (ExtractD src idx));
2420 ins_cost(2 * SVE_COST);
2421 format %{ "sve_extract_d $dst, D, $src, $idx\t# extract from vector(D)" %}
2422 ins_encode %{
2423 if ((as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg)) && ($idx$$constant == 0)) {
2424 /* empty */
2425 } else if ($idx$$constant == 0) {
2426 __ fmovd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
2427 } else if ($idx$$constant == 1) {
2428 __ ins(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), 0, 1);
2429 } else {
2430 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2431 __ sve_ext(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), $idx$$constant << 3);
2432 }
2433 %}
2434 ins_pipe(pipe_slow);
2435 %}
2436
2437 // ------------------------------- VectorTest ----------------------------------
2438
2439 instruct vtest_alltrue(iRegINoSp dst, pRegGov src1, pRegGov src2, pReg ptmp, rFlagsReg cr)
2440 %{
2441 predicate(UseSVE > 0 &&
2442 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
2443 match(Set dst (VectorTest src1 src2));
2444 effect(TEMP ptmp, KILL cr);
2445 ins_cost(SVE_COST);
2446 format %{ "sve_eors $ptmp, $src1, $src2\t# $src2 is all true mask\n"
2447 "csetw $dst, EQ\t# VectorTest (sve) - alltrue" %}
2448 ins_encode %{
2449 __ sve_eors(as_PRegister($ptmp$$reg), ptrue,
2450 as_PRegister($src1$$reg), as_PRegister($src2$$reg));
2451 __ csetw(as_Register($dst$$reg), Assembler::EQ);
2452 %}
2453 ins_pipe(pipe_slow);
2454 %}
2455
2456 instruct vtest_anytrue(iRegINoSp dst, pRegGov src1, pRegGov src2, rFlagsReg cr)
2457 %{
2458 predicate(UseSVE > 0 &&
2459 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
2460 match(Set dst (VectorTest src1 src2));
2461 effect(KILL cr);
2462 ins_cost(SVE_COST);
2463 format %{ "sve_ptest $src1\n\t"
2464 "csetw $dst, NE\t# VectorTest (sve) - anytrue" %}
2465 ins_encode %{
2466 // "src2" is not used for sve.
2467 __ sve_ptest(ptrue, as_PRegister($src1$$reg));
2468 __ csetw(as_Register($dst$$reg), Assembler::NE);
2469 %}
2470 ins_pipe(pipe_slow);
2471 %}
2472 dnl
2473 // ------------------------------ Vector insert ---------------------------------
2474
2475 instruct insertI_le128bits(vReg dst, vReg src, iRegIorL2I val, immI idx) %{
2476 predicate(UseSVE > 0 &&
2477 (Matcher::vector_length_in_bytes(n) == 8 || Matcher::vector_length_in_bytes(n) == 16) &&
2478 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2479 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2480 n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2481 match(Set dst (VectorInsert (Binary src val) idx));
2482 ins_cost(2 * INSN_COST);
2483 format %{ "orr $dst, T8/16B, $src, $src\n\t"
2484 "mov $dst, B/H/S, $idx, $val\t# insertI into vector(64/128bits)" %}
2485 ins_encode %{
2486 BasicType bt = Matcher::vector_element_basic_type(this);
2487 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
2488 __ orr(as_FloatRegister($dst$$reg), Matcher::vector_length_in_bytes(this) == 8 ? __ T8B : __ T16B,
2489 as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2490 }
2491 __ mov(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(Matcher::vector_element_basic_type(this)),
2492 $idx$$constant, $val$$Register);
2493 %}
2494 ins_pipe(pipe_slow);
2495 %}
2496
2497 instruct insertI_small_index(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg vtmp, pRegGov pgtmp, rFlagsReg cr) %{
2498 predicate(UseSVE > 0 && n->in(2)->get_int() < 32 &&
2499 Matcher::vector_length_in_bytes(n) > 16 &&
2500 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2501 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2502 n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2503 match(Set dst (VectorInsert (Binary src val) idx));
2504 effect(TEMP vtmp, TEMP pgtmp, KILL cr);
2505 ins_cost(4 * SVE_COST);
2506 format %{ "sve_index $vtmp, -16, 1\t# (B/H/S)\n\t"
2507 "sve_cmpeq $pgtmp, $vtmp, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2508 "sve_orr $dst, $src, $src\n\t"
2509 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
2510 ins_encode %{
2511 BasicType bt = Matcher::vector_element_basic_type(this, $src);
2512 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2513 __ block_comment("insert into vector (B/H/S) {");
2514 __ sve_index(as_FloatRegister($vtmp$$reg), size, -16, 1);
2515 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
2516 as_FloatRegister($vtmp$$reg), (int)($idx$$constant) - 16);
2517 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
2518 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2519 }
2520 __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), $val$$Register);
2521 __ block_comment("} insert into vector (B/H/S)");
2522 %}
2523 ins_pipe(pipe_slow);
2524 %}
2525
2526 instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg vtmp1, vReg vtmp2, pRegGov pgtmp, rFlagsReg cr) %{
2527 predicate(UseSVE > 0 && n->in(2)->get_int() >= 32 &&
2528 Matcher::vector_length_in_bytes(n) > 16 &&
2529 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2530 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2531 n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2532 match(Set dst (VectorInsert (Binary src val) idx));
2533 effect(TEMP vtmp1, TEMP vtmp2, TEMP pgtmp, KILL cr);
2534 ins_cost(5 * SVE_COST);
2535 format %{ "sve_index $vtmp1, 0, 1\t# (B/H/S)\n\t"
2536 "sve_dup $vtmp2, $idx\t# (B/H/S)\n\t"
2537 "sve_cmpeq $pgtmp, $vtmp1, $vtmp2\n\t"
2538 "sve_orr $dst, $src, $src\n\t"
2539 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
2540 ins_encode %{
2541 BasicType bt = Matcher::vector_element_basic_type(this, $src);
2542 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2543 __ block_comment("insert into vector (B/H/S) {");
2544 __ sve_index(as_FloatRegister($vtmp1$$reg), size, 0, 1);
2545 __ sve_dup(as_FloatRegister($vtmp2$$reg), size, (int)($idx$$constant));
2546 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
2547 as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg));
2548 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
2549 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2550 }
2551 __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), $val$$Register);
2552 __ block_comment("} insert into vector (B/H/S)");
2553 %}
2554 ins_pipe(pipe_slow);
2555 %}
2556
2557 instruct insertL_128bits(vReg dst, vReg src, iRegL val, immI idx) %{
2558 predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) == 16 &&
2559 n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2560 match(Set dst (VectorInsert (Binary src val) idx));
2561 ins_cost(2 * INSN_COST);
2562 format %{ "orr $dst, T16B, $src, $src\n\t"
2563 "mov $dst, D, $idx, $val\t# insertL into vector(128bits)" %}
2564 ins_encode %{
2565 BasicType bt = Matcher::vector_element_basic_type(this);
2566 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
2567 __ orr(as_FloatRegister($dst$$reg), __ T16B,
2568 as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2569 }
2570 __ mov(as_FloatRegister($dst$$reg), __ D, $idx$$constant, $val$$Register);
2571 %}
2572 ins_pipe(pipe_slow);
2573 %}
2574
2575 instruct insertL(vReg dst, vReg src, iRegL val, immI idx, vReg vtmp, pRegGov pgtmp, rFlagsReg cr) %{
2576 predicate(UseSVE > 0 &&
2577 Matcher::vector_length_in_bytes(n) > 16 &&
2578 n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2579 match(Set dst (VectorInsert (Binary src val) idx));
2580 effect(TEMP vtmp, TEMP pgtmp, KILL cr);
2581 ins_cost(4 * SVE_COST);
2582 format %{ "sve_index $vtmp, D, -16, 1\n\t"
2583 "sve_cmpeq $pgtmp, $vtmp, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2584 "sve_orr $dst, $src, $src\n\t"
2585 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (L)" %}
2586 ins_encode %{
2587 __ block_comment("insert into vector (L) {");
2588 __ sve_index(as_FloatRegister($vtmp$$reg), __ D, -16, 1);
2589 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
2590 as_FloatRegister($vtmp$$reg), (int)($idx$$constant) - 16);
2591 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
2592 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2593 }
2594 __ sve_cpy(as_FloatRegister($dst$$reg), __ D,
2595 as_PRegister($pgtmp$$reg), $val$$Register);
2596 __ block_comment("} insert into vector (L)");
2597 %}
2598 ins_pipe(pipe_slow);
2599 %}
2600
2601 instruct insertF_le128bits(vReg dst, vReg src, vRegF val, immI idx) %{
2602 predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
2603 (Matcher::vector_length_in_bytes(n) == 8 || Matcher::vector_length_in_bytes(n) == 16));
2604 match(Set dst (VectorInsert (Binary src val) idx));
2605 ins_cost(2 * INSN_COST);
2606 effect(TEMP_DEF dst);
2607 format %{ "orr $dst, T8/16B, $src, $src\n\t"
2608 "ins $dst, S, $val, $idx, 0\t# insertF into vector(64/128bits)" %}
2609 ins_encode %{
2610 __ orr(as_FloatRegister($dst$$reg), Matcher::vector_length_in_bytes(this) == 8 ? __ T8B : __ T16B,
2611 as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2612 __ ins(as_FloatRegister($dst$$reg), __ S,
2613 as_FloatRegister($val$$reg), $idx$$constant, 0);
2614 %}
2615 ins_pipe(pipe_slow);
2616 %}
2617
2618 instruct insertF_small_index(vReg dst, vReg src, vRegF val, immI idx, pRegGov pgtmp, rFlagsReg cr) %{
2619 predicate(UseSVE > 0 && n->in(2)->get_int() < 32 &&
2620 Matcher::vector_length_in_bytes(n) > 16 &&
2621 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2622 match(Set dst (VectorInsert (Binary src val) idx));
2623 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
2624 ins_cost(4 * SVE_COST);
2625 format %{ "sve_index $dst, S, -16, 1\n\t"
2626 "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2627 "sve_orr $dst, $src, $src\n\t"
2628 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
2629 ins_encode %{
2630 __ block_comment("insert into vector (F) {");
2631 __ sve_index(as_FloatRegister($dst$$reg), __ S, -16, 1);
2632 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
2633 as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2634 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2635 __ sve_cpy(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
2636 __ block_comment("} insert into vector (F)");
2637 %}
2638 ins_pipe(pipe_slow);
2639 %}
2640
2641 instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr) %{
2642 predicate(UseSVE > 0 && n->in(2)->get_int() >= 32 &&
2643 Matcher::vector_length_in_bytes(n) > 16 &&
2644 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2645 match(Set dst (VectorInsert (Binary src val) idx));
2646 effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
2647 ins_cost(5 * SVE_COST);
2648 format %{ "sve_index $tmp1, S, 0, 1\n\t"
2649 "sve_dup $dst, S, $idx\n\t"
2650 "sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
2651 "sve_orr $dst, $src, $src\n\t"
2652 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
2653 ins_encode %{
2654 __ block_comment("insert into vector (F) {");
2655 __ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
2656 __ sve_dup(as_FloatRegister($dst$$reg), __ S, (int)($idx$$constant));
2657 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
2658 as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
2659 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
2660 as_FloatRegister($src$$reg));
2661 __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
2662 as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
2663 __ block_comment("} insert into vector (F)");
2664 %}
2665 ins_pipe(pipe_slow);
2666 %}
2667
2668 instruct insertD_128bits(vReg dst, vReg src, vRegD val, immI idx) %{
2669 predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) == 16 &&
2670 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
2671 match(Set dst (VectorInsert (Binary src val) idx));
2672 ins_cost(2 * INSN_COST);
2673 effect(TEMP_DEF dst);
2674 format %{ "orr $dst, T16B, $src, $src\n\t"
2675 "ins $dst, D, $val, $idx, 0\t# insertD into vector(128bits)" %}
2676 ins_encode %{
2677 __ orr(as_FloatRegister($dst$$reg), __ T16B,
2678 as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2679 __ ins(as_FloatRegister($dst$$reg), __ D,
2680 as_FloatRegister($val$$reg), $idx$$constant, 0);
2681 %}
2682 ins_pipe(pipe_slow);
2683 %}
2684
2685 instruct insertD(vReg dst, vReg src, vRegD val, immI idx, pRegGov pgtmp, rFlagsReg cr) %{
2686 predicate(UseSVE > 0 &&
2687 Matcher::vector_length_in_bytes(n) > 16 &&
2688 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
2689 match(Set dst (VectorInsert (Binary src val) idx));
2690 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
2691 ins_cost(4 * SVE_COST);
2692 format %{ "sve_index $dst, D, -16, 1\n\t"
2693 "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2694 "sve_orr $dst, $src, $src\n\t"
2695 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (D)" %}
2696 ins_encode %{
2697 __ block_comment("insert into vector (D) {");
2698 __ sve_index(as_FloatRegister($dst$$reg), __ D, -16, 1);
2699 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
2700 as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2701 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
2702 as_FloatRegister($src$$reg));
2703 __ sve_cpy(as_FloatRegister($dst$$reg), __ D,
2704 as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
2705 __ block_comment("} insert into vector (D)");
2706 %}
2707 ins_pipe(pipe_slow);
2708 %}
2709
2710 // ------------------------------ Vector shuffle -------------------------------
2711
2712 instruct loadshuffle(vReg dst, vReg src) %{
2713 predicate(UseSVE > 0);
2714 match(Set dst (VectorLoadShuffle src));
2715 ins_cost(SVE_COST);
2716 format %{ "sve_loadshuffle $dst, $src\t# vector load shuffle (B/H/S/D)" %}
2717 ins_encode %{
2718 BasicType bt = Matcher::vector_element_basic_type(this);
2719 if (bt == T_BYTE) {
2720 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
2721 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
2722 as_FloatRegister($src$$reg));
2723 }
2724 } else {
2725 __ sve_vector_extend(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
2726 as_FloatRegister($src$$reg), __ B);
2727 }
2728 %}
2729 ins_pipe(pipe_slow);
2730 %}
2731
2732 // ------------------------------ Vector rearrange -------------------------------
2733
2734 instruct rearrange(vReg dst, vReg src, vReg shuffle)
2735 %{
2736 predicate(UseSVE > 0);
2737 match(Set dst (VectorRearrange src shuffle));
2738 ins_cost(SVE_COST);
2739 format %{ "sve_tbl $dst, $src, $shuffle\t# vector rearrange" %}
2740 ins_encode %{
2741 BasicType bt = Matcher::vector_element_basic_type(this, $src);
2742 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2743 __ sve_tbl(as_FloatRegister($dst$$reg), size,
2744 as_FloatRegister($src$$reg), as_FloatRegister($shuffle$$reg));
2745 %}
2746 ins_pipe(pipe_slow);
2747 %}
2748
2749 // ------------------------------ Vector Load Gather ---------------------------------
2750
2751 instruct gatherI(vReg dst, indirect mem, vReg idx) %{
2752 predicate(UseSVE > 0 &&
2753 n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
2754 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2755 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2756 match(Set dst (LoadVectorGather mem idx));
2757 ins_cost(SVE_COST);
2758 format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (S)" %}
2759 ins_encode %{
2760 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), ptrue,
2761 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2762 %}
2763 ins_pipe(pipe_slow);
2764 %}
2765
2766 instruct gatherL(vReg dst, indirect mem, vReg idx) %{
2767 predicate(UseSVE > 0 &&
2768 n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
2769 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2770 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2771 match(Set dst (LoadVectorGather mem idx));
2772 ins_cost(2 * SVE_COST);
2773 format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (D)" %}
2774 ins_encode %{
2775 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2776 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), ptrue, as_Register($mem$$base),
2777 as_FloatRegister($idx$$reg));
2778 %}
2779 ins_pipe(pipe_slow);
2780 %}
2781
2782 // ------------------------------ Vector Load Gather Partial-------------------------------
2783
2784 instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2785 predicate(UseSVE > 0 &&
2786 n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
2787 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2788 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2789 match(Set dst (LoadVectorGather mem idx));
2790 effect(TEMP ptmp, KILL cr);
2791 ins_cost(2 * SVE_COST + INSN_COST);
2792 format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (S)" %}
2793 ins_encode %{
2794 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this));
2795 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
2796 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2797 %}
2798 ins_pipe(pipe_slow);
2799 %}
2800
2801 instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2802 predicate(UseSVE > 0 &&
2803 n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
2804 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2805 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2806 match(Set dst (LoadVectorGather mem idx));
2807 effect(TEMP ptmp, KILL cr);
2808 ins_cost(3 * SVE_COST + INSN_COST);
2809 format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (D)" %}
2810 ins_encode %{
2811 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this));
2812 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2813 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
2814 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2815 %}
2816 ins_pipe(pipe_slow);
2817 %}
2818
2819 // ------------------------------ Vector Load Gather Predicated -------------------------------
2820
2821 instruct gatherI_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
2822 predicate(UseSVE > 0 &&
2823 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2824 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2825 match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
2826 ins_cost(SVE_COST);
2827 format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (S)" %}
2828 ins_encode %{
2829 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
2830 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2831 %}
2832 ins_pipe(pipe_slow);
2833 %}
2834
2835 instruct gatherL_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
2836 predicate(UseSVE > 0 &&
2837 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2838 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2839 match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
2840 ins_cost(2 * SVE_COST);
2841 format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (D)" %}
2842 ins_encode %{
2843 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2844 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
2845 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2846 %}
2847 ins_pipe(pipe_slow);
2848 %}
2849
2850 // ------------------------------ Vector Store Scatter -------------------------------
2851
2852 instruct scatterI(indirect mem, vReg src, vReg idx) %{
2853 predicate(UseSVE > 0 &&
2854 n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
2855 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2856 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2857 match(Set mem (StoreVectorScatter mem (Binary src idx)));
2858 ins_cost(SVE_COST);
2859 format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (S)" %}
2860 ins_encode %{
2861 __ sve_st1w_scatter(as_FloatRegister($src$$reg), ptrue,
2862 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2863 %}
2864 ins_pipe(pipe_slow);
2865 %}
2866
2867 instruct scatterL(indirect mem, vReg src, vReg idx) %{
2868 predicate(UseSVE > 0 &&
2869 n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
2870 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2871 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2872 match(Set mem (StoreVectorScatter mem (Binary src idx)));
2873 ins_cost(2 * SVE_COST);
2874 format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (D)" %}
2875 ins_encode %{
2876 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2877 __ sve_st1d_scatter(as_FloatRegister($src$$reg), ptrue,
2878 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2879 %}
2880 ins_pipe(pipe_slow);
2881 %}
2882
2883 // ------------------------------ Vector Store Scatter Partial -------------------------------
2884
2885 instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2886 predicate(UseSVE > 0 &&
2887 n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
2888 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2889 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2890 match(Set mem (StoreVectorScatter mem (Binary src idx)));
2891 effect(TEMP ptmp, KILL cr);
2892 ins_cost(2 * SVE_COST + INSN_COST);
2893 format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (S)" %}
2894 ins_encode %{
2895 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src));
2896 __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
2897 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2898 %}
2899 ins_pipe(pipe_slow);
2900 %}
2901
2902 instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2903 predicate(UseSVE > 0 &&
2904 n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
2905 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2906 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2907 match(Set mem (StoreVectorScatter mem (Binary src idx)));
2908 effect(TEMP ptmp, KILL cr);
2909 ins_cost(3 * SVE_COST + INSN_COST);
2910 format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (D)" %}
2911 ins_encode %{
2912 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src));
2913 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2914 __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
2915 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2916 %}
2917 ins_pipe(pipe_slow);
2918 %}
2919
2920 // ------------------------------ Vector Store Scatter Predicated -------------------------------
2921
2922 instruct scatterI_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
2923 predicate(UseSVE > 0 &&
2924 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2925 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2926 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
2927 ins_cost(SVE_COST);
2928 format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicate (S)" %}
2929 ins_encode %{
2930 __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
2931 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2932 %}
2933 ins_pipe(pipe_slow);
2934 %}
2935
2936 instruct scatterL_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
2937 predicate(UseSVE > 0 &&
2938 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2939 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2940 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
2941 ins_cost(2 * SVE_COST);
2942 format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated (D)" %}
2943 ins_encode %{
2944 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2945 __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
2946 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2947 %}
2948 ins_pipe(pipe_slow);
2949 %}
2950
2951 // ------------------------------ Vector Load Const -------------------------------
2952
2953 instruct loadconB(vReg dst, immI0 src) %{
2954 predicate(UseSVE > 0 &&
2955 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2956 match(Set dst (VectorLoadConst src));
2957 ins_cost(SVE_COST);
2958 format %{ "sve_index $dst, 0, 1\t# generate iota indices" %}
2959 ins_encode %{
2960 __ sve_index(as_FloatRegister($dst$$reg), __ B, 0, 1);
2961 %}
2962 ins_pipe(pipe_slow);
2963 %}
2964
2965 // Intrisics for String.indexOf(char)
2966
2967 dnl
2968 define(`STRING_INDEXOF_CHAR', `
2969 instruct string$1_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
2970 iRegI_R0 result, vReg ztmp1, vReg ztmp2,
2971 pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
2972 %{
2973 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
2974 predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::$1));
2975 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
2976
2977 format %{ "String$2 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
2978
2979 ins_encode %{
2980 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
2981 as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
2982 as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), $3 /* isL */);
2983 %}
2984 ins_pipe(pipe_class_memory);
2985 %}')dnl
2986 dnl $1 $2 $3
2987 STRING_INDEXOF_CHAR(L, Latin1, true)
2988 STRING_INDEXOF_CHAR(U, UTF16, false)
2989
2990 // ---------------------------- Vector mask reductions ---------------------------
2991 instruct vmask_truecount(iRegINoSp dst, pReg src) %{
2992 predicate(UseSVE > 0);
2993 match(Set dst (VectorMaskTrueCount src));
2994 ins_cost(SVE_COST);
2995 format %{ "vmask_truecount $dst, $src\t# vector mask truecount (sve)" %}
2996 ins_encode %{
2997 BasicType bt = Matcher::vector_element_basic_type(this, $src);
2998 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2999 __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($src$$reg));
3000 %}
3001 ins_pipe(pipe_slow);
3002 %}
3003
3004 // Return the index of the first mask lane that is set, or vector length if none of
3005 // them are set.
3006 instruct vmask_firsttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
3007 predicate(UseSVE > 0);
3008 match(Set dst (VectorMaskFirstTrue src));
3009 effect(TEMP ptmp);
3010 ins_cost(3 * SVE_COST);
3011 format %{ "vmask_firsttrue $dst, $src\t# vector mask firsttrue (sve)" %}
3012 ins_encode %{
3013 uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
3014 BasicType bt = Matcher::vector_element_basic_type(this, $src);
3015 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3016 // When the input predicate is all-false, the result should be the vector length
3017 // instead of max vector register size.
3018 if (length_in_bytes == MaxVectorSize) {
3019 __ sve_brkb(as_PRegister($ptmp$$reg), ptrue, as_PRegister($src$$reg), false);
3020 } else {
3021 __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
3022 __ sve_brkb(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg), as_PRegister($src$$reg), false);
3023 }
3024 __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($ptmp$$reg));
3025 %}
3026 ins_pipe(pipe_slow);
3027 %}
3028
3029 instruct vmask_lasttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
3030 predicate(UseSVE > 0);
3031 match(Set dst (VectorMaskLastTrue src));
3032 effect(TEMP ptmp);
3033 ins_cost(3 * SVE_COST);
3034 format %{ "vmask_lasttrue $dst, $src\t# vector mask lasttrue (sve)" %}
3035 ins_encode %{
3036 BasicType bt = Matcher::vector_element_basic_type(this, $src);
3037 __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($src$$reg), as_PRegister($ptmp$$reg));
3038 %}
3039 ins_pipe(pipe_slow);
3040 %}
3041
3042 instruct vmask_tolong(iRegLNoSp dst, pReg src, vReg vtmp1, vReg vtmp2) %{
3043 predicate(UseSVE > 0 &&
3044 n->in(1)->bottom_type()->is_vect()->length() <= 64);
3045 match(Set dst (VectorMaskToLong src));
3046 effect(TEMP vtmp1, TEMP vtmp2);
3047 ins_cost(13 * SVE_COST);
3048 format %{ "vmask_tolong $dst, $src\t# vector mask tolong (sve)" %}
3049 ins_encode %{
3050 __ sve_vmask_tolong(as_Register($dst$$reg), as_PRegister($src$$reg),
3051 Matcher::vector_element_basic_type(this, $src),
3052 Matcher::vector_length(this, $src),
3053 as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg));
3054 %}
3055 ins_pipe(pipe_slow);
3056 %}
3057
3058 // ---------------------------- Vector mask generation ---------------------------
3059 // The rules below set predicate registers. They can guarantee the high bits of dst
3060 // are cleared with zero when the vector length is less than the full size of
3061 // hardware vector register width.
3062
3063 define(`MASKALL_IMM', `
3064 instruct vmaskAll_imm$1(pRegGov dst, imm$1 src) %{
3065 predicate(UseSVE > 0);
3066 match(Set dst (MaskAll src));
3067 ins_cost(SVE_COST);
3068 format %{ "sve_ptrue_lanecnt/sve_pfalse $dst\t# mask all (sve) ($2)" %}
3069 ins_encode %{
3070 ifelse($1, `I', int, long) con = (ifelse($1, `I', int, long))$src$$constant;
3071 if (con == 0) {
3072 __ sve_pfalse(as_PRegister($dst$$reg));
3073 } else {
3074 assert(con == -1, "invalid constant value for mask");
3075 BasicType bt = Matcher::vector_element_basic_type(this);
3076 __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
3077 Matcher::vector_length(this));
3078 }
3079 %}
3080 ins_pipe(pipe_slow);
3081 %}')dnl
3082
3083 define(`MASKALL', `
3084 instruct vmaskAll$1(pRegGov dst, ifelse($1, `I', iRegIorL2I, iRegL) src, vReg tmp, rFlagsReg cr) %{
3085 predicate(UseSVE > 0);
3086 match(Set dst (MaskAll src));
3087 effect(TEMP tmp, KILL cr);
3088 ins_cost(3 * SVE_COST);
3089 format %{ "sve_dup $tmp, $src\n\t"
3090 "sve_ptrue $dst, vector_length\n\t"
3091 "sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) ($2)" %}
3092 ins_encode %{
3093 BasicType bt = Matcher::vector_element_basic_type(this);
3094 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3095 uint length_in_bytes = Matcher::vector_length_in_bytes(this);
3096 __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
3097 if (length_in_bytes < MaxVectorSize) {
3098 __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), size, Matcher::vector_length(this));
3099 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size,
3100 as_PRegister($dst$$reg), as_FloatRegister($tmp$$reg), 0);
3101 } else {
3102 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
3103 }
3104 %}
3105 ins_pipe(pipe_slow);
3106 %}')dnl
3107 dnl
3108 // maskAll (full or partial predicate size)
3109 MASKALL_IMM(I, B/H/S)
3110 MASKALL(I, B/H/S)
3111 MASKALL_IMM(L, D)
3112 MASKALL(L, D)
3113
3114 // vector mask compare
3115
3116 instruct vmaskcmp(pRegGov dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %{
3117 predicate(UseSVE > 0);
3118 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
3119 effect(KILL cr);
3120 ins_cost(2 * SVE_COST);
3121 format %{ "sve_cmp $dst, $src1, $src2\t# vector mask cmp (sve)" %}
3122 ins_encode %{
3123 uint length_in_bytes = Matcher::vector_length_in_bytes(this);
3124 BasicType bt = Matcher::vector_element_basic_type(this);
3125 if (length_in_bytes == MaxVectorSize) {
3126 __ sve_compare(as_PRegister($dst$$reg), bt, ptrue, as_FloatRegister($src1$$reg),
3127 as_FloatRegister($src2$$reg), (int)$cond$$constant);
3128 } else {
3129 __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
3130 Matcher::vector_length(this));
3131 __ sve_compare(as_PRegister($dst$$reg), bt, as_PRegister($dst$$reg), as_FloatRegister($src1$$reg),
3132 as_FloatRegister($src2$$reg), (int)$cond$$constant);
3133 }
3134 %}
3135 ins_pipe(pipe_slow);
3136 %}
3137
3138 instruct vmaskcmp_masked(pRegGov dst, vReg src1, vReg src2, immI cond, pRegGov pg, rFlagsReg cr) %{
3139 predicate(UseSVE > 0);
3140 match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond pg)));
3141 effect(KILL cr);
3142 ins_cost(SVE_COST);
3143 format %{ "sve_cmp $dst, $pg, $src1, $src2\t# vector mask cmp (sve)" %}
3144 ins_encode %{
3145 BasicType bt = Matcher::vector_element_basic_type(this);
3146 __ sve_compare(as_PRegister($dst$$reg), bt, as_PRegister($pg$$reg), as_FloatRegister($src1$$reg),
3147 as_FloatRegister($src2$$reg), (int)$cond$$constant);
3148 %}
3149 ins_pipe(pipe_slow);
3150 %}
3151
3152 // vector load mask
3153
3154 instruct vloadmaskB(pRegGov dst, vReg src, rFlagsReg cr) %{
3155 predicate(UseSVE > 0 &&
3156 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
3157 match(Set dst (VectorLoadMask src));
3158 effect(KILL cr);
3159 ins_cost(SVE_COST);
3160 format %{ "vloadmaskB $dst, $src\t# vector load mask (sve) (B)" %}
3161 ins_encode %{
3162 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ B,
3163 ptrue, as_FloatRegister($src$$reg), 0);
3164 %}
3165 ins_pipe(pipe_slow);
3166 %}
3167
3168 instruct vloadmask_extend(pRegGov dst, vReg src, vReg tmp, rFlagsReg cr) %{
3169 predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() != T_BYTE);
3170 match(Set dst (VectorLoadMask src));
3171 effect(TEMP tmp, KILL cr);
3172 ins_cost(3 * SVE_COST);
3173 format %{ "vloadmask $dst, $src\t# vector load mask (sve) (H/S/D)" %}
3174 ins_encode %{
3175 BasicType bt = Matcher::vector_element_basic_type(this);
3176 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3177 __ sve_vector_extend(as_FloatRegister($tmp$$reg), size, as_FloatRegister($src$$reg), __ B);
3178 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
3179 %}
3180 ins_pipe(pipe_slow);
3181 %}
3182
3183 instruct vmask_gen(pRegGov pg, iRegL len, rFlagsReg cr) %{
3184 predicate(UseSVE > 0);
3185 match(Set pg (VectorMaskGen len));
3186 effect(KILL cr);
3187 ins_cost(SVE_COST);
3188 format %{ "sve_whilelo $pg, zr, $len\t # sve" %}
3189 ins_encode %{
3190 BasicType bt = Matcher::vector_element_basic_type(this);
3191 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3192 __ sve_whilelo(as_PRegister($pg$$reg), size, zr, as_Register($len$$reg));
3193 %}
3194 ins_pipe(pipe_slow);
3195 %}