1 //
2 // Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2020, 2021, Arm Limited. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 dnl Generate the warning
27 // This file is automatically generated by running "m4 aarch64_sve_ad.m4". Do not edit ----
28 dnl
29
30 // AArch64 SVE Architecture Description File
31
32 dnl
33 define(`TYPE2DATATYPE',
34 `ifelse($1, `B', `BYTE',
35 $1, `S', `SHORT',
36 $1, `I', `INT',
37 $1, `L', `LONG',
38 $1, `F', `FLOAT',
39 $1, `D', `DOUBLE',
40 `error($1)')')dnl
41 dnl
42 dnl OPERAND_VMEMORYA_IMMEDIATE_OFFSET($1, $2, $3 $4 )
43 dnl OPERAND_VMEMORYA_IMMEDIATE_OFFSET(imm_type_abbr, imm_type, imm_len, scale)
44 define(`OPERAND_VMEMORYA_IMMEDIATE_OFFSET', `
45 operand vmemA_imm$1Offset$3()
46 %{
47 // (esize / msize) = $4
48 predicate(Address::offset_ok_for_sve_immed(n->get_$2(), $3,
49 Matcher::scalable_vector_reg_size(T_BYTE)ifelse($4, `1', `', ` / $4')));
50 match(Con$1);
51
52 op_cost(0);
53 format %{ %}
54 interface(CONST_INTER);
55 %}')dnl
56
57 // 4 bit signed offset -- for predicated load/store
58 OPERAND_VMEMORYA_IMMEDIATE_OFFSET(I, int, 4, 1)
59 OPERAND_VMEMORYA_IMMEDIATE_OFFSET(L, long, 4, 1)
60 dnl
61 dnl OPERAND_VMEMORYA_INDIRECT_OFFSET($1, $2 )
62 dnl OPERAND_VMEMORYA_INDIRECT_OFFSET(imm_type_abbr, imm_len)
63 define(`OPERAND_VMEMORYA_INDIRECT_OFFSET', `
64 operand vmemA_indOff$1$2$3(iRegP reg, vmemA_imm$1Offset$2 off)
65 %{
66 constraint(ALLOC_IN_RC(ptr_reg));
67 match(AddP reg off);
68 op_cost(0);
69 format %{ "[$reg, $off]" %}
70 interface(MEMORY_INTER) %{
71 base($reg);
72 `index'(0xffffffff);
73 scale(0x0);
74 disp($off);
75 %}
76 %}')dnl
77 OPERAND_VMEMORYA_INDIRECT_OFFSET(I, 4)
78 OPERAND_VMEMORYA_INDIRECT_OFFSET(L, 4)
79
80 // The indOff of vmemA is valid only when the vector element (load to/store from)
81 // size equals to memory element (load from/store to) size.
82 opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);
83
84 source_hpp %{
85 bool op_sve_supported(int opcode, int vlen, BasicType bt);
86 bool masked_op_sve_supported(int opcode, int vlen, BasicType bt);
87 %}
88
89 source %{
90
91 typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
92 PRegister Pg, const Address &adr);
93
94 // Predicated load/store, with optional ptrue to all elements of given predicate register.
95 static void loadStoreA_predicated(C2_MacroAssembler masm, bool is_store, FloatRegister reg,
96 PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
97 int opcode, Register base, int index, int size, int disp) {
98 sve_mem_insn_predicate insn;
99 int mesize = type2aelembytes(mem_elem_bt);
100 if (index == -1) {
101 assert(size == 0, "unsupported address mode: scale size = %d", size);
102 switch(mesize) {
103 case 1:
104 insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
105 break;
106 case 2:
107 insn = is_store ? &C2_MacroAssembler::sve_st1h : &C2_MacroAssembler::sve_ld1h;
108 break;
109 case 4:
110 insn = is_store ? &C2_MacroAssembler::sve_st1w : &C2_MacroAssembler::sve_ld1w;
111 break;
112 case 8:
113 insn = is_store ? &C2_MacroAssembler::sve_st1d : &C2_MacroAssembler::sve_ld1d;
114 break;
115 default:
116 assert(false, "unsupported");
117 ShouldNotReachHere();
118 }
119 int imm4 = disp / mesize / Matcher::scalable_vector_reg_size(vector_elem_bt);
120 (masm.*insn)(reg, Assembler::elemType_to_regVariant(vector_elem_bt), pg, Address(base, imm4));
121 } else {
122 assert(false, "unimplemented");
123 ShouldNotReachHere();
124 }
125 }
126
127 bool op_sve_supported(int opcode, int vlen, BasicType bt) {
128 int length_in_bytes = vlen * type2aelembytes(bt);
129 switch (opcode) {
130 case Op_MulAddVS2VI:
131 // No multiply reduction instructions
132 case Op_MulReductionVD:
133 case Op_MulReductionVF:
134 case Op_MulReductionVI:
135 case Op_MulReductionVL:
136 // Others
137 case Op_ExtractC:
138 case Op_ExtractUB:
139 return false;
140 // Vector API specific
141 case Op_VectorLoadShuffle:
142 case Op_VectorRearrange:
143 return vlen >= 4 && length_in_bytes <= MaxVectorSize;
144 case Op_LoadVector:
145 case Op_StoreVector:
146 return Matcher::vector_size_supported(bt, vlen);
147 default:
148 break;
149 }
150 // By default, we only support vector operations with no less than 8 bytes and 2 elements.
151 return 8 <= length_in_bytes && length_in_bytes <= MaxVectorSize && vlen >= 2;
152 }
153
154 bool masked_op_sve_supported(int opcode, int vlen, BasicType bt) {
155 if (opcode == Op_VectorRearrange) {
156 return false;
157 }
158 return op_sve_supported(opcode, vlen, bt);
159 }
160
161 %}
162
163 definitions %{
164 int_def SVE_COST (200, 200);
165 %}
166
167 dnl
168 dnl ELEMENT_SHORT_CHART($1, $2)
169 dnl ELEMENT_SHORT_CHART(etype, node)
170 define(`ELEMENT_SHORT_CHAR',`ifelse(`$1', `T_SHORT',
171 `($2->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
172 ($2->bottom_type()->is_vect()->element_basic_type() == T_CHAR))',
173 `($2->bottom_type()->is_vect()->element_basic_type() == $1)')')dnl
174 dnl
175
176 // All SVE instructions
177
178 // vector load/store
179
180 // Unpredicated vector load/store
181 instruct loadV(vReg dst, vmemA mem) %{
182 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() >= 16 &&
183 n->as_LoadVector()->memory_size() == MaxVectorSize);
184 match(Set dst (LoadVector mem));
185 ins_cost(4 * SVE_COST);
186 format %{ "sve_ldr $dst, $mem\t# vector (sve)" %}
187 ins_encode %{
188 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
189 BasicType bt = Matcher::vector_element_basic_type(this);
190 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg, ptrue,
191 bt, bt, $mem->opcode(),
192 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
193 %}
194 ins_pipe(pipe_slow);
195 %}
196
197 instruct storeV(vReg src, vmemA mem) %{
198 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() >= 16 &&
199 n->as_StoreVector()->memory_size() == MaxVectorSize);
200 match(Set mem (StoreVector mem src));
201 ins_cost(4 * SVE_COST);
202 format %{ "sve_str $mem, $src\t# vector (sve)" %}
203 ins_encode %{
204 FloatRegister src_reg = as_FloatRegister($src$$reg);
205 BasicType bt = Matcher::vector_element_basic_type(this, $src);
206 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg, ptrue,
207 bt, bt, $mem->opcode(),
208 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
209 %}
210 ins_pipe(pipe_slow);
211 %}dnl
212
213 dnl
214 define(`VLoadStore', `
215 // ifelse(load, $3, Load, Store) Vector ($6 bits)
216 instruct $3V$4_vreg`'(vReg $7, vmem$4 mem)
217 %{
218 predicate(UseSVE > 0 && `n->as_'ifelse(load, $3, Load, Store)Vector()->memory_size() == $4);
219 match(Set ifelse(load, $3, dst (LoadVector mem), mem (StoreVector mem src)));
220 ins_cost(4 * INSN_COST);
221 format %{ "$1 ifelse(load, $3, `$dst,$mem', `$mem,$src')\t# vector ($6 bits)" %}
222 ins_encode( `aarch64_enc_'ifelse(load, $3, ldr, str)v$2($7, mem) );
223 ins_pipe(v$3`_reg_mem'ifelse(eval($4 * 8), 128, 128, 64));
224 %}')dnl
225 dnl $1 $2 $3 $4 $5 $6 $7
226 VLoadStore(ldrh, H, load, 2, D, 16, dst)
227 VLoadStore(strh, H, store, 2, D, 16, src)
228 VLoadStore(ldrs, S, load, 4, D, 32, dst)
229 VLoadStore(strs, S, store, 4, D, 32, src)
230 VLoadStore(ldrd, D, load, 8, D, 64, dst)
231 VLoadStore(strd, D, store, 8, D, 64, src)
232 VLoadStore(ldrq, Q, load, 16, X, 128, dst)
233 VLoadStore(strq, Q, store, 16, X, 128, src)
234
235 // Predicated vector load/store, based on the vector length of the node.
236 // Only load/store values in the range of the memory_size. This is needed
237 // when the memory_size is lower than the hardware supported max vector size.
238 // And this might happen for Vector API mask vector load/store.
239 instruct loadV_partial(vReg dst, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
240 predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() > 16 &&
241 n->as_LoadVector()->memory_size() < MaxVectorSize);
242 match(Set dst (LoadVector mem));
243 effect(TEMP pgtmp, KILL cr);
244 ins_cost(6 * SVE_COST);
245 format %{ "sve_whilelo_zr_imm $pgtmp, vector_length\n\t"
246 "sve_ldr $dst, $pgtmp, $mem\t# load vector partial" %}
247 ins_encode %{
248 BasicType bt = Matcher::vector_element_basic_type(this);
249 __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
250 Matcher::vector_length(this));
251 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
252 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg,
253 as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
254 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
255 %}
256 ins_pipe(pipe_slow);
257 %}
258
259 instruct storeV_partial(vReg src, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
260 predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() > 16 &&
261 n->as_StoreVector()->memory_size() < MaxVectorSize);
262 match(Set mem (StoreVector mem src));
263 effect(TEMP pgtmp, KILL cr);
264 ins_cost(5 * SVE_COST);
265 format %{ "sve_whilelo_zr_imm $pgtmp, vector_length\n\t"
266 "sve_str $src, $pgtmp, $mem\t# store vector partial" %}
267 ins_encode %{
268 BasicType bt = Matcher::vector_element_basic_type(this, $src);
269 __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
270 Matcher::vector_length(this, $src));
271 FloatRegister src_reg = as_FloatRegister($src$$reg);
272 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg,
273 as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
274 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
275 %}
276 ins_pipe(pipe_slow);
277 %}
278
279 // vector load/store - predicated
280
281 instruct loadV_masked(vReg dst, vmemA mem, pRegGov pg) %{
282 predicate(UseSVE > 0 &&
283 n->as_LoadVector()->memory_size() == MaxVectorSize);
284 match(Set dst (LoadVectorMasked mem pg));
285 ins_cost(4 * SVE_COST);
286 format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated (sve)" %}
287 ins_encode %{
288 BasicType bt = Matcher::vector_element_basic_type(this);
289 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
290 as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
291 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
292 %}
293 ins_pipe(pipe_slow);
294 %}
295
296 instruct loadV_masked_partial(vReg dst, vmemA mem, pRegGov pg, pRegGov pgtmp, rFlagsReg cr) %{
297 predicate(UseSVE > 0 &&
298 n->as_LoadVector()->memory_size() < MaxVectorSize);
299 match(Set dst (LoadVectorMasked mem pg));
300 effect(TEMP pgtmp, KILL cr);
301 ins_cost(6 * SVE_COST);
302 format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated partial (sve)" %}
303 ins_encode %{
304 BasicType bt = Matcher::vector_element_basic_type(this);
305 __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
306 Matcher::vector_length(this));
307 __ sve_and(as_PRegister($pgtmp$$reg), as_PRegister($pgtmp$$reg),
308 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
309 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
310 as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
311 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
312 %}
313 ins_pipe(pipe_slow);
314 %}
315
316 instruct storeV_masked(vReg src, vmemA mem, pRegGov pg) %{
317 predicate(UseSVE > 0 &&
318 n->as_StoreVector()->memory_size() == MaxVectorSize);
319 match(Set mem (StoreVectorMasked mem (Binary src pg)));
320 ins_cost(4 * SVE_COST);
321 format %{ "sve_str $mem, $pg, $src\t# store vector predicated (sve)" %}
322 ins_encode %{
323 BasicType bt = Matcher::vector_element_basic_type(this, $src);
324 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
325 as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
326 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
327 %}
328 ins_pipe(pipe_slow);
329 %}
330
331 instruct storeV_masked_partial(vReg src, vmemA mem, pRegGov pg, pRegGov pgtmp, rFlagsReg cr) %{
332 predicate(UseSVE > 0 &&
333 n->as_StoreVector()->memory_size() < MaxVectorSize);
334 match(Set mem (StoreVectorMasked mem (Binary src pg)));
335 effect(TEMP pgtmp, KILL cr);
336 ins_cost(6 * SVE_COST);
337 format %{ "sve_str $mem, $pg, $src\t# store vector predicated partial (sve)" %}
338 ins_encode %{
339 BasicType bt = Matcher::vector_element_basic_type(this, $src);
340 __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
341 Matcher::vector_length(this, $src));
342 __ sve_and(as_PRegister($pgtmp$$reg), as_PRegister($pgtmp$$reg),
343 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
344 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
345 as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
346 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
347 %}
348 ins_pipe(pipe_slow);
349 %}
350
351 dnl
352 dnl MASKALL_IMM($1, $2 )
353 dnl MASKALL_IMM(type, size)
354 define(`MASKALL_IMM', `
355 instruct vmaskAll_imm$1(pRegGov dst, imm$1 src) %{
356 predicate(UseSVE > 0);
357 match(Set dst (MaskAll src));
358 ins_cost(SVE_COST);
359 format %{ "sve_ptrue_lanecnt/sve_pfalse $dst\t# mask all (sve) ($2)" %}
360 ins_encode %{
361 ifelse($1, `I', int, long) con = (ifelse($1, `I', int, long))$src$$constant;
362 if (con == 0) {
363 __ sve_pfalse(as_PRegister($dst$$reg));
364 } else {
365 assert(con == -1, "invalid constant value for mask");
366 BasicType bt = Matcher::vector_element_basic_type(this);
367 __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
368 Matcher::vector_length(this));
369 }
370 %}
371 ins_pipe(pipe_slow);
372 %}')dnl
373 dnl
374 dnl MASKALL($1, $2 )
375 dnl MASKALL(type, size)
376 define(`MASKALL', `
377 instruct vmaskAll$1(pRegGov dst, ifelse($1, `I', iRegIorL2I, iRegL) src, vReg tmp, rFlagsReg cr) %{
378 predicate(UseSVE > 0);
379 match(Set dst (MaskAll src));
380 effect(TEMP tmp, KILL cr);
381 ins_cost(3 * SVE_COST);
382 format %{ "sve_dup $tmp, $src\n\t"
383 "sve_ptrue_lanecnt $dst\n\t"
384 "sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) ($2)" %}
385 ins_encode %{
386 BasicType bt = Matcher::vector_element_basic_type(this);
387 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
388 uint length_in_bytes = Matcher::vector_length_in_bytes(this);
389 __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
390 if (length_in_bytes < MaxVectorSize) {
391 __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), size, Matcher::vector_length(this));
392 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size,
393 as_PRegister($dst$$reg), as_FloatRegister($tmp$$reg), 0);
394 } else {
395 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
396 }
397 %}
398 ins_pipe(pipe_slow);
399 %}')dnl
400 dnl
401 // maskAll (full or partial predicate size)
402 MASKALL_IMM(I, B/H/S)
403 MASKALL(I, B/H/S)
404 MASKALL_IMM(L, D)
405 MASKALL(L, D)
406
407 dnl
408 dnl MASK_LOGICAL_OP($1, $2, $3 )
409 dnl MASK_LOGICAL_OP(insn_name, op_name, insn)
410 define(`MASK_LOGICAL_OP', `
411 instruct vmask_$1(pRegGov pd, pRegGov pn, pRegGov pm) %{
412 predicate(UseSVE > 0);
413 match(Set pd ($2 pn pm));
414 ins_cost(SVE_COST);
415 format %{ "$3 $pd, $pn, $pm\t# predicate (sve)" %}
416 ins_encode %{
417 __ $3(as_PRegister($pd$$reg), ptrue,
418 as_PRegister($pn$$reg), as_PRegister($pm$$reg));
419 %}
420 ins_pipe(pipe_slow);
421 %}')dnl
422 dnl
423 // mask logical and/or/xor
424 MASK_LOGICAL_OP(and, AndVMask, sve_and)
425 MASK_LOGICAL_OP(or, OrVMask, sve_orr)
426 MASK_LOGICAL_OP(xor, XorVMask, sve_eor)
427
428 dnl
429 dnl MASK_LOGICAL_AND_NOT($1, $2 )
430 dnl MASK_LOGICAL_AND_NOT(type, size)
431 define(`MASK_LOGICAL_AND_NOT', `
432 instruct vmask_and_not$1(pRegGov pd, pRegGov pn, pRegGov pm, imm$1_M1 m1) %{
433 predicate(UseSVE > 0);
434 match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
435 ins_cost(SVE_COST);
436 format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) ($2)" %}
437 ins_encode %{
438 __ sve_bic(as_PRegister($pd$$reg), ptrue,
439 as_PRegister($pn$$reg), as_PRegister($pm$$reg));
440 %}
441 ins_pipe(pipe_slow);
442 %}')dnl
443 dnl
444 // mask logical and_not
445 MASK_LOGICAL_AND_NOT(I, B/H/S)
446 MASK_LOGICAL_AND_NOT(L, D)
447
448 // vector reinterpret
449
450 instruct reinterpret(vReg dst) %{
451 predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() ==
452 n->in(1)->bottom_type()->is_vect()->length_in_bytes()); // src == dst
453 match(Set dst (VectorReinterpret dst));
454 ins_cost(0);
455 format %{ "# reinterpret $dst\t# do nothing" %}
456 ins_encode %{
457 // empty
458 %}
459 ins_pipe(pipe_class_empty);
460 %}
461
462 instruct reinterpretResize(vReg dst, vReg src, pRegGov pgtmp, rFlagsReg cr) %{
463 predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() !=
464 n->in(1)->bottom_type()->is_vect()->length_in_bytes()); // src != dst
465 match(Set dst (VectorReinterpret src));
466 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
467 ins_cost(3 * SVE_COST);
468 format %{ "reinterpretResize $dst, $src\t# vector (sve)" %}
469 ins_encode %{
470 uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
471 uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
472 uint length_in_bytes_resize = length_in_bytes_src < length_in_bytes_dst ?
473 length_in_bytes_src : length_in_bytes_dst;
474 assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
475 "invalid vector length");
476 __ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ B, length_in_bytes_resize);
477 __ sve_dup(as_FloatRegister($dst$$reg), __ B, 0);
478 __ sve_sel(as_FloatRegister($dst$$reg), __ B, as_PRegister($pgtmp$$reg),
479 as_FloatRegister($src$$reg), as_FloatRegister($dst$$reg));
480 %}
481 ins_pipe(pipe_slow);
482 %}
483
484 // vector mask reinterpret
485
486 instruct vmask_reinterpret_same_esize(pRegGov dst_src) %{
487 predicate(UseSVE > 0 &&
488 n->as_Vector()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
489 n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
490 match(Set dst_src (VectorReinterpret dst_src));
491 ins_cost(0);
492 format %{ "# vmask_reinterpret $dst_src\t# do nothing" %}
493 ins_encode %{
494 // empty
495 %}
496 ins_pipe(pipe_class_empty);
497 %}
498
499 instruct vmask_reinterpret_diff_esize(pRegGov dst, pRegGov src, vReg tmp, rFlagsReg cr) %{
500 predicate(UseSVE > 0 &&
501 n->as_Vector()->length() != n->in(1)->bottom_type()->is_vect()->length() &&
502 n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
503 match(Set dst (VectorReinterpret src));
504 effect(TEMP tmp, KILL cr);
505 ins_cost(2 * SVE_COST);
506 format %{ "# vmask_reinterpret $dst, $src\t# vector (sve)" %}
507 ins_encode %{
508 BasicType from_bt = Matcher::vector_element_basic_type(this, $src);
509 Assembler::SIMD_RegVariant from_size = __ elemType_to_regVariant(from_bt);
510 BasicType to_bt = Matcher::vector_element_basic_type(this);
511 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
512 __ sve_cpy(as_FloatRegister($tmp$$reg), from_size, as_PRegister($src$$reg), -1, false);
513 __ sve_cmp(Assembler::EQ, as_PRegister($dst$$reg), to_size, ptrue, as_FloatRegister($tmp$$reg), -1);
514 %}
515 ins_pipe(pipe_slow);
516 %}
517 dnl
518 dnl UNARY_OP_TRUE_PREDICATE($1, $2, $3, $4 )
519 dnl UNARY_OP_TRUE_PREDICATE(insn_name, op_name, size, insn)
520 define(`UNARY_OP_TRUE_PREDICATE', `
521 instruct $1(vReg dst, vReg src) %{
522 predicate(UseSVE > 0 &&
523 !n->as_Vector()->is_predicated_vector());
524 match(Set dst ($2 src));
525 ins_cost(SVE_COST);
526 format %{ "$4 $dst, $src\t# vector (sve) ($3)" %}
527 ins_encode %{
528 __ $4(as_FloatRegister($dst$$reg), __ $3,
529 ptrue, as_FloatRegister($src$$reg));
530 %}
531 ins_pipe(pipe_slow);
532 %}')dnl
533 dnl
534
535 // vector abs
536 UNARY_OP_TRUE_PREDICATE(vabsB, AbsVB, B, sve_abs)
537 UNARY_OP_TRUE_PREDICATE(vabsS, AbsVS, H, sve_abs)
538 UNARY_OP_TRUE_PREDICATE(vabsI, AbsVI, S, sve_abs)
539 UNARY_OP_TRUE_PREDICATE(vabsL, AbsVL, D, sve_abs)
540 UNARY_OP_TRUE_PREDICATE(vabsF, AbsVF, S, sve_fabs)
541 UNARY_OP_TRUE_PREDICATE(vabsD, AbsVD, D, sve_fabs)
542
543 dnl UNARY_OP_PREDICATE($1, $2, $3, $4 )
544 dnl UNARY_OP_PREDICATE(insn_name, op_name, size, insn)
545 define(`UNARY_OP_PREDICATE', `
546 instruct $1_masked(vReg dst_src, pRegGov pg) %{
547 predicate(UseSVE > 0);
548 match(Set dst_src ($2 dst_src pg));
549 ins_cost(SVE_COST);
550 format %{ "$4 $dst_src, $pg, $dst_src\t# vector (sve) ($3)" %}
551 ins_encode %{
552 __ $4(as_FloatRegister($dst_src$$reg), __ $3,
553 as_PRegister($pg$$reg),
554 as_FloatRegister($dst_src$$reg));
555 %}
556 ins_pipe(pipe_slow);
557 %}')dnl
558 // vector abs - predicated
559 UNARY_OP_PREDICATE(vabsB, AbsVB, B, sve_abs)
560 UNARY_OP_PREDICATE(vabsS, AbsVS, H, sve_abs)
561 UNARY_OP_PREDICATE(vabsI, AbsVI, S, sve_abs)
562 UNARY_OP_PREDICATE(vabsL, AbsVL, D, sve_abs)
563 UNARY_OP_PREDICATE(vabsF, AbsVF, S, sve_fabs)
564 UNARY_OP_PREDICATE(vabsD, AbsVD, D, sve_fabs)
565
566 dnl
567 dnl BINARY_OP_UNPREDICATE($1, $2 $3, $4 $5 )
568 dnl BINARY_OP_UNPREDICATE(insn_name, op_name, size, min_vec_len, insn)
569 define(`BINARY_OP_UNPREDICATE', `
570 instruct $1(vReg dst, vReg src1, vReg src2) %{
571 predicate(UseSVE > 0);
572 match(Set dst ($2 src1 src2));
573 ins_cost(SVE_COST);
574 format %{ "$5 $dst, $src1, $src2\t # vector (sve) ($3)" %}
575 ins_encode %{
576 __ $5(as_FloatRegister($dst$$reg), __ $3,
577 as_FloatRegister($src1$$reg),
578 as_FloatRegister($src2$$reg));
579 %}
580 ins_pipe(pipe_slow);
581 %}')dnl
582 dnl
583 dnl
584 dnl BINARY_OP_PREDICATE($1, $2, $3, $4 )
585 dnl BINARY_OP_PREDICATE(insn_name, op_name, size, insn)
586 define(`BINARY_OP_PREDICATE', `
587 instruct $1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
588 predicate(UseSVE > 0);
589 match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
590 ins_cost(SVE_COST);
591 format %{ "$4 $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) ($3)" %}
592 ins_encode %{
593 __ $4(as_FloatRegister($dst_src1$$reg), __ $3,
594 as_PRegister($pg$$reg),
595 as_FloatRegister($src2$$reg));
596 %}
597 ins_pipe(pipe_slow);
598 %}')dnl
599 dnl
600 // vector add
601 BINARY_OP_UNPREDICATE(vaddB, AddVB, B, 16, sve_add)
602 BINARY_OP_UNPREDICATE(vaddS, AddVS, H, 8, sve_add)
603 BINARY_OP_UNPREDICATE(vaddI, AddVI, S, 4, sve_add)
604 BINARY_OP_UNPREDICATE(vaddL, AddVL, D, 2, sve_add)
605 BINARY_OP_UNPREDICATE(vaddF, AddVF, S, 4, sve_fadd)
606 BINARY_OP_UNPREDICATE(vaddD, AddVD, D, 2, sve_fadd)
607
608 // vector add - predicated
609 BINARY_OP_PREDICATE(vaddB, AddVB, B, sve_add)
610 BINARY_OP_PREDICATE(vaddS, AddVS, H, sve_add)
611 BINARY_OP_PREDICATE(vaddI, AddVI, S, sve_add)
612 BINARY_OP_PREDICATE(vaddL, AddVL, D, sve_add)
613 BINARY_OP_PREDICATE(vaddF, AddVF, S, sve_fadd)
614 BINARY_OP_PREDICATE(vaddD, AddVD, D, sve_fadd)
615 dnl
616 dnl ADD_IMM($1, $2, $3 )
617 dnl ADD_IMM(name_suffix, size, imm_type)
618 define(`ADD_IMM', `
619 instruct vaddImm$1(vReg dst_src, $3 con) %{
620 predicate(UseSVE > 0);
621 match(Set dst_src (AddV$1 dst_src (Replicate$1 con)));
622 ins_cost(SVE_COST);
623 format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) ($2)" %}
624 ins_encode %{
625 int32_t val = $con$$constant;
626 if (val > 0){
627 __ sve_add(as_FloatRegister($dst_src$$reg), __ $2, val);
628 } else if (val < 0){
629 __ sve_sub(as_FloatRegister($dst_src$$reg), __ $2, -val);
630 }
631 %}
632 ins_pipe(pipe_slow);
633 %}')dnl
634
635 // vector add reg imm (unpredicated)
636 ADD_IMM(B, B, immBAddSubV)
637 ADD_IMM(S, H, immIAddSubV)
638 ADD_IMM(I, S, immIAddSubV)
639 ADD_IMM(L, D, immLAddSubV)
640 dnl
641 dnl BITWISE_OP_IMM($1, $2 $3, $4 $5 )
642 dnl BITWISE_OP_IMM(insn_name, op_name1, size, type, op_name2)
643 define(`BITWISE_OP_IMM', `
644 instruct $1(vReg dst_src, imm$4Log con) %{
645 predicate(UseSVE > 0);
646 match(Set dst_src ($2 dst_src (Replicate$4 con)));
647 ins_cost(SVE_COST);
648 format %{ "$5 $dst_src, $dst_src, $con\t # vector (sve) ($3)" %}
649 ins_encode %{
650 __ $5(as_FloatRegister($dst_src$$reg), __ $3,
651 (uint64_t)($con$$constant));
652 %}
653 ins_pipe(pipe_slow);
654 %}')dnl
655
656 // vector binary op reg imm (unpredicated)
657 BITWISE_OP_IMM(vandB, AndV, B, B, sve_and)
658 BITWISE_OP_IMM(vandH, AndV, H, S, sve_and)
659 BITWISE_OP_IMM(vandS, AndV, S, I, sve_and)
660 BITWISE_OP_IMM(vandD, AndV, D, L, sve_and)
661 BITWISE_OP_IMM(vorB, OrV, B, B, sve_orr)
662 BITWISE_OP_IMM(vorH, OrV, H, S, sve_orr)
663 BITWISE_OP_IMM(vorS, OrV, S, I, sve_orr)
664 BITWISE_OP_IMM(vorD, OrV, D, L, sve_orr)
665 BITWISE_OP_IMM(vxorB, XorV, B, B, sve_eor)
666 BITWISE_OP_IMM(vxorH, XorV, H, S, sve_eor)
667 BITWISE_OP_IMM(vxorS, XorV, S, I, sve_eor)
668 BITWISE_OP_IMM(vxorD, XorV, D, L, sve_eor)
669 dnl
670 dnl
671 dnl BINARY_OP_UNSIZED($1, $2, $3 )
672 dnl BINARY_OP_UNSIZED(insn_name, op_name, insn)
673 define(`BINARY_OP_UNSIZED', `
674 instruct $1(vReg dst, vReg src1, vReg src2) %{
675 predicate(UseSVE > 0);
676 match(Set dst ($2 src1 src2));
677 ins_cost(SVE_COST);
678 format %{ "$3 $dst, $src1, $src2\t# vector (sve)" %}
679 ins_encode %{
680 __ $3(as_FloatRegister($dst$$reg),
681 as_FloatRegister($src1$$reg),
682 as_FloatRegister($src2$$reg));
683 %}
684 ins_pipe(pipe_slow);
685 %}')dnl
686 dnl
687 // vector and
688 BINARY_OP_UNSIZED(vand, AndV, sve_and)
689
690 // vector or
691 BINARY_OP_UNSIZED(vor, OrV, sve_orr)
692
693 // vector xor
694 BINARY_OP_UNSIZED(vxor, XorV, sve_eor)
695
696 dnl BINARY_LOGIC_OP_PREDICATE($1, $2, $3 )
697 dnl BINARY_LOGIC_OP_PREDICATE(insn_name, op_name, insn)
698 define(`BINARY_LOGIC_OP_PREDICATE', `
699 instruct $1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
700 predicate(UseSVE > 0);
701 match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
702 ins_cost(SVE_COST);
703 format %{ "$3 $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
704 ins_encode %{
705 BasicType bt = Matcher::vector_element_basic_type(this);
706 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
707 __ $3(as_FloatRegister($dst_src1$$reg), size,
708 as_PRegister($pg$$reg),
709 as_FloatRegister($src2$$reg));
710 %}
711 ins_pipe(pipe_slow);
712 %}')dnl
713 dnl
714 // vector and - predicated
715 BINARY_LOGIC_OP_PREDICATE(vand, AndV, sve_and)
716
717 // vector or - predicated
718 BINARY_LOGIC_OP_PREDICATE(vor, OrV, sve_orr)
719
720 // vector xor - predicated
721 BINARY_LOGIC_OP_PREDICATE(vxor, XorV, sve_eor)
722
723 // vector not
724 dnl
725 define(`MATCH_RULE', `ifelse($1, I,
726 `match(Set dst (XorV src (ReplicateB m1)));
727 match(Set dst (XorV src (ReplicateS m1)));
728 match(Set dst (XorV src (ReplicateI m1)));',
729 `match(Set dst (XorV src (ReplicateL m1)));')')dnl
730 dnl
731 define(`VECTOR_NOT', `
732 instruct vnot$1`'(vReg dst, vReg src, imm$1_M1 m1) %{
733 predicate(UseSVE > 0);
734 MATCH_RULE($1)
735 ins_cost(SVE_COST);
736 format %{ "sve_not $dst, $src\t# vector (sve) $2" %}
737 ins_encode %{
738 __ sve_not(as_FloatRegister($dst$$reg), __ D,
739 ptrue, as_FloatRegister($src$$reg));
740 %}
741 ins_pipe(pipe_slow);
742 %}')dnl
743 dnl $1,$2
744 VECTOR_NOT(I, B/H/S)
745 VECTOR_NOT(L, D)
746 undefine(MATCH_RULE)
747 dnl
748 // vector and_not
749 dnl
750 define(`MATCH_RULE', `ifelse($1, I,
751 `match(Set dst (AndV src1 (XorV src2 (ReplicateB m1))));
752 match(Set dst (AndV src1 (XorV src2 (ReplicateS m1))));
753 match(Set dst (AndV src1 (XorV src2 (ReplicateI m1))));',
754 `match(Set dst (AndV src1 (XorV src2 (ReplicateL m1))));')')dnl
755 dnl
756 define(`VECTOR_AND_NOT', `
757 instruct vand_not$1`'(vReg dst, vReg src1, vReg src2, imm$1_M1 m1) %{
758 predicate(UseSVE > 0);
759 MATCH_RULE($1)
760 ins_cost(SVE_COST);
761 format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) $2" %}
762 ins_encode %{
763 __ sve_bic(as_FloatRegister($dst$$reg),
764 as_FloatRegister($src1$$reg),
765 as_FloatRegister($src2$$reg));
766 %}
767 ins_pipe(pipe_slow);
768 %}')dnl
769 dnl $1,$2
770 VECTOR_AND_NOT(I, B/H/S)
771 VECTOR_AND_NOT(L, D)
772 undefine(MATCH_RULE)
773 dnl
774 dnl VDIVF($1, $2 , $3 )
775 dnl VDIVF(name_suffix, size, min_vec_len)
776 define(`VDIVF', `
777 instruct vdiv$1(vReg dst_src1, vReg src2) %{
778 predicate(UseSVE > 0);
779 match(Set dst_src1 (DivV$1 dst_src1 src2));
780 ins_cost(SVE_COST);
781 format %{ "sve_fdiv $dst_src1, $dst_src1, $src2\t# vector (sve) ($2)" %}
782 ins_encode %{
783 __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ $2,
784 ptrue, as_FloatRegister($src2$$reg));
785 %}
786 ins_pipe(pipe_slow);
787 %}')dnl
788 dnl
789 // vector float div
790 VDIVF(F, S, 4)
791 VDIVF(D, D, 2)
792
793 // vector float div - predicated
794 BINARY_OP_PREDICATE(vfdivF, DivVF, S, sve_fdiv)
795 BINARY_OP_PREDICATE(vfdivD, DivVD, D, sve_fdiv)
796
797 dnl
798 dnl VMINMAX($1 , $2, $3 , $4 )
799 dnl VMINMAX(op_name, op, finsn, insn)
800 define(`VMINMAX', `
801 instruct v$1(vReg dst_src1, vReg src2) %{
802 predicate(UseSVE > 0);
803 match(Set dst_src1 ($2 dst_src1 src2));
804 ins_cost(SVE_COST);
805 format %{ "sve_$1 $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
806 ins_encode %{
807 BasicType bt = Matcher::vector_element_basic_type(this);
808 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
809 if (is_floating_point_type(bt)) {
810 __ $3(as_FloatRegister($dst_src1$$reg), size,
811 ptrue, as_FloatRegister($src2$$reg));
812 } else {
813 assert(is_integral_type(bt), "unsupported type");
814 __ $4(as_FloatRegister($dst_src1$$reg), size,
815 ptrue, as_FloatRegister($src2$$reg));
816 }
817 %}
818 ins_pipe(pipe_slow);
819 %}')dnl
820 dnl
821 // vector min/max
822 VMINMAX(min, MinV, sve_fmin, sve_smin)
823 VMINMAX(max, MaxV, sve_fmax, sve_smax)
824
825 dnl
826 dnl VMINMAX_PREDICATE($1 , $2, $3 , $4 )
827 dnl VMINMAX_PREDICATE(op_name, op, finsn, insn)
828 define(`VMINMAX_PREDICATE', `
829 instruct v$1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
830 predicate(UseSVE > 0);
831 match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
832 ins_cost(SVE_COST);
833 format %{ "sve_$1 $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %}
834 ins_encode %{
835 BasicType bt = Matcher::vector_element_basic_type(this);
836 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
837 if (is_floating_point_type(bt)) {
838 __ $3(as_FloatRegister($dst_src1$$reg), size,
839 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
840 } else {
841 assert(is_integral_type(bt), "unsupported type");
842 __ $4(as_FloatRegister($dst_src1$$reg), size,
843 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
844 }
845 %}
846 ins_pipe(pipe_slow);
847 %}')dnl
848 dnl
849 // vector min/max - predicated
850 VMINMAX_PREDICATE(min, MinV, sve_fmin, sve_smin)
851 VMINMAX_PREDICATE(max, MaxV, sve_fmax, sve_smax)
852
853 dnl
854 dnl VFMLA($1 $2 $3 )
855 dnl VFMLA(name_suffix, size, min_vec_len)
856 define(`VFMLA', `
857 // dst_src1 = dst_src1 + src2 * src3
858 instruct vfmla$1(vReg dst_src1, vReg src2, vReg src3) %{
859 predicate(UseFMA && UseSVE > 0);
860 match(Set dst_src1 (FmaV$1 dst_src1 (Binary src2 src3)));
861 ins_cost(SVE_COST);
862 format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
863 ins_encode %{
864 __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ $2,
865 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
866 %}
867 ins_pipe(pipe_slow);
868 %}')dnl
869 dnl
870 // vector fmla
871 VFMLA(F, S, 4)
872 VFMLA(D, D, 2)
873
874 dnl
875 dnl VFMLA_PREDICATE($1, $2 )
876 dnl VFMLA_PREDICATE(type, size)
877 define(`VFMLA_PREDICATE', `
878 // dst_src1 = dst_src1 * src2 + src3
879 instruct vfmla$1_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
880 predicate(UseFMA && UseSVE > 0);
881 match(Set dst_src1 (FmaV$1 (Binary dst_src1 src2) (Binary src3 pg)));
882 ins_cost(SVE_COST);
883 format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) ($2)" %}
884 ins_encode %{
885 __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ $2, as_PRegister($pg$$reg),
886 as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
887 %}
888 ins_pipe(pipe_slow);
889 %}')dnl
890 dnl
891 // vector fmla - predicated
892 VFMLA_PREDICATE(F, S)
893 VFMLA_PREDICATE(D, D)
894
895 dnl
896 dnl VFMLS($1 $2 $3 )
897 dnl VFMLS(name_suffix, size, min_vec_len)
898 define(`VFMLS', `
899 // dst_src1 = dst_src1 + -src2 * src3
900 // dst_src1 = dst_src1 + src2 * -src3
901 instruct vfmls$1(vReg dst_src1, vReg src2, vReg src3) %{
902 predicate(UseFMA && UseSVE > 0);
903 match(Set dst_src1 (FmaV$1 dst_src1 (Binary (NegV$1 src2) src3)));
904 match(Set dst_src1 (FmaV$1 dst_src1 (Binary src2 (NegV$1 src3))));
905 ins_cost(SVE_COST);
906 format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
907 ins_encode %{
908 __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ $2,
909 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
910 %}
911 ins_pipe(pipe_slow);
912 %}')dnl
913 dnl
914 // vector fmls
915 VFMLS(F, S, 4)
916 VFMLS(D, D, 2)
917
918 dnl
919 dnl VFNMLA($1 $2 $3 )
920 dnl VFNMLA(name_suffix, size, min_vec_len)
921 define(`VFNMLA', `
922 // dst_src1 = -dst_src1 + -src2 * src3
923 // dst_src1 = -dst_src1 + src2 * -src3
924 instruct vfnmla$1(vReg dst_src1, vReg src2, vReg src3) %{
925 predicate(UseFMA && UseSVE > 0);
926 match(Set dst_src1 (FmaV$1 (NegV$1 dst_src1) (Binary (NegV$1 src2) src3)));
927 match(Set dst_src1 (FmaV$1 (NegV$1 dst_src1) (Binary src2 (NegV$1 src3))));
928 ins_cost(SVE_COST);
929 format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
930 ins_encode %{
931 __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ $2,
932 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
933 %}
934 ins_pipe(pipe_slow);
935 %}')dnl
936 dnl
937 // vector fnmla
938 VFNMLA(F, S, 4)
939 VFNMLA(D, D, 2)
940
941 dnl
942 dnl VFNMLS($1 $2 $3 )
943 dnl VFNMLS(name_suffix, size, min_vec_len)
944 define(`VFNMLS', `
945 // dst_src1 = -dst_src1 + src2 * src3
946 instruct vfnmls$1(vReg dst_src1, vReg src2, vReg src3) %{
947 predicate(UseFMA && UseSVE > 0);
948 match(Set dst_src1 (FmaV$1 (NegV$1 dst_src1) (Binary src2 src3)));
949 ins_cost(SVE_COST);
950 format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) ($2)" %}
951 ins_encode %{
952 __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ $2,
953 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
954 %}
955 ins_pipe(pipe_slow);
956 %}')dnl
957 dnl
958 // vector fnmls
959 VFNMLS(F, S, 4)
960 VFNMLS(D, D, 2)
961
962 dnl
963 dnl VMLA($1 $2 $3 )
964 dnl VMLA(name_suffix, size, min_vec_len)
965 define(`VMLA', `
966 // dst_src1 = dst_src1 + src2 * src3
967 instruct vmla$1(vReg dst_src1, vReg src2, vReg src3)
968 %{
969 predicate(UseSVE > 0);
970 match(Set dst_src1 (AddV$1 dst_src1 (MulV$1 src2 src3)));
971 ins_cost(SVE_COST);
972 format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) ($2)" %}
973 ins_encode %{
974 __ sve_mla(as_FloatRegister($dst_src1$$reg), __ $2,
975 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
976 %}
977 ins_pipe(pipe_slow);
978 %}')dnl
979 dnl
980 // vector mla
981 VMLA(B, B, 16)
982 VMLA(S, H, 8)
983 VMLA(I, S, 4)
984 VMLA(L, D, 2)
985
986 dnl
987 dnl VMLS($1 $2 $3 )
988 dnl VMLS(name_suffix, size, min_vec_len)
989 define(`VMLS', `
990 // dst_src1 = dst_src1 - src2 * src3
991 instruct vmls$1(vReg dst_src1, vReg src2, vReg src3)
992 %{
993 predicate(UseSVE > 0);
994 match(Set dst_src1 (SubV$1 dst_src1 (MulV$1 src2 src3)));
995 ins_cost(SVE_COST);
996 format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) ($2)" %}
997 ins_encode %{
998 __ sve_mls(as_FloatRegister($dst_src1$$reg), __ $2,
999 ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
1000 %}
1001 ins_pipe(pipe_slow);
1002 %}')dnl
1003 dnl
1004 // vector mls
1005 VMLS(B, B, 16)
1006 VMLS(S, H, 8)
1007 VMLS(I, S, 4)
1008 VMLS(L, D, 2)
1009
1010 dnl
1011 dnl BINARY_OP_TRUE_PREDICATE($1, $2, $3, $4, $5 )
1012 dnl BINARY_OP_TRUE_PREDICATE(insn_name, op_name, size, min_vec_len, insn)
1013 define(`BINARY_OP_TRUE_PREDICATE', `
1014 instruct $1(vReg dst_src1, vReg src2) %{
1015 predicate(UseSVE > 0);
1016 match(Set dst_src1 ($2 dst_src1 src2));
1017 ins_cost(SVE_COST);
1018 format %{ "$5 $dst_src1, $dst_src1, $src2\t # vector (sve) ($3)" %}
1019 ins_encode %{
1020 __ $5(as_FloatRegister($dst_src1$$reg), __ $3,
1021 ptrue, as_FloatRegister($src2$$reg));
1022 %}
1023 ins_pipe(pipe_slow);
1024 %}')dnl
1025 dnl
1026 // vector mul
1027 BINARY_OP_TRUE_PREDICATE(vmulB, MulVB, B, 16, sve_mul)
1028 BINARY_OP_TRUE_PREDICATE(vmulS, MulVS, H, 8, sve_mul)
1029 BINARY_OP_TRUE_PREDICATE(vmulI, MulVI, S, 4, sve_mul)
1030 BINARY_OP_TRUE_PREDICATE(vmulL, MulVL, D, 2, sve_mul)
1031 BINARY_OP_UNPREDICATE(vmulF, MulVF, S, 4, sve_fmul)
1032 BINARY_OP_UNPREDICATE(vmulD, MulVD, D, 2, sve_fmul)
1033
1034 // vector mul - predicated
1035 BINARY_OP_PREDICATE(vmulB, MulVB, B, sve_mul)
1036 BINARY_OP_PREDICATE(vmulS, MulVS, H, sve_mul)
1037 BINARY_OP_PREDICATE(vmulI, MulVI, S, sve_mul)
1038 BINARY_OP_PREDICATE(vmulL, MulVL, D, sve_mul)
1039 BINARY_OP_PREDICATE(vmulF, MulVF, S, sve_fmul)
1040 BINARY_OP_PREDICATE(vmulD, MulVD, D, sve_fmul)
1041
1042 // vector fneg
1043 UNARY_OP_TRUE_PREDICATE(vnegF, NegVF, S, sve_fneg)
1044 UNARY_OP_TRUE_PREDICATE(vnegD, NegVD, D, sve_fneg)
1045
1046 // vector fneg - predicated
1047 UNARY_OP_PREDICATE(vnegF, NegVF, S, sve_fneg)
1048 UNARY_OP_PREDICATE(vnegD, NegVD, D, sve_fneg)
1049
1050 // popcount vector
1051
1052 instruct vpopcountI(vReg dst, vReg src) %{
1053 predicate(UseSVE > 0);
1054 match(Set dst (PopCountVI src));
1055 format %{ "sve_cnt $dst, $src\t# vector (sve) (S)\n\t" %}
1056 ins_encode %{
1057 __ sve_cnt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg));
1058 %}
1059 ins_pipe(pipe_slow);
1060 %}
1061
1062 // vector mask compare
1063
1064 instruct vmaskcmp(pRegGov dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %{
1065 predicate(UseSVE > 0);
1066 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
1067 effect(KILL cr);
1068 ins_cost(SVE_COST);
1069 format %{ "sve_cmp $dst, $src1, $src2\t# vector mask cmp (sve)" %}
1070 ins_encode %{
1071 BasicType bt = Matcher::vector_element_basic_type(this);
1072 __ sve_compare(as_PRegister($dst$$reg), bt, ptrue, as_FloatRegister($src1$$reg),
1073 as_FloatRegister($src2$$reg), (int)$cond$$constant);
1074 %}
1075 ins_pipe(pipe_slow);
1076 %}
1077
1078 instruct vmaskcmp_masked(pRegGov dst, vReg src1, vReg src2, immI cond, pRegGov pg, rFlagsReg cr) %{
1079 predicate(UseSVE > 0);
1080 match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond pg)));
1081 effect(KILL cr);
1082 ins_cost(SVE_COST);
1083 format %{ "sve_cmp $dst, $pg, $src1, $src2\t# vector mask cmp (sve)" %}
1084 ins_encode %{
1085 BasicType bt = Matcher::vector_element_basic_type(this);
1086 __ sve_compare(as_PRegister($dst$$reg), bt, as_PRegister($pg$$reg), as_FloatRegister($src1$$reg),
1087 as_FloatRegister($src2$$reg), (int)$cond$$constant);
1088 %}
1089 ins_pipe(pipe_slow);
1090 %}
1091
1092 // vector blend
1093
1094 instruct vblend(vReg dst, vReg src1, vReg src2, pRegGov pg) %{
1095 predicate(UseSVE > 0);
1096 match(Set dst (VectorBlend (Binary src1 src2) pg));
1097 ins_cost(SVE_COST);
1098 format %{ "sve_sel $dst, $pg, $src2, $src1\t# vector blend (sve)" %}
1099 ins_encode %{
1100 Assembler::SIMD_RegVariant size =
1101 __ elemType_to_regVariant(Matcher::vector_element_basic_type(this));
1102 __ sve_sel(as_FloatRegister($dst$$reg), size, as_PRegister($pg$$reg),
1103 as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
1104 %}
1105 ins_pipe(pipe_slow);
1106 %}
1107
1108 // vector load mask
1109
1110 instruct vloadmaskB(pRegGov dst, vReg src, rFlagsReg cr) %{
1111 predicate(UseSVE > 0 &&
1112 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
1113 match(Set dst (VectorLoadMask src));
1114 effect(KILL cr);
1115 ins_cost(SVE_COST);
1116 format %{ "vloadmaskB $dst, $src\t# vector load mask (sve) (B)" %}
1117 ins_encode %{
1118 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ B,
1119 ptrue, as_FloatRegister($src$$reg), 0);
1120 %}
1121 ins_pipe(pipe_slow);
1122 %}
1123
1124 instruct vloadmask_extend(pRegGov dst, vReg src, vReg tmp, rFlagsReg cr) %{
1125 predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() != T_BYTE);
1126 match(Set dst (VectorLoadMask src));
1127 effect(TEMP tmp, KILL cr);
1128 ins_cost(3 * SVE_COST);
1129 format %{ "vloadmask $dst, $src\t# vector load mask (sve) (H/S/D)" %}
1130 ins_encode %{
1131 BasicType bt = Matcher::vector_element_basic_type(this);
1132 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
1133 __ sve_vector_extend(as_FloatRegister($tmp$$reg), size, as_FloatRegister($src$$reg), __ B);
1134 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
1135 %}
1136 ins_pipe(pipe_slow);
1137 %}
1138
1139 // vector store mask
1140
1141 instruct vstoremaskB(vReg dst, pRegGov src, immI_1 size) %{
1142 predicate(UseSVE > 0);
1143 match(Set dst (VectorStoreMask src size));
1144 ins_cost(SVE_COST);
1145 format %{ "vstoremask $dst, $src\t# vector store mask (sve) (B)" %}
1146 ins_encode %{
1147 __ sve_cpy(as_FloatRegister($dst$$reg), __ B, as_PRegister($src$$reg), 1, false);
1148 %}
1149 ins_pipe(pipe_slow);
1150 %}
1151
1152 instruct vstoremask_narrow(vReg dst, pRegGov src, vReg tmp, immI_gt_1 size) %{
1153 predicate(UseSVE > 0);
1154 match(Set dst (VectorStoreMask src size));
1155 effect(TEMP_DEF dst, TEMP tmp);
1156 ins_cost(3 * SVE_COST);
1157 format %{ "vstoremask $dst, $src\t# vector store mask (sve) (H/S/D)" %}
1158 ins_encode %{
1159 Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant((int)$size$$constant);
1160 __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($src$$reg), 1, false);
1161 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
1162 as_FloatRegister($dst$$reg), size, as_FloatRegister($tmp$$reg));
1163 %}
1164 ins_pipe(pipe_slow);
1165 %}
1166
1167 // Combine LoadVector+VectorLoadMask when the vector element type is not T_BYTE
1168
1169 instruct vloadmask_loadV(pRegGov dst, indirect mem, vReg tmp, rFlagsReg cr) %{
1170 predicate(UseSVE > 0 &&
1171 n->as_Vector()->length_in_bytes() == MaxVectorSize &&
1172 type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
1173 match(Set dst (VectorLoadMask (LoadVector mem)));
1174 effect(TEMP tmp, KILL cr);
1175 ins_cost(3 * SVE_COST);
1176 format %{ "sve_ld1b $tmp, $mem\n\t"
1177 "sve_cmpne $dst, $tmp, 0\t# load vector mask (sve) (H/S/D)" %}
1178 ins_encode %{
1179 // Load mask values which are boolean type, and extend them to the
1180 // expected vector element type. Convert the vector to predicate.
1181 BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
1182 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($tmp$$reg),
1183 ptrue, T_BOOLEAN, to_vect_bt, $mem->opcode(),
1184 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1185 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ elemType_to_regVariant(to_vect_bt),
1186 ptrue, as_FloatRegister($tmp$$reg), 0);
1187 %}
1188 ins_pipe(pipe_slow);
1189 %}
1190
1191 instruct vloadmask_loadV_partial(pRegGov dst, indirect mem, vReg vtmp, pRegGov ptmp, rFlagsReg cr) %{
1192 predicate(UseSVE > 0 &&
1193 n->as_Vector()->length_in_bytes() > 16 &&
1194 n->as_Vector()->length_in_bytes() < MaxVectorSize &&
1195 type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
1196 match(Set dst (VectorLoadMask (LoadVector mem)));
1197 effect(TEMP vtmp, TEMP ptmp, KILL cr);
1198 ins_cost(6 * SVE_COST);
1199 format %{ "vloadmask_loadV $dst, $mem\t# load vector mask partial (sve) (H/S/D)" %}
1200 ins_encode %{
1201 // Load valid mask values which are boolean type, and extend them to the
1202 // expected vector element type. Convert the vector to predicate.
1203 BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
1204 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(to_vect_bt);
1205 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this));
1206 loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($vtmp$$reg),
1207 as_PRegister($ptmp$$reg), T_BOOLEAN, to_vect_bt, $mem->opcode(),
1208 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1209 __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($vtmp$$reg), 0);
1210 %}
1211 ins_pipe(pipe_slow);
1212 %}
1213
1214 // Combine VectorStoreMask+StoreVector when the vector element type is not T_BYTE
1215
1216 instruct storeV_vstoremask(indirect mem, pRegGov src, vReg tmp, immI_gt_1 esize) %{
1217 predicate(UseSVE > 0 &&
1218 Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
1219 match(Set mem (StoreVector mem (VectorStoreMask src esize)));
1220 effect(TEMP tmp);
1221 ins_cost(3 * SVE_COST);
1222 format %{ "sve_cpy $tmp, $src, 1\n\t"
1223 "sve_st1b $tmp, $mem\t# store vector mask (sve) (H/S/D)" %}
1224 ins_encode %{
1225 BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
1226 assert(type2aelembytes(from_vect_bt) == (int)$esize$$constant, "unsupported type.");
1227 Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
1228 __ sve_cpy(as_FloatRegister($tmp$$reg), size, as_PRegister($src$$reg), 1, false);
1229 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($tmp$$reg),
1230 ptrue, T_BOOLEAN, from_vect_bt, $mem->opcode(),
1231 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1232 %}
1233 ins_pipe(pipe_slow);
1234 %}
1235
1236 instruct storeV_vstoremask_partial(indirect mem, pRegGov src, vReg vtmp,
1237 immI_gt_1 esize, pRegGov ptmp, rFlagsReg cr) %{
1238 predicate(UseSVE > 0 &&
1239 n->as_StoreVector()->memory_size() > 16 &&
1240 type2aelembytes(n->as_StoreVector()->vect_type()->element_basic_type()) > 1 &&
1241 Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
1242 match(Set mem (StoreVector mem (VectorStoreMask src esize)));
1243 effect(TEMP vtmp, TEMP ptmp, KILL cr);
1244 format %{ "storeV_vstoremask $src, $mem\t# store vector mask partial (sve) (H/S/D)" %}
1245 ins_cost(6 * SVE_COST);
1246 ins_encode %{
1247 // Convert the valid src predicate to vector, and store the vector
1248 // elements as boolean values.
1249 BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
1250 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(from_vect_bt);
1251 __ sve_cpy(as_FloatRegister($vtmp$$reg), size, as_PRegister($src$$reg), 1, false);
1252 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
1253 loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($vtmp$$reg),
1254 as_PRegister($ptmp$$reg), T_BOOLEAN, from_vect_bt, $mem->opcode(),
1255 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
1256 %}
1257 ins_pipe(pipe_slow);
1258 %}
1259 dnl
1260 dnl REDUCE_I($1, $2 )
1261 dnl REDUCE_I(insn_name, op_name)
1262 define(`REDUCE_I', `
1263 instruct reduce_$1I(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
1264 ifelse($2, AddReductionVI,
1265 `predicate(UseSVE > 0 &&
1266 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);',
1267 `predicate(UseSVE > 0 &&
1268 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1269 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);')
1270 match(Set dst ($2 src1 src2));
1271 effect(TEMP_DEF dst, TEMP tmp);
1272 ins_cost(SVE_COST);
1273 format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction (sve) (may extend)" %}
1274 ins_encode %{
1275 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1276 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1277 $src1$$Register, as_FloatRegister($src2$$reg),
1278 ptrue, as_FloatRegister($tmp$$reg));
1279 %}
1280 ins_pipe(pipe_slow);
1281 %}')dnl
1282 dnl
1283 dnl
1284 dnl REDUCE_L($1, $2 )
1285 dnl REDUCE_L(insn_name, op_name)
1286 define(`REDUCE_L', `
1287 instruct reduce_$1L(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
1288 ifelse($2, AddReductionVL,
1289 `predicate(UseSVE > 0 &&
1290 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);',
1291 `predicate(UseSVE > 0 &&
1292 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1293 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);')
1294 match(Set dst ($2 src1 src2));
1295 effect(TEMP_DEF dst, TEMP tmp);
1296 ins_cost(SVE_COST);
1297 format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction (sve)" %}
1298 ins_encode %{
1299 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1300 $src1$$Register, as_FloatRegister($src2$$reg),
1301 ptrue, as_FloatRegister($tmp$$reg));
1302 %}
1303 ins_pipe(pipe_slow);
1304 %}')dnl
1305 dnl
1306 dnl REDUCE_I_PARTIAL($1, $2 )
1307 dnl REDUCE_I_PARTIAL(insn_name, op_name)
1308 define(`REDUCE_I_PARTIAL', `
1309 instruct reduce_$1I_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1310 pRegGov ptmp, rFlagsReg cr) %{
1311 ifelse($2, AddReductionVI,
1312 `predicate(UseSVE > 0 &&
1313 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);',
1314 `predicate(UseSVE > 0 &&
1315 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1316 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);')
1317 match(Set dst ($2 src1 src2));
1318 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1319 ins_cost(2 * SVE_COST);
1320 format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction partial (sve) (may extend)" %}
1321 ins_encode %{
1322 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1323 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1324 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1325 Matcher::vector_length(this, $src2));
1326 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1327 $src1$$Register, as_FloatRegister($src2$$reg),
1328 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1329 %}
1330 ins_pipe(pipe_slow);
1331 %}')dnl
1332 dnl
1333 dnl REDUCE_L_PARTIAL($1, $2 )
1334 dnl REDUCE_L_PARTIAL(insn_name, op_name)
1335 define(`REDUCE_L_PARTIAL', `
1336 instruct reduce_$1L_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1337 pRegGov ptmp, rFlagsReg cr) %{
1338 ifelse($2, AddReductionVL,
1339 `predicate(UseSVE > 0 &&
1340 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);',
1341 `predicate(UseSVE > 0 &&
1342 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1343 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);')
1344 match(Set dst ($2 src1 src2));
1345 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1346 ins_cost(2 * SVE_COST);
1347 format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction partial (sve)" %}
1348 ins_encode %{
1349 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1350 Matcher::vector_length(this, $src2));
1351 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1352 $src1$$Register, as_FloatRegister($src2$$reg),
1353 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1354 %}
1355 ins_pipe(pipe_slow);
1356 %}')dnl
1357 dnl
1358 dnl REDUCE_ADDF($1, $2, $3, $4 )
1359 dnl REDUCE_ADDF(insn_name, op_name, reg_dst, size)
1360 define(`REDUCE_ADDF', `
1361 instruct reduce_$1($3 src1_dst, vReg src2) %{
1362 predicate(UseSVE > 0 &&
1363 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1364 match(Set src1_dst ($2 src1_dst src2));
1365 ins_cost(SVE_COST);
1366 format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) ($4)" %}
1367 ins_encode %{
1368 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1369 ptrue, as_FloatRegister($src2$$reg));
1370 %}
1371 ins_pipe(pipe_slow);
1372 %}')dnl
1373 dnl
1374 dnl
1375 dnl REDUCE_ADDF_PARTIAL($1, $2, $3, $4 )
1376 dnl REDUCE_ADDF_PARTIAL(insn_name, suffix, reg_dst, size)
1377 define(`REDUCE_ADDF_PARTIAL', `
1378 instruct reduce_$1_partial($3 src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
1379 predicate(UseSVE > 0 &&
1380 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1381 match(Set src1_dst ($2 src1_dst src2));
1382 ins_cost(SVE_COST);
1383 effect(TEMP ptmp, KILL cr);
1384 format %{ "sve_reduce_$1 $src1_dst, $src1_dst, $src2\t# $1 reduction partial (sve) ($4)" %}
1385 ins_encode %{
1386 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
1387 Matcher::vector_length(this, $src2));
1388 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1389 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1390 %}
1391 ins_pipe(pipe_slow);
1392 %}')dnl
1393 dnl
1394 dnl
1395 dnl REDUCE_I_PREDICATE($1, $2 )
1396 dnl REDUCE_I_PREDICATE(insn_name, op_name)
1397 define(`REDUCE_I_PREDICATE', `
1398 instruct reduce_$1I_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
1399 ifelse($2, AddReductionVI,
1400 `predicate(UseSVE > 0 &&
1401 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);',
1402 `predicate(UseSVE > 0 &&
1403 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1404 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);')
1405 match(Set dst ($2 (Binary src1 src2) pg));
1406 effect(TEMP_DEF dst, TEMP tmp);
1407 ins_cost(SVE_COST);
1408 format %{ "sve_reduce_$1I $dst, $src1, $pg, $src2\t# $1I reduction predicated (sve) (may extend)" %}
1409 ins_encode %{
1410 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1411 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1412 $src1$$Register, as_FloatRegister($src2$$reg),
1413 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
1414 %}
1415 ins_pipe(pipe_slow);
1416 %}')dnl
1417 dnl
1418 dnl REDUCE_L_PREDICATE($1, $2 )
1419 dnl REDUCE_L_PREDICATE(insn_name, op_name)
1420 define(`REDUCE_L_PREDICATE', `
1421 instruct reduce_$1L_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
1422 ifelse($2, AddReductionVL,
1423 `predicate(UseSVE > 0 &&
1424 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);',
1425 `predicate(UseSVE > 0 &&
1426 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1427 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);')
1428 match(Set dst ($2 (Binary src1 src2) pg));
1429 effect(TEMP_DEF dst, TEMP tmp);
1430 ins_cost(SVE_COST);
1431 format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated (sve)" %}
1432 ins_encode %{
1433 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1434 $src1$$Register, as_FloatRegister($src2$$reg),
1435 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
1436 %}
1437 ins_pipe(pipe_slow);
1438 %}')dnl
1439 dnl
1440 dnl REDUCE_I_PREDICATE_PARTIAL($1, $2 )
1441 dnl REDUCE_I_PREDICATE_PARTIAL(insn_name, op_name)
1442 define(`REDUCE_I_PREDICATE_PARTIAL', `
1443 instruct reduce_$1I_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1444 pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
1445 ifelse($2, AddReductionVI,
1446 `predicate(UseSVE > 0 &&
1447 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);',
1448 `predicate(UseSVE > 0 &&
1449 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1450 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);')
1451 match(Set dst ($2 (Binary src1 src2) pg));
1452 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1453 ins_cost(3 * SVE_COST);
1454 format %{ "sve_reduce_$1I $dst, $src1, $pg, $src2\t# $1I reduction predicated partial (sve) (may extend)" %}
1455 ins_encode %{
1456 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1457 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1458 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1459 Matcher::vector_length(this, $src2));
1460 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1461 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1462 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1463 $src1$$Register, as_FloatRegister($src2$$reg),
1464 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1465 %}
1466 ins_pipe(pipe_slow);
1467 %}')dnl
1468 dnl
1469 dnl REDUCE_L_PREDICATE_PARTIAL($1, $2 )
1470 dnl REDUCE_L_PREDICATE_PARTIAL(insn_name, op_name)
1471 define(`REDUCE_L_PREDICATE_PARTIAL', `
1472 instruct reduce_$1L_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1473 pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
1474 ifelse($2, AddReductionVL,
1475 `predicate(UseSVE > 0 &&
1476 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);',
1477 `predicate(UseSVE > 0 &&
1478 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
1479 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);')
1480 match(Set dst ($2 (Binary src1 src2) pg));
1481 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1482 ins_cost(3 * SVE_COST);
1483 format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated partial (sve)" %}
1484 ins_encode %{
1485 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1486 Matcher::vector_length(this, $src2));
1487 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1488 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1489 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1490 $src1$$Register, as_FloatRegister($src2$$reg),
1491 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1492 %}
1493 ins_pipe(pipe_slow);
1494 %}')dnl
1495 dnl
1496 dnl REDUCE_ADDF_PREDICATE($1, $2, $3, $4 )
1497 dnl REDUCE_ADDF_PREDICATE(insn_name, op_name, reg_dst, size)
1498 define(`REDUCE_ADDF_PREDICATE', `
1499 instruct reduce_$1_masked($3 src1_dst, vReg src2, pRegGov pg) %{
1500 predicate(UseSVE > 0 &&
1501 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1502 match(Set src1_dst ($2 (Binary src1_dst src2) pg));
1503 ins_cost(SVE_COST);
1504 format %{ "sve_reduce_$1 $src1_dst, $pg, $src2\t# $1 reduction predicated (sve)" %}
1505 ins_encode %{
1506 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1507 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1508 %}
1509 ins_pipe(pipe_slow);
1510 %}')dnl
1511 dnl
1512 dnl REDUCE_ADDF_PREDICATE_PARTIAL($1, $2, $3, $4 )
1513 dnl REDUCE_ADDF_PREDICATE_PARTIAL(insn_name, op_name, reg_dst, size)
1514 define(`REDUCE_ADDF_PREDICATE_PARTIAL', `
1515 instruct reduce_$1_masked_partial($3 src1_dst, vReg src2, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
1516 predicate(UseSVE > 0 &&
1517 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1518 match(Set src1_dst ($2 (Binary src1_dst src2) pg));
1519 effect(TEMP ptmp, KILL cr);
1520 ins_cost(SVE_COST);
1521 format %{ "sve_reduce_$1 $src1_dst, $pg, $src2\t# $1 reduction predicated partial (sve)" %}
1522 ins_encode %{
1523 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
1524 Matcher::vector_length(this, $src2));
1525 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1526 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1527 __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
1528 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1529 %}
1530 ins_pipe(pipe_slow);
1531 %}')dnl
1532 dnl
1533
1534 // vector add reduction
1535 REDUCE_I(add, AddReductionVI)
1536 REDUCE_L(add, AddReductionVL)
1537 REDUCE_ADDF(addF, AddReductionVF, vRegF, S)
1538 REDUCE_ADDF(addD, AddReductionVD, vRegD, D)
1539 REDUCE_I_PARTIAL(add, AddReductionVI)
1540 REDUCE_L_PARTIAL(add, AddReductionVL)
1541 REDUCE_ADDF_PARTIAL(addF, AddReductionVF, vRegF, S)
1542 REDUCE_ADDF_PARTIAL(addD, AddReductionVD, vRegD, D)
1543
1544 // vector add reduction - predicated
1545 REDUCE_I_PREDICATE(add, AddReductionVI)
1546 REDUCE_L_PREDICATE(add, AddReductionVL)
1547 REDUCE_ADDF_PREDICATE(addF, AddReductionVF, vRegF, S)
1548 REDUCE_ADDF_PREDICATE(addD, AddReductionVD, vRegD, D)
1549 REDUCE_I_PREDICATE_PARTIAL(add, AddReductionVI)
1550 REDUCE_L_PREDICATE_PARTIAL(add, AddReductionVL)
1551 REDUCE_ADDF_PREDICATE_PARTIAL(addF, AddReductionVF, vRegF, S)
1552 REDUCE_ADDF_PREDICATE_PARTIAL(addD, AddReductionVD, vRegD, D)
1553
1554 // vector and reduction
1555 REDUCE_I(and, AndReductionV)
1556 REDUCE_L(and, AndReductionV)
1557 REDUCE_I_PARTIAL(and, AndReductionV)
1558 REDUCE_L_PARTIAL(and, AndReductionV)
1559
1560 // vector and reduction - predicated
1561 REDUCE_I_PREDICATE(and, AndReductionV)
1562 REDUCE_L_PREDICATE(and, AndReductionV)
1563 REDUCE_I_PREDICATE_PARTIAL(and, AndReductionV)
1564 REDUCE_L_PREDICATE_PARTIAL(and, AndReductionV)
1565
1566 // vector or reduction
1567 REDUCE_I(or, OrReductionV)
1568 REDUCE_L(or, OrReductionV)
1569 REDUCE_I_PARTIAL(or, OrReductionV)
1570 REDUCE_L_PARTIAL(or, OrReductionV)
1571
1572 // vector or reduction - predicated
1573 REDUCE_I_PREDICATE(or, OrReductionV)
1574 REDUCE_L_PREDICATE(or, OrReductionV)
1575 REDUCE_I_PREDICATE_PARTIAL(or, OrReductionV)
1576 REDUCE_L_PREDICATE_PARTIAL(or, OrReductionV)
1577
1578 // vector xor reduction
1579 REDUCE_I(eor, XorReductionV)
1580 REDUCE_L(eor, XorReductionV)
1581 REDUCE_I_PARTIAL(eor, XorReductionV)
1582 REDUCE_L_PARTIAL(eor, XorReductionV)
1583
1584 // vector xor reduction - predicated
1585 REDUCE_I_PREDICATE(eor, XorReductionV)
1586 REDUCE_L_PREDICATE(eor, XorReductionV)
1587 REDUCE_I_PREDICATE_PARTIAL(eor, XorReductionV)
1588 REDUCE_L_PREDICATE_PARTIAL(eor, XorReductionV)
1589
1590 dnl
1591 dnl REDUCE_MAXMIN_I($1, $2 )
1592 dnl REDUCE_MAXMIN_I(insn_name, op_name)
1593 define(`REDUCE_MAXMIN_I', `
1594 instruct reduce_$1I(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
1595 predicate(UseSVE > 0 &&
1596 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1597 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1598 is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
1599 match(Set dst ($2 src1 src2));
1600 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1601 ins_cost(SVE_COST);
1602 format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction (sve)" %}
1603 ins_encode %{
1604 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1605 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1606 $src1$$Register, as_FloatRegister($src2$$reg),
1607 ptrue, as_FloatRegister($tmp$$reg));
1608 %}
1609 ins_pipe(pipe_slow);
1610 %}')dnl
1611 dnl
1612 dnl REDUCE_MAXMIN_L($1, $2 )
1613 dnl REDUCE_MAXMIN_L(insn_name, op_name)
1614 define(`REDUCE_MAXMIN_L', `
1615 instruct reduce_$1L(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
1616 predicate(UseSVE > 0 &&
1617 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1618 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1619 match(Set dst ($2 src1 src2));
1620 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1621 ins_cost(SVE_COST);
1622 format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction (sve)" %}
1623 ins_encode %{
1624 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1625 $src1$$Register, as_FloatRegister($src2$$reg),
1626 ptrue, as_FloatRegister($tmp$$reg));
1627 %}
1628 ins_pipe(pipe_slow);
1629 %}')dnl
1630 dnl
1631 dnl REDUCE_MAXMIN_I_PARTIAL($1 , $2 )
1632 dnl REDUCE_MAXMIN_I_PARTIAL(min_max, op_name)
1633 define(`REDUCE_MAXMIN_I_PARTIAL', `
1634 instruct reduce_$1I_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1635 pRegGov ptmp, rFlagsReg cr) %{
1636 predicate(UseSVE > 0 &&
1637 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1638 n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1639 is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
1640 match(Set dst ($2 src1 src2));
1641 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1642 ins_cost(2 * SVE_COST);
1643 format %{ "sve_reduce_$1I $dst, $src1, $src2\t# $1I reduction partial (sve)" %}
1644 ins_encode %{
1645 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1646 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1647 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1648 Matcher::vector_length(this, $src2));
1649 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1650 $src1$$Register, as_FloatRegister($src2$$reg),
1651 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1652 %}
1653 ins_pipe(pipe_slow);
1654 %}')dnl
1655 dnl
1656 dnl REDUCE_MAXMIN_L_PARTIAL($1 , $2 )
1657 dnl REDUCE_MAXMIN_L_PARTIAL(min_max, op_name)
1658 define(`REDUCE_MAXMIN_L_PARTIAL', `
1659 instruct reduce_$1L_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1660 pRegGov ptmp, rFlagsReg cr) %{
1661 predicate(UseSVE > 0 &&
1662 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1663 n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1664 match(Set dst ($2 src1 src2));
1665 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1666 ins_cost(2 * SVE_COST);
1667 format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction partial (sve)" %}
1668 ins_encode %{
1669 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1670 Matcher::vector_length(this, $src2));
1671 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1672 $src1$$Register, as_FloatRegister($src2$$reg),
1673 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1674 %}
1675 ins_pipe(pipe_slow);
1676 %}')dnl
1677 dnl
1678 dnl REDUCE_MAXMIN_I_PREDICATE($1 , $2 )
1679 dnl REDUCE_MAXMIN_I_PREDICATE(min_max, op_name)
1680 define(`REDUCE_MAXMIN_I_PREDICATE', `
1681 instruct reduce_$1I_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp,
1682 pRegGov pg, rFlagsReg cr) %{
1683 predicate(UseSVE > 0 &&
1684 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1685 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1686 is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
1687 match(Set dst ($2 (Binary src1 src2) pg));
1688 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1689 ins_cost(SVE_COST);
1690 format %{ "sve_reduce_$1I $dst, $src1, $pg, $src2\t# $1I reduction predicated (sve)" %}
1691 ins_encode %{
1692 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1693 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1694 $src1$$Register, as_FloatRegister($src2$$reg),
1695 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
1696 %}
1697 ins_pipe(pipe_slow);
1698 %}')dnl
1699 dnl
1700 dnl REDUCE_MAXMIN_L_PREDICATE($1 , $2 )
1701 dnl REDUCE_MAXMIN_L_PREDICATE(min_max, op_name)
1702 define(`REDUCE_MAXMIN_L_PREDICATE', `
1703 instruct reduce_$1L_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp,
1704 pRegGov pg, rFlagsReg cr) %{
1705 predicate(UseSVE > 0 &&
1706 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
1707 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1708 match(Set dst ($2 (Binary src1 src2) pg));
1709 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
1710 ins_cost(SVE_COST);
1711 format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated (sve)" %}
1712 ins_encode %{
1713 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1714 $src1$$Register, as_FloatRegister($src2$$reg),
1715 as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
1716 %}
1717 ins_pipe(pipe_slow);
1718 %}')dnl
1719 dnl
1720 dnl REDUCE_MAXMIN_I_PREDICATE_PARTIAL($1 , $2 )
1721 dnl REDUCE_MAXMIN_I_PREDICATE_PARTIAL(min_max, op_name)
1722 define(`REDUCE_MAXMIN_I_PREDICATE_PARTIAL', `
1723 instruct reduce_$1I_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
1724 pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
1725 predicate(UseSVE > 0 &&
1726 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1727 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
1728 is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
1729 match(Set dst ($2 (Binary src1 src2) pg));
1730 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1731 ins_cost(3 * SVE_COST);
1732 format %{ "sve_reduce_$1I $dst, $src1, $pg, $src2\t# $1I reduction predicated partial (sve)" %}
1733 ins_encode %{
1734 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
1735 Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
1736 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
1737 Matcher::vector_length(this, $src2));
1738 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1739 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1740 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
1741 $src1$$Register, as_FloatRegister($src2$$reg),
1742 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1743 %}
1744 ins_pipe(pipe_slow);
1745 %}')dnl
1746 dnl
1747 dnl REDUCE_MAXMIN_L_PREDICATE_PARTIAL($1 , $2 )
1748 dnl REDUCE_MAXMIN_L_PREDICATE_PARTIAL(min_max, op_name)
1749 define(`REDUCE_MAXMIN_L_PREDICATE_PARTIAL', `
1750 instruct reduce_$1L_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
1751 pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
1752 predicate(UseSVE > 0 &&
1753 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
1754 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
1755 match(Set dst ($2 (Binary src1 src2) pg));
1756 effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
1757 ins_cost(3 * SVE_COST);
1758 format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated partial (sve)" %}
1759 ins_encode %{
1760 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
1761 Matcher::vector_length(this, $src2));
1762 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1763 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1764 __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
1765 $src1$$Register, as_FloatRegister($src2$$reg),
1766 as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
1767 %}
1768 ins_pipe(pipe_slow);
1769 %}')dnl
1770 dnl
1771 dnl REDUCE_FMINMAX($1, $2, $3, $4, $5 )
1772 dnl REDUCE_FMINMAX(min_max, name_suffix, element_type, size, reg_src_dst)
1773 define(`REDUCE_FMINMAX', `
1774 instruct reduce_$1$2($5 dst, $5 src1, vReg src2) %{
1775 predicate(UseSVE > 0 &&
1776 n->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1777 n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1778 match(Set dst (translit($1, `m', `M')ReductionV src1 src2));
1779 ins_cost(INSN_COST);
1780 effect(TEMP_DEF dst);
1781 format %{ "sve_reduce_$1$2 $dst, $src1, $src2\t# $1$2 reduction (sve)" %}
1782 ins_encode %{
1783 __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src2$$reg));
1784 __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1785 %}
1786 ins_pipe(pipe_slow);
1787 %}')dnl
1788 dnl
1789 dnl REDUCE_FMINMAX_PARTIAL($1, $2, $3, $4, $5 )
1790 dnl REDUCE_FMINMAX_PARTIAL(min_max, name_suffix, element_type, size, reg_src_dst)
1791 define(`REDUCE_FMINMAX_PARTIAL', `
1792 instruct reduce_$1$2_partial($5 dst, $5 src1, vReg src2,
1793 pRegGov ptmp, rFlagsReg cr) %{
1794 predicate(UseSVE > 0 &&
1795 n->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1796 n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1797 match(Set dst (translit($1, `m', `M')ReductionV src1 src2));
1798 ins_cost(INSN_COST);
1799 effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
1800 format %{ "sve_reduce_$1$2 $dst, $src1, $src2\t# $1$2 reduction partial (sve)" %}
1801 ins_encode %{
1802 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
1803 Matcher::vector_length(this, $src2));
1804 __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1805 __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1806 %}
1807 ins_pipe(pipe_slow);
1808 %}')dnl
1809 dnl
1810 dnl REDUCE_FMINMAX_PREDICATE($1, $2, $3, $4, $5 )
1811 dnl REDUCE_FMINMAX_PREDICATE(min_max, name_suffix, element_type, size, reg_src_dst)
1812 define(`REDUCE_FMINMAX_PREDICATE', `
1813 instruct reduce_$1$2_masked($5 dst, $5 src1, vReg src2, pRegGov pg) %{
1814 predicate(UseSVE > 0 &&
1815 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1816 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
1817 match(Set dst (translit($1, `m', `M')ReductionV (Binary src1 src2) pg));
1818 ins_cost(SVE_COST);
1819 effect(TEMP_DEF dst);
1820 format %{ "sve_reduce_$1$2 $dst, $src1, $pg, $src2\t# $1$2 reduction predicated (sve)" %}
1821 ins_encode %{
1822 __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
1823 __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1824 %}
1825 ins_pipe(pipe_slow);
1826 %}')dnl
1827 dnl
1828 dnl REDUCE_FMINMAX_PREDICATE_PARTIAL($1, $2, $3, $4, $5 )
1829 dnl REDUCE_FMINMAX_PREDICATE_PARTIAL(min_max, name_suffix, element_type, size, reg_src_dst)
1830 define(`REDUCE_FMINMAX_PREDICATE_PARTIAL', `
1831 instruct reduce_$1$2_masked_partial($5 dst, $5 src1, vReg src2, pRegGov pg,
1832 pRegGov ptmp, rFlagsReg cr) %{
1833 predicate(UseSVE > 0 &&
1834 n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == $3 &&
1835 n->in(1)->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
1836 match(Set dst (translit($1, `m', `M')ReductionV (Binary src1 src2) pg));
1837 effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
1838 ins_cost(3 * SVE_COST);
1839 format %{ "sve_reduce_$1$2 $dst, $src1, $pg, $src2\t# $1$2 reduction predicated partial (sve)" %}
1840 ins_encode %{
1841 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
1842 Matcher::vector_length(this, $src2));
1843 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
1844 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
1845 __ sve_f$1v(as_FloatRegister($dst$$reg), __ $4,
1846 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
1847 __ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
1848 %}
1849 ins_pipe(pipe_slow);
1850 %}')dnl
1851 // vector max reduction
1852 REDUCE_MAXMIN_I(max, MaxReductionV)
1853 REDUCE_MAXMIN_L(max, MaxReductionV)
1854 REDUCE_MAXMIN_I_PARTIAL(max, MaxReductionV)
1855 REDUCE_MAXMIN_L_PARTIAL(max, MaxReductionV)
1856 REDUCE_FMINMAX(max, F, T_FLOAT, S, vRegF)
1857 REDUCE_FMINMAX_PARTIAL(max, F, T_FLOAT, S, vRegF)
1858 REDUCE_FMINMAX(max, D, T_DOUBLE, D, vRegD)
1859 REDUCE_FMINMAX_PARTIAL(max, D, T_DOUBLE, D, vRegD)
1860
1861 // vector max reduction - predicated
1862 REDUCE_MAXMIN_I_PREDICATE(max, MaxReductionV)
1863 REDUCE_MAXMIN_L_PREDICATE(max, MaxReductionV)
1864 REDUCE_MAXMIN_I_PREDICATE_PARTIAL(max, MaxReductionV)
1865 REDUCE_MAXMIN_L_PREDICATE_PARTIAL(max, MaxReductionV)
1866 REDUCE_FMINMAX_PREDICATE(max, F, T_FLOAT, S, vRegF)
1867 REDUCE_FMINMAX_PREDICATE(max, D, T_DOUBLE, D, vRegD)
1868 REDUCE_FMINMAX_PREDICATE_PARTIAL(max, F, T_FLOAT, S, vRegF)
1869 REDUCE_FMINMAX_PREDICATE_PARTIAL(max, D, T_DOUBLE, D, vRegD)
1870
1871 // vector min reduction
1872 REDUCE_MAXMIN_I(min, MinReductionV)
1873 REDUCE_MAXMIN_L(min, MinReductionV)
1874 REDUCE_MAXMIN_I_PARTIAL(min, MinReductionV)
1875 REDUCE_MAXMIN_L_PARTIAL(min, MinReductionV)
1876 REDUCE_FMINMAX(min, F, T_FLOAT, S, vRegF)
1877 REDUCE_FMINMAX_PARTIAL(min, F, T_FLOAT, S, vRegF)
1878 REDUCE_FMINMAX(min, D, T_DOUBLE, D, vRegD)
1879 REDUCE_FMINMAX_PARTIAL(min, D, T_DOUBLE, D, vRegD)
1880
1881 // vector min reduction - predicated
1882 REDUCE_MAXMIN_I_PREDICATE(min, MinReductionV)
1883 REDUCE_MAXMIN_L_PREDICATE(min, MinReductionV)
1884 REDUCE_MAXMIN_I_PREDICATE_PARTIAL(min, MinReductionV)
1885 REDUCE_MAXMIN_L_PREDICATE_PARTIAL(min, MinReductionV)
1886 REDUCE_FMINMAX_PREDICATE(min, F, T_FLOAT, S, vRegF)
1887 REDUCE_FMINMAX_PREDICATE(min, D, T_DOUBLE, D, vRegD)
1888 REDUCE_FMINMAX_PREDICATE_PARTIAL(min, F, T_FLOAT, S, vRegF)
1889 REDUCE_FMINMAX_PREDICATE_PARTIAL(min, D, T_DOUBLE, D, vRegD)
1890
1891 // vector Math.rint, floor, ceil
1892
1893 instruct vroundD(vReg dst, vReg src, immI rmode) %{
1894 predicate(UseSVE > 0 &&
1895 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
1896 match(Set dst (RoundDoubleModeV src rmode));
1897 format %{ "sve_frint $dst, $src, $rmode\t# vector (sve) (D)" %}
1898 ins_encode %{
1899 switch ($rmode$$constant) {
1900 case RoundDoubleModeNode::rmode_rint:
1901 __ sve_frintn(as_FloatRegister($dst$$reg), __ D,
1902 ptrue, as_FloatRegister($src$$reg));
1903 break;
1904 case RoundDoubleModeNode::rmode_floor:
1905 __ sve_frintm(as_FloatRegister($dst$$reg), __ D,
1906 ptrue, as_FloatRegister($src$$reg));
1907 break;
1908 case RoundDoubleModeNode::rmode_ceil:
1909 __ sve_frintp(as_FloatRegister($dst$$reg), __ D,
1910 ptrue, as_FloatRegister($src$$reg));
1911 break;
1912 }
1913 %}
1914 ins_pipe(pipe_slow);
1915 %}
1916 dnl
1917 dnl REPLICATE($1, $2, $3, $4, $5 )
1918 dnl REPLICATE(insn_name, op_name, reg_src, size, min_vec_len)
1919 define(`REPLICATE', `
1920 instruct $1(vReg dst, $3 src) %{
1921 predicate(UseSVE > 0);
1922 match(Set dst ($2 src));
1923 ins_cost(SVE_COST);
1924 format %{ "sve_dup $dst, $src\t# vector (sve) ($4)" %}
1925 ins_encode %{
1926 __ sve_dup(as_FloatRegister($dst$$reg), __ $4, as_Register($src$$reg));
1927 %}
1928 ins_pipe(pipe_slow);
1929 %}')dnl
1930 dnl
1931 dnl REPLICATE_IMM8($1, $2, $3, $4, $5 )
1932 dnl REPLICATE_IMM8(insn_name, op_name, imm_type, size, min_vec_len)
1933 define(`REPLICATE_IMM8', `
1934 instruct $1(vReg dst, $3 con) %{
1935 predicate(UseSVE > 0);
1936 match(Set dst ($2 con));
1937 ins_cost(SVE_COST);
1938 format %{ "sve_dup $dst, $con\t# vector (sve) ($4)" %}
1939 ins_encode %{
1940 __ sve_dup(as_FloatRegister($dst$$reg), __ $4, $con$$constant);
1941 %}
1942 ins_pipe(pipe_slow);
1943 %}')dnl
1944 dnl
1945 dnl FREPLICATE($1, $2, $3, $4)
1946 dnl FREPLICATE(insn_name, op_name, reg_src, size)
1947 define(`FREPLICATE', `
1948 instruct $1(vReg dst, $3 src) %{
1949 predicate(UseSVE > 0);
1950 match(Set dst ($2 src));
1951 ins_cost(SVE_COST);
1952 format %{ "sve_cpy $dst, $src\t# vector (sve) ($4)" %}
1953 ins_encode %{
1954 __ sve_cpy(as_FloatRegister($dst$$reg), __ $4,
1955 ptrue, as_FloatRegister($src$$reg));
1956 %}
1957 ins_pipe(pipe_slow);
1958 %}')dnl
1959
1960 // vector replicate
1961 REPLICATE(replicateB, ReplicateB, iRegIorL2I, B, 16)
1962 REPLICATE(replicateS, ReplicateS, iRegIorL2I, H, 8)
1963 REPLICATE(replicateI, ReplicateI, iRegIorL2I, S, 4)
1964 REPLICATE(replicateL, ReplicateL, iRegL, D, 2)
1965 REPLICATE_IMM8(replicateB_imm8, ReplicateB, immI8, B, 16)
1966 REPLICATE_IMM8(replicateS_imm8, ReplicateS, immI8_shift8, H, 8)
1967 REPLICATE_IMM8(replicateI_imm8, ReplicateI, immI8_shift8, S, 4)
1968 REPLICATE_IMM8(replicateL_imm8, ReplicateL, immL8_shift8, D, 2)
1969 FREPLICATE(replicateF, ReplicateF, vRegF, S, 4)
1970 FREPLICATE(replicateD, ReplicateD, vRegD, D, 2)
1971 dnl
1972 dnl VSHIFT_TRUE_PREDICATE($1, $2, $3, $4, $5 )
1973 dnl VSHIFT_TRUE_PREDICATE(insn_name, op_name, size, min_vec_len, insn)
1974 define(`VSHIFT_TRUE_PREDICATE', `
1975 instruct $1(vReg dst, vReg shift) %{
1976 predicate(UseSVE > 0);
1977 match(Set dst ($2 dst shift));
1978 ins_cost(SVE_COST);
1979 format %{ "$5 $dst, $dst, $shift\t# vector (sve) ($3)" %}
1980 ins_encode %{
1981 __ $5(as_FloatRegister($dst$$reg), __ $3,
1982 ptrue, as_FloatRegister($shift$$reg));
1983 %}
1984 ins_pipe(pipe_slow);
1985 %}')dnl
1986 dnl
1987 dnl VSHIFT_IMM_UNPREDICATED($1, $2, $3, $4, $5, $6 )
1988 dnl VSHIFT_IMM_UNPREDICATED(insn_name, op_name, op_name2, size, min_vec_len, insn)
1989 define(`VSHIFT_IMM_UNPREDICATED', `
1990 instruct $1(vReg dst, vReg src, immI shift) %{
1991 predicate(UseSVE > 0);
1992 match(Set dst ($2 src ($3 shift)));
1993 ins_cost(SVE_COST);
1994 format %{ "$6 $dst, $src, $shift\t# vector (sve) ($4)" %}
1995 ins_encode %{
1996 int con = (int)$shift$$constant;dnl
1997 ifelse(eval(index(`$1', `vasr') == 0 || index(`$1', `vlsr') == 0), 1, `
1998 if (con == 0) {
1999 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
2000 as_FloatRegister($src$$reg));
2001 return;
2002 }')dnl
2003 ifelse(eval(index(`$1', `vasr') == 0), 1, `ifelse(eval(index(`$4', `B') == 0), 1, `
2004 if (con >= 8) con = 7;')ifelse(eval(index(`$4', `H') == 0), 1, `
2005 if (con >= 16) con = 15;')')dnl
2006 ifelse(eval(index(`$1', `vlsl') == 0 || index(`$1', `vlsr') == 0), 1, `ifelse(eval(index(`$4', `B') == 0), 1, `
2007 if (con >= 8) {
2008 __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
2009 as_FloatRegister($src$$reg));
2010 return;
2011 }')ifelse(eval(index(`$4', `H') == 0), 1, `
2012 if (con >= 16) {
2013 __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
2014 as_FloatRegister($src$$reg));
2015 return;
2016 }')')
2017 __ $6(as_FloatRegister($dst$$reg), __ $4,
2018 as_FloatRegister($src$$reg), con);
2019 %}
2020 ins_pipe(pipe_slow);
2021 %}')dnl
2022 dnl
2023 dnl VSHIFT_COUNT($1, $2, $3, $4 )
2024 dnl VSHIFT_COUNT(insn_name, size, min_vec_len, type)
2025 define(`VSHIFT_COUNT', `
2026 instruct $1(vReg dst, iRegIorL2I cnt) %{
2027 predicate(UseSVE > 0 &&
2028 ELEMENT_SHORT_CHAR($4, n));
2029 match(Set dst (LShiftCntV cnt));
2030 match(Set dst (RShiftCntV cnt));
2031 format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) ($2)" %}
2032 ins_encode %{
2033 __ sve_dup(as_FloatRegister($dst$$reg), __ $2, as_Register($cnt$$reg));
2034 %}
2035 ins_pipe(pipe_slow);
2036 %}')dnl
2037
2038 // vector shift
2039 VSHIFT_TRUE_PREDICATE(vasrB, RShiftVB, B, 16, sve_asr)
2040 VSHIFT_TRUE_PREDICATE(vasrS, RShiftVS, H, 8, sve_asr)
2041 VSHIFT_TRUE_PREDICATE(vasrI, RShiftVI, S, 4, sve_asr)
2042 VSHIFT_TRUE_PREDICATE(vasrL, RShiftVL, D, 2, sve_asr)
2043 VSHIFT_TRUE_PREDICATE(vlslB, LShiftVB, B, 16, sve_lsl)
2044 VSHIFT_TRUE_PREDICATE(vlslS, LShiftVS, H, 8, sve_lsl)
2045 VSHIFT_TRUE_PREDICATE(vlslI, LShiftVI, S, 4, sve_lsl)
2046 VSHIFT_TRUE_PREDICATE(vlslL, LShiftVL, D, 2, sve_lsl)
2047 VSHIFT_TRUE_PREDICATE(vlsrB, URShiftVB, B, 16, sve_lsr)
2048 VSHIFT_TRUE_PREDICATE(vlsrS, URShiftVS, H, 8, sve_lsr)
2049 VSHIFT_TRUE_PREDICATE(vlsrI, URShiftVI, S, 4, sve_lsr)
2050 VSHIFT_TRUE_PREDICATE(vlsrL, URShiftVL, D, 2, sve_lsr)
2051 VSHIFT_IMM_UNPREDICATED(vasrB_imm, RShiftVB, RShiftCntV, B, 16, sve_asr)
2052 VSHIFT_IMM_UNPREDICATED(vasrS_imm, RShiftVS, RShiftCntV, H, 8, sve_asr)
2053 VSHIFT_IMM_UNPREDICATED(vasrI_imm, RShiftVI, RShiftCntV, S, 4, sve_asr)
2054 VSHIFT_IMM_UNPREDICATED(vasrL_imm, RShiftVL, RShiftCntV, D, 2, sve_asr)
2055 VSHIFT_IMM_UNPREDICATED(vlsrB_imm, URShiftVB, RShiftCntV, B, 16, sve_lsr)
2056 VSHIFT_IMM_UNPREDICATED(vlsrS_imm, URShiftVS, RShiftCntV, H, 8, sve_lsr)
2057 VSHIFT_IMM_UNPREDICATED(vlsrI_imm, URShiftVI, RShiftCntV, S, 4, sve_lsr)
2058 VSHIFT_IMM_UNPREDICATED(vlsrL_imm, URShiftVL, RShiftCntV, D, 2, sve_lsr)
2059 VSHIFT_IMM_UNPREDICATED(vlslB_imm, LShiftVB, LShiftCntV, B, 16, sve_lsl)
2060 VSHIFT_IMM_UNPREDICATED(vlslS_imm, LShiftVS, LShiftCntV, H, 8, sve_lsl)
2061 VSHIFT_IMM_UNPREDICATED(vlslI_imm, LShiftVI, LShiftCntV, S, 4, sve_lsl)
2062 VSHIFT_IMM_UNPREDICATED(vlslL_imm, LShiftVL, LShiftCntV, D, 2, sve_lsl)
2063 VSHIFT_COUNT(vshiftcntB, B, 16, T_BYTE)
2064 VSHIFT_COUNT(vshiftcntS, H, 8, T_SHORT)
2065 VSHIFT_COUNT(vshiftcntI, S, 4, T_INT)
2066 VSHIFT_COUNT(vshiftcntL, D, 2, T_LONG)
2067
2068 // vector shift - predicated
2069 BINARY_OP_PREDICATE(vasrB, RShiftVB, B, sve_asr)
2070 BINARY_OP_PREDICATE(vasrS, RShiftVS, H, sve_asr)
2071 BINARY_OP_PREDICATE(vasrI, RShiftVI, S, sve_asr)
2072 BINARY_OP_PREDICATE(vasrL, RShiftVL, D, sve_asr)
2073 BINARY_OP_PREDICATE(vlslB, LShiftVB, B, sve_lsl)
2074 BINARY_OP_PREDICATE(vlslS, LShiftVS, H, sve_lsl)
2075 BINARY_OP_PREDICATE(vlslI, LShiftVI, S, sve_lsl)
2076 BINARY_OP_PREDICATE(vlslL, LShiftVL, D, sve_lsl)
2077 BINARY_OP_PREDICATE(vlsrB, URShiftVB, B, sve_lsr)
2078 BINARY_OP_PREDICATE(vlsrS, URShiftVS, H, sve_lsr)
2079 BINARY_OP_PREDICATE(vlsrI, URShiftVI, S, sve_lsr)
2080 BINARY_OP_PREDICATE(vlsrL, URShiftVL, D, sve_lsr)
2081 dnl
2082 dnl VSHIFT_IMM_PREDICATED($1, $2, $3, $4, $5, $6 )
2083 dnl VSHIFT_IMM_PREDICATED(insn_name, op_name, op_name2, type, size, insn)
2084 define(`VSHIFT_IMM_PREDICATED', `
2085 instruct $1_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
2086 predicate(UseSVE > 0);
2087 match(Set dst_src ($2 (Binary dst_src ($3 shift)) pg));
2088 ins_cost(SVE_COST);
2089 format %{ "$6 $dst_src, $pg, $dst_src, $shift\t# vector (sve) ($4)" %}
2090 ins_encode %{
2091 int con = (int)$shift$$constant;
2092 assert(con ifelse(index(`$1', `vlsl'), 0, `>=', `>') 0 && con < $5, "invalid shift immediate");
2093 __ $6(as_FloatRegister($dst_src$$reg), __ $4, as_PRegister($pg$$reg), con);
2094 %}
2095 ins_pipe(pipe_slow);
2096 %}')dnl
2097 dnl
2098 VSHIFT_IMM_PREDICATED(vasrB, RShiftVB, RShiftCntV, B, 8, sve_asr)
2099 VSHIFT_IMM_PREDICATED(vasrS, RShiftVS, RShiftCntV, H, 16, sve_asr)
2100 VSHIFT_IMM_PREDICATED(vasrI, RShiftVI, RShiftCntV, S, 32, sve_asr)
2101 VSHIFT_IMM_PREDICATED(vasrL, RShiftVL, RShiftCntV, D, 64, sve_asr)
2102 VSHIFT_IMM_PREDICATED(vlsrB, URShiftVB, RShiftCntV, B, 8, sve_lsr)
2103 VSHIFT_IMM_PREDICATED(vlsrS, URShiftVS, RShiftCntV, H, 16, sve_lsr)
2104 VSHIFT_IMM_PREDICATED(vlsrI, URShiftVI, RShiftCntV, S, 32, sve_lsr)
2105 VSHIFT_IMM_PREDICATED(vlsrL, URShiftVL, RShiftCntV, D, 64, sve_lsr)
2106 VSHIFT_IMM_PREDICATED(vlslB, LShiftVB, LShiftCntV, B, 8, sve_lsl)
2107 VSHIFT_IMM_PREDICATED(vlslS, LShiftVS, LShiftCntV, H, 16, sve_lsl)
2108 VSHIFT_IMM_PREDICATED(vlslI, LShiftVI, LShiftCntV, S, 32, sve_lsl)
2109 VSHIFT_IMM_PREDICATED(vlslL, LShiftVL, LShiftCntV, D, 64, sve_lsl)
2110
2111 // vector sqrt
2112 UNARY_OP_TRUE_PREDICATE(vsqrtF, SqrtVF, S, sve_fsqrt)
2113 UNARY_OP_TRUE_PREDICATE(vsqrtD, SqrtVD, D, sve_fsqrt)
2114
2115 // vector sqrt - predicated
2116 UNARY_OP_PREDICATE(vsqrtF, SqrtVF, S, sve_fsqrt)
2117 UNARY_OP_PREDICATE(vsqrtD, SqrtVD, D, sve_fsqrt)
2118
2119 // vector sub
2120 BINARY_OP_UNPREDICATE(vsubB, SubVB, B, 16, sve_sub)
2121 BINARY_OP_UNPREDICATE(vsubS, SubVS, H, 8, sve_sub)
2122 BINARY_OP_UNPREDICATE(vsubI, SubVI, S, 4, sve_sub)
2123 BINARY_OP_UNPREDICATE(vsubL, SubVL, D, 2, sve_sub)
2124 BINARY_OP_UNPREDICATE(vsubF, SubVF, S, 4, sve_fsub)
2125 BINARY_OP_UNPREDICATE(vsubD, SubVD, D, 2, sve_fsub)
2126
2127 // vector sub - predicated
2128 BINARY_OP_PREDICATE(vsubB, SubVB, B, sve_sub)
2129 BINARY_OP_PREDICATE(vsubS, SubVS, H, sve_sub)
2130 BINARY_OP_PREDICATE(vsubI, SubVI, S, sve_sub)
2131 BINARY_OP_PREDICATE(vsubL, SubVL, D, sve_sub)
2132 BINARY_OP_PREDICATE(vsubF, SubVF, S, sve_fsub)
2133 BINARY_OP_PREDICATE(vsubD, SubVD, D, sve_fsub)
2134
2135 // ------------------------------ Vector mask cast --------------------------
2136
2137 instruct vmaskcast(pRegGov dst_src) %{
2138 predicate(UseSVE > 0 &&
2139 n->bottom_type()->is_vect()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
2140 n->bottom_type()->is_vect()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
2141 match(Set dst_src (VectorMaskCast dst_src));
2142 ins_cost(0);
2143 format %{ "vmaskcast $dst_src\t# empty (sve)" %}
2144 ins_encode %{
2145 // empty
2146 %}
2147 ins_pipe(pipe_class_empty);
2148 %}
2149
2150 instruct vmaskcast_extend(pRegGov dst, pReg src)
2151 %{
2152 predicate(UseSVE > 0 &&
2153 (Matcher::vector_length_in_bytes(n) == 2 * Matcher::vector_length_in_bytes(n->in(1)) ||
2154 Matcher::vector_length_in_bytes(n) == 4 * Matcher::vector_length_in_bytes(n->in(1)) ||
2155 Matcher::vector_length_in_bytes(n) == 8 * Matcher::vector_length_in_bytes(n->in(1))));
2156 match(Set dst (VectorMaskCast src));
2157 ins_cost(SVE_COST * 3);
2158 format %{ "sve_vmaskcast_extend $dst, $src\t# extend predicate $src" %}
2159 ins_encode %{
2160 __ sve_vmaskcast_extend(as_PRegister($dst$$reg), as_PRegister($src$$reg),
2161 Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
2162 %}
2163 ins_pipe(pipe_slow);
2164 %}
2165
2166 instruct vmaskcast_narrow(pRegGov dst, pReg src)
2167 %{
2168 predicate(UseSVE > 0 &&
2169 (Matcher::vector_length_in_bytes(n) * 2 == Matcher::vector_length_in_bytes(n->in(1)) ||
2170 Matcher::vector_length_in_bytes(n) * 4 == Matcher::vector_length_in_bytes(n->in(1)) ||
2171 Matcher::vector_length_in_bytes(n) * 8 == Matcher::vector_length_in_bytes(n->in(1))));
2172 match(Set dst (VectorMaskCast src));
2173 ins_cost(SVE_COST * 3);
2174 format %{ "sve_vmaskcast_narrow $dst, $src\t# narrow predicate $src" %}
2175 ins_encode %{
2176 __ sve_vmaskcast_narrow(as_PRegister($dst$$reg), as_PRegister($src$$reg),
2177 Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
2178 %}
2179 ins_pipe(pipe_slow);
2180 %}
2181 dnl
2182
2183 // ------------------------------ Vector cast -------------------------------
2184 dnl
2185 dnl
2186 define(`VECTOR_CAST_X2X', `
2187 instruct vcvt$1to$2`'(vReg dst, vReg src)
2188 %{
2189 predicate(UseSVE > 0 &&
2190 n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
2191 match(Set dst (VectorCast$1`'2X src));
2192 ins_cost(SVE_COST);
2193 format %{ "sve_vectorcast_$5 $dst, $src\t# convert $1 to $2 vector" %}
2194 ins_encode %{
2195 __ sve_$3(as_FloatRegister($dst$$reg), __ $4, ptrue, as_FloatRegister($src$$reg), __ $4);
2196 %}
2197 ins_pipe(pipe_slow);
2198 %}')dnl
2199
2200 dnl
2201 dnl Start of vector cast rules
2202 dnl
2203 instruct vcvtBtoX_extend(vReg dst, vReg src)
2204 %{
2205 predicate(UseSVE > 0);
2206 match(Set dst (VectorCastB2X src));
2207 ins_cost(2 * SVE_COST);
2208 format %{ "sve_vectorcast_b2x $dst, $src\t# convert B to X vector (extend)" %}
2209 ins_encode %{
2210 BasicType to_bt = Matcher::vector_element_basic_type(this);
2211 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2212 __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ B);
2213 if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
2214 __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
2215 }
2216 %}
2217 ins_pipe(pipe_slow);
2218 %}
2219
2220 instruct vcvtStoB(vReg dst, vReg src, vReg tmp)
2221 %{
2222 predicate(UseSVE > 0 &&
2223 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2224 match(Set dst (VectorCastS2X src));
2225 effect(TEMP tmp);
2226 ins_cost(2 * SVE_COST);
2227 format %{ "sve_vectorcast_s2b $dst, $src\t# convert H to B vector" %}
2228 ins_encode %{
2229 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
2230 as_FloatRegister($src$$reg), __ H, as_FloatRegister($tmp$$reg));
2231 %}
2232 ins_pipe(pipe_slow);
2233 %}
2234
2235 instruct vcvtStoX_extend(vReg dst, vReg src)
2236 %{
2237 predicate(UseSVE > 0 &&
2238 type2aelembytes(Matcher::vector_element_basic_type(n)) > 2);
2239 match(Set dst (VectorCastS2X src));
2240 ins_cost(2 * SVE_COST);
2241 format %{ "sve_vectorcast_s2x $dst, $src\t# convert H to X vector (extend)" %}
2242 ins_encode %{
2243 BasicType to_bt = Matcher::vector_element_basic_type(this);
2244 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2245 __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ H);
2246 if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
2247 __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
2248 }
2249 %}
2250 ins_pipe(pipe_slow);
2251 %}
2252
2253 instruct vcvtItoB(vReg dst, vReg src, vReg tmp)
2254 %{
2255 predicate(UseSVE > 0 &&
2256 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
2257 match(Set dst (VectorCastI2X src));
2258 effect(TEMP_DEF dst, TEMP tmp);
2259 ins_cost(3 * SVE_COST);
2260 format %{ "sve_vectorcast_i2b $dst, $src\t# convert I to B vector" %}
2261 ins_encode %{
2262 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
2263 as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
2264 %}
2265 ins_pipe(pipe_slow);
2266 %}
2267
2268 instruct vcvtItoS(vReg dst, vReg src, vReg tmp)
2269 %{
2270 predicate(UseSVE > 0 &&
2271 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
2272 match(Set dst (VectorCastI2X src));
2273 effect(TEMP tmp);
2274 ins_cost(2 * SVE_COST);
2275 format %{ "sve_vectorcast_i2s $dst, $src\t# convert I to H vector" %}
2276 ins_encode %{
2277 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ H,
2278 as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
2279 %}
2280 ins_pipe(pipe_slow);
2281 %}
2282
2283 instruct vcvtItoL(vReg dst, vReg src)
2284 %{
2285 predicate(UseSVE > 0 &&
2286 n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
2287 match(Set dst (VectorCastI2X src));
2288 ins_cost(SVE_COST);
2289 format %{ "sve_vectorcast_i2l $dst, $src\t# convert I to L vector" %}
2290 ins_encode %{
2291 __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
2292 %}
2293 ins_pipe(pipe_slow);
2294 %}
2295 dnl
2296 dnl vcvtItoF
2297 VECTOR_CAST_X2X(I, F, scvtf, S, i2f)
2298
2299 instruct vcvtItoD(vReg dst, vReg src)
2300 %{
2301 predicate(UseSVE > 0 &&
2302 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
2303 match(Set dst (VectorCastI2X src));
2304 ins_cost(2 * SVE_COST);
2305 format %{ "sve_vectorcast_i2d $dst, $src\t# convert I to D vector" %}
2306 ins_encode %{
2307 __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
2308 __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ D);
2309 %}
2310 ins_pipe(pipe_slow);
2311 %}
2312
2313 instruct vcvtLtoX_narrow(vReg dst, vReg src, vReg tmp)
2314 %{
2315 predicate(UseSVE > 0 && is_integral_type(Matcher::vector_element_basic_type(n)));
2316 match(Set dst (VectorCastL2X src));
2317 effect(TEMP_DEF dst, TEMP tmp);
2318 ins_cost(2 * SVE_COST);
2319 format %{ "sve_vectorcast_l2x $dst, $src\t# convert L to B/H/S vector (narrow)" %}
2320 ins_encode %{
2321 BasicType to_bt = Matcher::vector_element_basic_type(this);
2322 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2323 __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
2324 as_FloatRegister($src$$reg), __ D, as_FloatRegister($tmp$$reg));
2325 %}
2326 ins_pipe(pipe_slow);
2327 %}
2328
2329 instruct vcvtLtoF(vReg dst, vReg src, vReg tmp)
2330 %{
2331 predicate(UseSVE > 0 &&
2332 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2333 match(Set dst (VectorCastL2X src));
2334 effect(TEMP_DEF dst, TEMP tmp);
2335 ins_cost(3 * SVE_COST);
2336 format %{ "sve_vectorcast_l2f $dst, $src\t# convert L to F vector" %}
2337 ins_encode %{
2338 __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
2339 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
2340 as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
2341
2342 %}
2343 ins_pipe(pipe_slow);
2344 %}
2345 dnl
2346 dnl vcvtLtoD
2347 VECTOR_CAST_X2X(L, D, scvtf, D, l2d)
2348
2349 instruct vcvtFtoX_narrow(vReg dst, vReg src, vReg tmp)
2350 %{
2351 predicate(UseSVE > 0 &&
2352 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2353 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT));
2354 match(Set dst (VectorCastF2X src));
2355 effect(TEMP_DEF dst, TEMP tmp);
2356 ins_cost(3 * SVE_COST);
2357 format %{ "sve_vectorcast_f2x $dst, $src\t# convert F to B/H vector" %}
2358 ins_encode %{
2359 BasicType to_bt = Matcher::vector_element_basic_type(this);
2360 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2361 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
2362 __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
2363 as_FloatRegister($dst$$reg), __ S, as_FloatRegister($tmp$$reg));
2364 %}
2365 ins_pipe(pipe_slow);
2366 %}
2367
2368 instruct vcvtFtoI(vReg dst, vReg src)
2369 %{
2370 predicate(UseSVE > 0 &&
2371 (n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2372 match(Set dst (VectorCastF2X src));
2373 ins_cost(SVE_COST);
2374 format %{ "sve_vectorcast_f2x $dst, $src\t# convert F to I vector" %}
2375 ins_encode %{
2376 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
2377 %}
2378 ins_pipe(pipe_slow);
2379 %}
2380
2381 instruct vcvtFtoL(vReg dst, vReg src)
2382 %{
2383 predicate(UseSVE > 0 &&
2384 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
2385 match(Set dst (VectorCastF2X src));
2386 ins_cost(SVE_COST * 2);
2387 format %{ "sve_vectorcast_f2x $dst, $src\t# convert F to L vector" %}
2388 ins_encode %{
2389 __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
2390 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);
2391 %}
2392 ins_pipe(pipe_slow);
2393 %}
2394
2395 instruct vcvtFtoD(vReg dst, vReg src)
2396 %{
2397 predicate(UseSVE > 0 &&
2398 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
2399 match(Set dst (VectorCastF2X src));
2400 ins_cost(2 * SVE_COST);
2401 format %{ "sve_vectorcast_f2d $dst, $dst\t# convert F to D vector" %}
2402 ins_encode %{
2403 __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
2404 __ sve_fcvt(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);
2405 %}
2406 ins_pipe(pipe_slow);
2407 %}
2408
2409 instruct vcvtDtoX_narrow(vReg dst, vReg src, vReg tmp)
2410 %{
2411 predicate(UseSVE > 0 &&
2412 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2413 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2414 n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2415 match(Set dst (VectorCastD2X src));
2416 effect(TEMP_DEF dst, TEMP tmp);
2417 ins_cost(3 * SVE_COST);
2418 format %{ "sve_vectorcast_d2x $dst, $src\t# convert D to X vector (narrow)" %}
2419 ins_encode %{
2420 BasicType to_bt = Matcher::vector_element_basic_type(this);
2421 Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
2422 __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
2423 __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
2424 as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
2425 %}
2426 ins_pipe(pipe_slow);
2427 %}
2428 dnl
2429 dnl vcvtDtoL
2430 VECTOR_CAST_X2X(D, L, fcvtzs, D, d2l)
2431
2432 instruct vcvtDtoF(vReg dst, vReg src, vReg tmp)
2433 %{
2434 predicate(UseSVE > 0 &&
2435 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2436 match(Set dst (VectorCastD2X src));
2437 effect(TEMP_DEF dst, TEMP tmp);
2438 ins_cost(3 * SVE_COST);
2439 format %{ "sve_vectorcast_d2f $dst, S, $dst\t# convert D to F vector" %}
2440 ins_encode %{
2441 __ sve_fcvt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
2442 __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
2443 as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
2444 %}
2445 ins_pipe(pipe_slow);
2446 %}
2447
2448 dnl
2449 dnl
2450 // ------------------------------ Vector extract ---------------------------------
2451 define(`VECTOR_EXTRACT_SXT', `
2452 instruct extract$1`'($2 dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
2453 %{
2454 predicate(UseSVE > 0);
2455 match(Set dst (Extract$1 src idx));
2456 effect(TEMP pgtmp, KILL cr);
2457 ins_cost(2 * SVE_COST);
2458 format %{ "sve_extract $dst, $3, $pgtmp, $src, $idx\n\t"
2459 "sbfmw $dst, $dst, 0U, $5\t# extract from vector($1)" %}
2460 ins_encode %{
2461 __ sve_extract(as_$4($dst$$reg), __ $3, as_PRegister($pgtmp$$reg),
2462 as_FloatRegister($src$$reg), (int)($idx$$constant));
2463 __ sbfmw(as_$4($dst$$reg), as_$4($dst$$reg), 0U, $5);
2464 %}
2465 ins_pipe(pipe_slow);
2466 %}')dnl
2467 dnl $1 $2 $3 $4 $5
2468 VECTOR_EXTRACT_SXT(B, iRegINoSp, B, Register, 7U)
2469 VECTOR_EXTRACT_SXT(S, iRegINoSp, H, Register, 15U)
2470
2471 dnl
2472 define(`VECTOR_EXTRACT', `
2473 instruct extract$1`'($2 dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
2474 %{
2475 predicate(UseSVE > 0);
2476 match(Set dst (Extract$1 src idx));
2477 effect(TEMP pgtmp, KILL cr);
2478 ins_cost(2 * SVE_COST);
2479 format %{ "sve_extract $dst, $3, $pgtmp, $src, $idx\t# extract from vector($1)" %}
2480 ins_encode %{
2481 __ sve_extract(as_$4($dst$$reg), __ $3, as_PRegister($pgtmp$$reg),
2482 as_FloatRegister($src$$reg), (int)($idx$$constant));
2483 %}
2484 ins_pipe(pipe_slow);
2485 %}')dnl
2486 dnl $1 $2 $3 $4
2487 VECTOR_EXTRACT(I, iRegINoSp, S, Register)
2488 VECTOR_EXTRACT(L, iRegLNoSp, D, Register)
2489 VECTOR_EXTRACT(F, vRegF, S, FloatRegister)
2490 VECTOR_EXTRACT(D, vRegD, D, FloatRegister)
2491
2492 // ------------------------------- VectorTest ----------------------------------
2493
2494 instruct vtest_alltrue(iRegINoSp dst, pRegGov src1, pRegGov src2, pReg ptmp, rFlagsReg cr)
2495 %{
2496 predicate(UseSVE > 0 &&
2497 n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2498 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
2499 match(Set dst (VectorTest src1 src2));
2500 effect(TEMP ptmp, KILL cr);
2501 ins_cost(SVE_COST);
2502 format %{ "sve_eors $ptmp, $src1, $src2\t# $src2 is all true mask\n"
2503 "csetw $dst, EQ\t# VectorTest (sve) - alltrue" %}
2504 ins_encode %{
2505 __ sve_eors(as_PRegister($ptmp$$reg), ptrue,
2506 as_PRegister($src1$$reg), as_PRegister($src2$$reg));
2507 __ csetw(as_Register($dst$$reg), Assembler::EQ);
2508 %}
2509 ins_pipe(pipe_slow);
2510 %}
2511
2512 instruct vtest_anytrue(iRegINoSp dst, pRegGov src1, pRegGov src2, rFlagsReg cr)
2513 %{
2514 predicate(UseSVE > 0 &&
2515 n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
2516 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
2517 match(Set dst (VectorTest src1 src2));
2518 effect(KILL cr);
2519 ins_cost(SVE_COST);
2520 format %{ "sve_ptest $src1\n\t"
2521 "csetw $dst, NE\t# VectorTest (sve) - anytrue" %}
2522 ins_encode %{
2523 // "src2" is not used for sve.
2524 __ sve_ptest(ptrue, as_PRegister($src1$$reg));
2525 __ csetw(as_Register($dst$$reg), Assembler::NE);
2526 %}
2527 ins_pipe(pipe_slow);
2528 %}
2529 dnl
2530 dnl
2531 dnl VTEST_PARTIAL($1, $2, $3, $4 )
2532 dnl VTEST_PARTIAL(op_name, pred, inst, cond)
2533 define(`VTEST_PARTIAL', `
2534 instruct vtest_$1_partial`'(iRegINoSp dst, pRegGov src1, pRegGov src2, pRegGov ptmp, rFlagsReg cr)
2535 %{
2536 predicate(UseSVE > 0 &&
2537 n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
2538 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::$2);
2539 match(Set dst (VectorTest src1 src2));
2540 effect(TEMP ptmp, KILL cr);
2541 ins_cost(SVE_COST);
2542 format %{ "vtest_$1_partial $dst, $src1, $src2\t# VectorTest partial (sve) - $1" %}
2543 ins_encode %{
2544 BasicType bt = Matcher::vector_element_basic_type(this, $src1);
2545 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2546 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size,
2547 Matcher::vector_length(this, $src1));
2548 __ $3(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2549 as_PRegister($src1$$reg), as_PRegister($src2$$reg));
2550 __ csetw(as_Register($dst$$reg), Assembler::$4);
2551 %}
2552 ins_pipe(pipe_slow);
2553 %}')dnl
2554 dnl
2555 VTEST_PARTIAL(alltrue, overflow, sve_eors, EQ)
2556 VTEST_PARTIAL(anytrue, ne, sve_ands, NE)
2557
2558 // ------------------------------ Vector insert ---------------------------------
2559
2560 instruct insertI_small(vReg dst, vReg src, iRegIorL2I val, immI idx, pRegGov pgtmp, rFlagsReg cr)
2561 %{
2562 predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
2563 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2564 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2565 n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2566 match(Set dst (VectorInsert (Binary src val) idx));
2567 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
2568 ins_cost(4 * SVE_COST);
2569 format %{ "sve_index $dst, -16, 1\t# (B/H/S)\n\t"
2570 "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2571 "sve_orr $dst, $src, $src\n\t"
2572 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
2573 ins_encode %{
2574 BasicType bt = Matcher::vector_element_basic_type(this, $src);
2575 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2576 __ sve_index(as_FloatRegister($dst$$reg), size, -16, 1);
2577 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
2578 as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2579 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2580 __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
2581 %}
2582 ins_pipe(pipe_slow);
2583 %}
2584
2585 instruct insertF_small(vReg dst, vReg src, vRegF val, immI idx, pRegGov pgtmp, rFlagsReg cr)
2586 %{
2587 predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
2588 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2589 match(Set dst (VectorInsert (Binary src val) idx));
2590 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
2591 ins_cost(4 * SVE_COST);
2592 format %{ "sve_index $dst, S, -16, 1\n\t"
2593 "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2594 "sve_orr $dst, $src, $src\n\t"
2595 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
2596 ins_encode %{
2597 __ sve_index(as_FloatRegister($dst$$reg), __ S, -16, 1);
2598 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
2599 as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2600 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2601 __ sve_cpy(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
2602 %}
2603 ins_pipe(pipe_slow);
2604 %}
2605
2606 instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr)
2607 %{
2608 predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
2609 (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
2610 n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
2611 n->bottom_type()->is_vect()->element_basic_type() == T_INT));
2612 match(Set dst (VectorInsert (Binary src val) idx));
2613 effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
2614 ins_cost(5 * SVE_COST);
2615 format %{ "sve_index $tmp1, 0, 1\t# (B/H/S)\n\t"
2616 "sve_dup $dst, $idx\t# (B/H/S)\n\t"
2617 "sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
2618 "sve_orr $dst, $src, $src\n\t"
2619 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
2620 ins_encode %{
2621 BasicType bt = Matcher::vector_element_basic_type(this, $src);
2622 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2623 __ sve_index(as_FloatRegister($tmp1$$reg), size, 0, 1);
2624 __ sve_dup(as_FloatRegister($dst$$reg), size, (int)($idx$$constant));
2625 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
2626 as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
2627 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2628 __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
2629 %}
2630 ins_pipe(pipe_slow);
2631 %}
2632 dnl
2633 dnl
2634 define(`VECTOR_INSERT_D', `
2635 instruct insert$1`'(vReg dst, vReg src, $2 val, immI idx, pRegGov pgtmp, rFlagsReg cr)
2636 %{
2637 predicate(UseSVE > 0 &&
2638 n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($1));
2639 match(Set dst (VectorInsert (Binary src val) idx));
2640 effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
2641 ins_cost(4 * SVE_COST);
2642 format %{ "sve_index $dst, $3, -16, 1\n\t"
2643 "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
2644 "sve_orr $dst, $src, $src\n\t"
2645 "sve_cpy $dst, $pgtmp, $val\t# insert into vector ($1)" %}
2646 ins_encode %{
2647 __ sve_index(as_FloatRegister($dst$$reg), __ $3, -16, 1);
2648 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ $3, ptrue,
2649 as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
2650 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
2651 __ sve_cpy(as_FloatRegister($dst$$reg), __ $3, as_PRegister($pgtmp$$reg), as_$4($val$$reg));
2652 %}
2653 ins_pipe(pipe_slow);
2654 %}')dnl
2655 dnl $1 $2 $3 $4
2656 VECTOR_INSERT_D(L, iRegL, D, Register)
2657 VECTOR_INSERT_D(D, vRegD, D, FloatRegister)
2658
2659 instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr)
2660 %{
2661 predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
2662 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
2663 match(Set dst (VectorInsert (Binary src val) idx));
2664 effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
2665 ins_cost(5 * SVE_COST);
2666 format %{ "sve_index $tmp1, S, 0, 1\n\t"
2667 "sve_dup $dst, S, $idx\n\t"
2668 "sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
2669 "sve_orr $dst, $src, $src\n\t"
2670 "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
2671 ins_encode %{
2672 __ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
2673 __ sve_dup(as_FloatRegister($dst$$reg), __ S, (int)($idx$$constant));
2674 __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
2675 as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
2676 __ sve_orr(as_FloatRegister($dst$$reg),
2677 as_FloatRegister($src$$reg),
2678 as_FloatRegister($src$$reg));
2679 __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
2680 as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
2681 %}
2682 ins_pipe(pipe_slow);
2683 %}
2684
2685 // ------------------------------ Vector shuffle -------------------------------
2686
2687 instruct loadshuffle(vReg dst, vReg src) %{
2688 predicate(UseSVE > 0);
2689 match(Set dst (VectorLoadShuffle src));
2690 ins_cost(SVE_COST);
2691 format %{ "sve_loadshuffle $dst, $src\t# vector load shuffle (B/H/S/D)" %}
2692 ins_encode %{
2693 BasicType bt = Matcher::vector_element_basic_type(this);
2694 if (bt == T_BYTE) {
2695 if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
2696 __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
2697 as_FloatRegister($src$$reg));
2698 }
2699 } else {
2700 __ sve_vector_extend(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
2701 as_FloatRegister($src$$reg), __ B);
2702 }
2703 %}
2704 ins_pipe(pipe_slow);
2705 %}
2706
2707 // ------------------------------ Vector rearrange -------------------------------
2708
2709 instruct rearrange(vReg dst, vReg src, vReg shuffle)
2710 %{
2711 predicate(UseSVE > 0);
2712 match(Set dst (VectorRearrange src shuffle));
2713 ins_cost(SVE_COST);
2714 format %{ "sve_tbl $dst, $src, $shuffle\t# vector rearrange" %}
2715 ins_encode %{
2716 BasicType bt = Matcher::vector_element_basic_type(this, $src);
2717 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
2718 __ sve_tbl(as_FloatRegister($dst$$reg), size,
2719 as_FloatRegister($src$$reg), as_FloatRegister($shuffle$$reg));
2720 %}
2721 ins_pipe(pipe_slow);
2722 %}
2723
2724 // ------------------------------ Vector Load Gather ---------------------------------
2725
2726 instruct gatherI(vReg dst, indirect mem, vReg idx) %{
2727 predicate(UseSVE > 0 &&
2728 n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
2729 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2730 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2731 match(Set dst (LoadVectorGather mem idx));
2732 ins_cost(SVE_COST);
2733 format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (S)" %}
2734 ins_encode %{
2735 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), ptrue,
2736 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2737 %}
2738 ins_pipe(pipe_slow);
2739 %}
2740
2741 instruct gatherL(vReg dst, indirect mem, vReg idx) %{
2742 predicate(UseSVE > 0 &&
2743 n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
2744 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2745 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2746 match(Set dst (LoadVectorGather mem idx));
2747 ins_cost(2 * SVE_COST);
2748 format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (D)" %}
2749 ins_encode %{
2750 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2751 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), ptrue, as_Register($mem$$base),
2752 as_FloatRegister($idx$$reg));
2753 %}
2754 ins_pipe(pipe_slow);
2755 %}
2756
2757 // ------------------------------ Vector Load Gather Partial-------------------------------
2758
2759 instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2760 predicate(UseSVE > 0 &&
2761 n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
2762 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2763 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2764 match(Set dst (LoadVectorGather mem idx));
2765 effect(TEMP ptmp, KILL cr);
2766 ins_cost(2 * SVE_COST + INSN_COST);
2767 format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (S)" %}
2768 ins_encode %{
2769 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this));
2770 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
2771 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2772 %}
2773 ins_pipe(pipe_slow);
2774 %}
2775
2776 instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2777 predicate(UseSVE > 0 &&
2778 n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
2779 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2780 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2781 match(Set dst (LoadVectorGather mem idx));
2782 effect(TEMP ptmp, KILL cr);
2783 ins_cost(3 * SVE_COST + INSN_COST);
2784 format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (D)" %}
2785 ins_encode %{
2786 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2787 Matcher::vector_length(this));
2788 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2789 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
2790 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2791 %}
2792 ins_pipe(pipe_slow);
2793 %}
2794
2795 // ------------------------------ Vector Load Gather Predicated -------------------------------
2796
2797 instruct gatherI_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
2798 predicate(UseSVE > 0 &&
2799 n->as_LoadVector()->memory_size() == MaxVectorSize &&
2800 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2801 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2802 match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
2803 ins_cost(SVE_COST);
2804 format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (S)" %}
2805 ins_encode %{
2806 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
2807 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2808 %}
2809 ins_pipe(pipe_slow);
2810 %}
2811
2812 instruct gatherL_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
2813 predicate(UseSVE > 0 &&
2814 n->as_LoadVector()->memory_size() == MaxVectorSize &&
2815 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2816 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2817 match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
2818 ins_cost(2 * SVE_COST);
2819 format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (D)" %}
2820 ins_encode %{
2821 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2822 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
2823 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2824 %}
2825 ins_pipe(pipe_slow);
2826 %}
2827
2828 // ------------------------------ Vector Load Gather Predicated Partial -------------------------------
2829
2830 instruct gatherI_masked_partial(vReg dst, indirect mem, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2831 predicate(UseSVE > 0 &&
2832 n->as_LoadVector()->memory_size() < MaxVectorSize &&
2833 (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2834 n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2835 match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
2836 effect(TEMP ptmp, KILL cr);
2837 ins_cost(3 * SVE_COST);
2838 format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated partial (S)" %}
2839 ins_encode %{
2840 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2841 Matcher::vector_length(this));
2842 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2843 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2844 __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
2845 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2846 %}
2847 ins_pipe(pipe_slow);
2848 %}
2849
2850 instruct gatherL_masked_partial(vReg dst, indirect mem, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2851 predicate(UseSVE > 0 &&
2852 n->as_LoadVector()->memory_size() < MaxVectorSize &&
2853 (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2854 n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2855 match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
2856 effect(TEMP ptmp, KILL cr);
2857 ins_cost(4 * SVE_COST);
2858 format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated partial (D)" %}
2859 ins_encode %{
2860 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this));
2861 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2862 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2863 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2864 __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
2865 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2866 %}
2867 ins_pipe(pipe_slow);
2868 %}
2869
2870 // ------------------------------ Vector Store Scatter -------------------------------
2871
2872 instruct scatterI(indirect mem, vReg src, vReg idx) %{
2873 predicate(UseSVE > 0 &&
2874 n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
2875 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2876 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2877 match(Set mem (StoreVectorScatter mem (Binary src idx)));
2878 ins_cost(SVE_COST);
2879 format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (S)" %}
2880 ins_encode %{
2881 __ sve_st1w_scatter(as_FloatRegister($src$$reg), ptrue,
2882 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2883 %}
2884 ins_pipe(pipe_slow);
2885 %}
2886
2887 instruct scatterL(indirect mem, vReg src, vReg idx) %{
2888 predicate(UseSVE > 0 &&
2889 n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
2890 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2891 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2892 match(Set mem (StoreVectorScatter mem (Binary src idx)));
2893 ins_cost(2 * SVE_COST);
2894 format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (D)" %}
2895 ins_encode %{
2896 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2897 __ sve_st1d_scatter(as_FloatRegister($src$$reg), ptrue,
2898 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2899 %}
2900 ins_pipe(pipe_slow);
2901 %}
2902
2903 // ------------------------------ Vector Store Scatter Partial -------------------------------
2904
2905 instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2906 predicate(UseSVE > 0 &&
2907 n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
2908 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2909 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2910 match(Set mem (StoreVectorScatter mem (Binary src idx)));
2911 effect(TEMP ptmp, KILL cr);
2912 ins_cost(2 * SVE_COST + INSN_COST);
2913 format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (S)" %}
2914 ins_encode %{
2915 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2916 Matcher::vector_length(this, $src));
2917 __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
2918 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2919 %}
2920 ins_pipe(pipe_slow);
2921 %}
2922
2923 instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
2924 predicate(UseSVE > 0 &&
2925 n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
2926 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2927 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2928 match(Set mem (StoreVectorScatter mem (Binary src idx)));
2929 effect(TEMP ptmp, KILL cr);
2930 ins_cost(3 * SVE_COST + INSN_COST);
2931 format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (D)" %}
2932 ins_encode %{
2933 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
2934 Matcher::vector_length(this, $src));
2935 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2936 __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
2937 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2938 %}
2939 ins_pipe(pipe_slow);
2940 %}
2941
2942 // ------------------------------ Vector Store Scatter Predicated -------------------------------
2943
2944 instruct scatterI_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
2945 predicate(UseSVE > 0 &&
2946 n->as_StoreVector()->memory_size() == MaxVectorSize &&
2947 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2948 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2949 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
2950 ins_cost(SVE_COST);
2951 format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicate (S)" %}
2952 ins_encode %{
2953 __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
2954 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2955 %}
2956 ins_pipe(pipe_slow);
2957 %}
2958
2959 instruct scatterL_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
2960 predicate(UseSVE > 0 &&
2961 n->as_StoreVector()->memory_size() == MaxVectorSize &&
2962 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
2963 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
2964 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
2965 ins_cost(2 * SVE_COST);
2966 format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated (D)" %}
2967 ins_encode %{
2968 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
2969 __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
2970 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2971 %}
2972 ins_pipe(pipe_slow);
2973 %}
2974
2975 // ------------------------------ Vector Store Scatter Predicated Partial -------------------------------
2976
2977 instruct scatterI_masked_partial(indirect mem, vReg src, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2978 predicate(UseSVE > 0 &&
2979 n->as_StoreVector()->memory_size() < MaxVectorSize &&
2980 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
2981 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
2982 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
2983 effect(TEMP ptmp, KILL cr);
2984 ins_cost(3 * SVE_COST);
2985 format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated partial (S)" %}
2986 ins_encode %{
2987 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
2988 Matcher::vector_length(this, $src));
2989 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
2990 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
2991 __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
2992 as_Register($mem$$base), as_FloatRegister($idx$$reg));
2993 %}
2994 ins_pipe(pipe_slow);
2995 %}
2996
2997 instruct scatterL_masked_partial(indirect mem, vReg src, vReg idx, pRegGov pg, pRegGov ptmp, rFlagsReg cr) %{
2998 predicate(UseSVE > 0 &&
2999 n->as_StoreVector()->memory_size() < MaxVectorSize &&
3000 (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
3001 n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
3002 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
3003 effect(TEMP ptmp, KILL cr);
3004 ins_cost(4 * SVE_COST);
3005 format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated partial (D)" %}
3006 ins_encode %{
3007 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
3008 Matcher::vector_length(this, $src));
3009 __ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
3010 as_PRegister($pg$$reg), as_PRegister($pg$$reg));
3011 __ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
3012 __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
3013 as_Register($mem$$base), as_FloatRegister($idx$$reg));
3014 %}
3015 ins_pipe(pipe_slow);
3016 %}
3017
3018 // ------------------------------ Vector Load Const -------------------------------
3019
3020 instruct loadconB(vReg dst, immI0 src) %{
3021 predicate(UseSVE > 0 &&
3022 n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
3023 match(Set dst (VectorLoadConst src));
3024 ins_cost(SVE_COST);
3025 format %{ "sve_index $dst, 0, 1\t# generate iota indices" %}
3026 ins_encode %{
3027 __ sve_index(as_FloatRegister($dst$$reg), __ B, 0, 1);
3028 %}
3029 ins_pipe(pipe_slow);
3030 %}
3031
3032 // Intrisics for String.indexOf(char)
3033
3034 dnl
3035 define(`STRING_INDEXOF_CHAR', `
3036 instruct string$1_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
3037 iRegI_R0 result, vReg ztmp1, vReg ztmp2,
3038 pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
3039 %{
3040 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
3041 predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::$1));
3042 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
3043
3044 format %{ "String$2 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
3045
3046 ins_encode %{
3047 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
3048 as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
3049 as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), $3 /* isL */);
3050 %}
3051 ins_pipe(pipe_class_memory);
3052 %}')dnl
3053 dnl $1 $2 $3
3054 STRING_INDEXOF_CHAR(L, Latin1, true)
3055 STRING_INDEXOF_CHAR(U, UTF16, false)
3056
3057 // Intrisics for String.compareTo()
3058
3059 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
3060 // these string_compare variants as NEON register type for convenience so that the prototype of
3061 // string_compare can be shared with all variants.
3062
3063 dnl
3064 define(`STRING_COMPARETO', `
3065 instruct string_compare$1_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
3066 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
3067 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
3068 pRegGov_P1 pgtmp2, rFlagsReg cr)
3069 %{
3070 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::$1));
3071 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
3072 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
3073 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
3074
3075 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
3076 ins_encode %{
3077 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
3078 __ string_compare($str1$$Register, $str2$$Register,
3079 $cnt1$$Register, $cnt2$$Register, $result$$Register,
3080 $tmp1$$Register, $tmp2$$Register,
3081 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
3082 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
3083 StrIntrinsicNode::$1);
3084 %}
3085 ins_pipe(pipe_class_memory);
3086 %}')dnl
3087 dnl $1
3088 STRING_COMPARETO(LL)
3089 STRING_COMPARETO(LU)
3090 STRING_COMPARETO(UL)
3091 STRING_COMPARETO(UU)
3092
3093 // ---------------------------- Vector mask reductions ---------------------------
3094 instruct vmask_truecount(iRegINoSp dst, pReg src) %{
3095 predicate(UseSVE > 0 &&
3096 n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3097 match(Set dst (VectorMaskTrueCount src));
3098 ins_cost(SVE_COST);
3099 format %{ "vmask_truecount $dst, $src\t# vector mask truecount (sve)" %}
3100 ins_encode %{
3101 BasicType bt = Matcher::vector_element_basic_type(this, $src);
3102 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3103 __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($src$$reg));
3104 %}
3105 ins_pipe(pipe_slow);
3106 %}
3107
3108 instruct vmask_firsttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
3109 predicate(UseSVE > 0 &&
3110 n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3111 match(Set dst (VectorMaskFirstTrue src));
3112 effect(TEMP ptmp);
3113 ins_cost(2 * SVE_COST);
3114 format %{ "vmask_firsttrue $dst, $src\t# vector mask firsttrue (sve)" %}
3115 ins_encode %{
3116 BasicType bt = Matcher::vector_element_basic_type(this, $src);
3117 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3118 __ sve_brkb(as_PRegister($ptmp$$reg), ptrue, as_PRegister($src$$reg), false);
3119 __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($ptmp$$reg));
3120 %}
3121 ins_pipe(pipe_slow);
3122 %}
3123
3124 instruct vmask_lasttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
3125 predicate(UseSVE > 0 &&
3126 n->in(1)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
3127 match(Set dst (VectorMaskLastTrue src));
3128 effect(TEMP ptmp);
3129 ins_cost(3 * SVE_COST);
3130 format %{ "vmask_lasttrue $dst, $src\t# vector mask lasttrue (sve)" %}
3131 ins_encode %{
3132 BasicType bt = Matcher::vector_element_basic_type(this, $src);
3133 __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($src$$reg), as_PRegister($ptmp$$reg));
3134 %}
3135 ins_pipe(pipe_slow);
3136 %}
3137
3138 instruct vmask_truecount_partial(iRegINoSp dst, pReg src, pReg ptmp, rFlagsReg cr) %{
3139 predicate(UseSVE > 0 &&
3140 n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3141 match(Set dst (VectorMaskTrueCount src));
3142 effect(TEMP ptmp, KILL cr);
3143 ins_cost(2 * SVE_COST);
3144 format %{ "vmask_truecount_partial $dst, $src\t# vector mask truecount partial (sve)" %}
3145 ins_encode %{
3146 BasicType bt = Matcher::vector_element_basic_type(this, $src);
3147 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3148 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
3149 __ sve_cntp($dst$$Register, size, as_PRegister($ptmp$$reg), as_PRegister($src$$reg));
3150 %}
3151 ins_pipe(pipe_slow);
3152 %}
3153
3154 instruct vmask_firsttrue_partial(iRegINoSp dst, pReg src, pReg ptmp1, pReg ptmp2, rFlagsReg cr) %{
3155 predicate(UseSVE > 0 &&
3156 n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3157 match(Set dst (VectorMaskFirstTrue src));
3158 effect(TEMP ptmp1, TEMP ptmp2, KILL cr);
3159 ins_cost(3 * SVE_COST);
3160 format %{ "vmask_firsttrue_partial $dst, $src\t# vector mask firsttrue partial (sve)" %}
3161 ins_encode %{
3162 BasicType bt = Matcher::vector_element_basic_type(this, $src);
3163 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3164 __ sve_whilelo_zr_imm(as_PRegister($ptmp1$$reg), size,
3165 Matcher::vector_length(this, $src));
3166 __ sve_brkb(as_PRegister($ptmp2$$reg), as_PRegister($ptmp1$$reg), as_PRegister($src$$reg), false);
3167 __ sve_cntp($dst$$Register, size, as_PRegister($ptmp1$$reg), as_PRegister($ptmp2$$reg));
3168 %}
3169 ins_pipe(pipe_slow);
3170 %}
3171
3172 instruct vmask_lasttrue_partial(iRegINoSp dst, pReg src, pReg ptmp, rFlagsReg cr) %{
3173 predicate(UseSVE > 0 &&
3174 n->in(1)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
3175 match(Set dst (VectorMaskLastTrue src));
3176 effect(TEMP ptmp, KILL cr);
3177 ins_cost(5 * SVE_COST);
3178 format %{ "vmask_lasttrue_partial $dst, $src\t# vector mask lasttrue partial (sve)" %}
3179 ins_encode %{
3180 BasicType bt = Matcher::vector_element_basic_type(this, $src);
3181 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3182 __ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
3183 __ sve_and(as_PRegister($ptmp$$reg), ptrue, as_PRegister($ptmp$$reg), as_PRegister($src$$reg));
3184 __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg));
3185 %}
3186 ins_pipe(pipe_slow);
3187 %}
3188
3189 instruct vmask_tolong(iRegLNoSp dst, pReg src, vReg vtmp1, vReg vtmp2, pRegGov pgtmp, rFlagsReg cr) %{
3190 predicate(UseSVE > 0 &&
3191 n->in(1)->bottom_type()->is_vect()->length() <= 64);
3192 match(Set dst (VectorMaskToLong src));
3193 effect(TEMP vtmp1, TEMP vtmp2, TEMP pgtmp, KILL cr);
3194 ins_cost(13 * SVE_COST);
3195 format %{ "vmask_tolong $dst, $src\t# vector mask tolong (sve)" %}
3196 ins_encode %{
3197 __ sve_vmask_tolong(as_Register($dst$$reg), as_PRegister($src$$reg),
3198 Matcher::vector_element_basic_type(this, $src),
3199 Matcher::vector_length(this, $src),
3200 as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg),
3201 as_PRegister($pgtmp$$reg));
3202 %}
3203 ins_pipe(pipe_slow);
3204 %}dnl
3205
3206 // ---------------------------- Vector mask generation ---------------------------
3207 instruct vmask_gen(pRegGov pg, iRegL len, rFlagsReg cr) %{
3208 predicate(UseSVE > 0);
3209 match(Set pg (VectorMaskGen len));
3210 effect(KILL cr);
3211 ins_cost(SVE_COST);
3212 format %{ "sve_whilelo $pg, zr, $len\t # sve" %}
3213 ins_encode %{
3214 BasicType bt = Matcher::vector_element_basic_type(this);
3215 Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
3216 __ sve_whilelo(as_PRegister($pg$$reg), size, zr, as_Register($len$$reg));
3217 %}
3218 ins_pipe(pipe_slow);
3219 %}