1 //
2 // Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2020, 2023, Arm Limited. All rights reserved.
4 // Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
5 // Copyright (c) 2023, 2025, Rivos Inc. All rights reserved.
6 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
7 //
8 // This code is free software; you can redistribute it and/or modify it
9 // under the terms of the GNU General Public License version 2 only, as
10 // published by the Free Software Foundation.
11 //
12 // This code is distributed in the hope that it will be useful, but WITHOUT
13 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 // version 2 for more details (a copy is included in the LICENSE file that
16 // accompanied this code).
17 //
18 // You should have received a copy of the GNU General Public License version
19 // 2 along with this work; if not, write to the Free Software Foundation,
20 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
21 //
22 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
23 // or visit www.oracle.com if you need additional information or have any
24 // questions.
25 //
26 //
27
28 // RISCV Vector Extension Architecture Description File
29
30 opclass vmemA(indirect);
31
32 source %{
33
34 static void loadStore(C2_MacroAssembler* masm, bool is_store,
35 VectorRegister reg, BasicType bt, Register base,
36 uint vector_length, Assembler::VectorMask vm = Assembler::unmasked) {
37 Assembler::SEW sew = Assembler::elemtype_to_sew(bt);
38 __ vsetvli_helper(bt, vector_length);
39
40 if (is_store) {
41 __ vsex_v(reg, base, sew, vm);
42 } else {
43 if (vm == Assembler::v0_t) {
44 __ vxor_vv(reg, reg, reg);
45 }
46 __ vlex_v(reg, base, sew, vm);
47 }
48 }
49
50 bool Matcher::match_rule_supported_auto_vectorization(int opcode, int vlen, BasicType bt) {
51 return match_rule_supported_vector(opcode, vlen, bt);
52 }
53
54 // Identify extra cases that we might want to provide match rules for vector nodes
55 // and other intrinsics guarded with vector length (vlen) and element type (bt).
56 bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
57 if (!UseRVV) {
58 return false;
59 }
60
61 if (!match_rule_supported(opcode) || !vector_size_supported(bt, vlen)) {
62 return false;
63 }
64
65 switch (opcode) {
66 case Op_VectorMaskLastTrue:
67 if (!UseZbb || vlen > XLEN) {
68 return false;
69 }
70 break;
71 case Op_VectorMaskToLong:
72 case Op_VectorLongToMask:
73 if (vlen > XLEN) {
74 return false;
75 }
76 break;
77 case Op_CountTrailingZerosV:
78 case Op_CountLeadingZerosV:
79 case Op_PopCountVL:
80 case Op_PopCountVI:
81 case Op_ReverseBytesV:
82 case Op_ReverseV:
83 return UseZvbb;
84 case Op_RotateLeftV:
85 case Op_RotateRightV:
86 if (bt != T_INT && bt != T_LONG) {
87 return false;
88 }
89 return UseZvbb;
90 case Op_LoadVectorGather:
91 case Op_LoadVectorGatherMasked:
92 case Op_StoreVectorScatter:
93 case Op_StoreVectorScatterMasked:
94 if (is_subword_type(bt)) {
95 return false;
96 }
97 break;
98 case Op_VectorLoadShuffle:
99 case Op_VectorRearrange:
100 // vlen >= 4 is required, because min vector size for byte is 4 on riscv,
101 // VectorLoadShuffle is from byte to X, so it requires vlen >= 4.
102 // VectorRearrange depends on VectorLoadShuffle, so it also requires vlen >= 4.
103 if (vlen < 4) {
104 return false;
105 }
106 break;
107 case Op_MulReductionVI:
108 case Op_MulReductionVL:
109 // When vlen < 4, our log2(vlen) implementation does not help to gain performance improvement.
110 if (vlen < 4) {
111 return false;
112 }
113 break;
114 case Op_VectorCastHF2F:
115 case Op_VectorCastF2HF:
116 return UseZvfh || UseZvfhmin;
117 case Op_AddVHF:
118 case Op_SubVHF:
119 case Op_MulVHF:
120 case Op_DivVHF:
121 case Op_MaxVHF:
122 case Op_MinVHF:
123 case Op_SqrtVHF:
124 return UseZvfh;
125 case Op_FmaVHF:
126 return UseZvfh && UseFMA;
127 case Op_FmaVF:
128 case Op_FmaVD:
129 return UseFMA;
130
131 // For float, current test shows that, it brings performance gain when vlen >= 8, but brings
132 // regression when vlen == 4. So only enable this intrinsic when vlen >= 8.
133 // For double, current test shows that even with vlen == 4, there is still some regression.
134 // Although there is no hardware to verify it, from the trend of performance data on hardwares
135 // (with vlen == 2 and 4 respectively), it's promising to bring better performance rather than
136 // regression for double when vlen == 8. So only enable this intrinsic when vlen >= 8.
137 case Op_RoundVF:
138 case Op_RoundVD:
139 return vlen >= 8;
140
141 default:
142 break;
143 }
144 return true;
145 }
146
147 bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
148 if (!UseRVV) {
149 return false;
150 }
151
152 switch (opcode) {
153 case Op_SelectFromTwoVector:
154 // There is no masked version of selectFrom two vector, i.e. selectFrom(av, bv, mask) in vector API.
155 return false;
156 // Currently, the masked versions of the following 8 Float16 operations are disabled.
157 // When the support for Float16 vector classes is added in VectorAPI and the masked
158 // Float16 IR can be generated, these masked operations will be enabled and relevant
159 // backend support added.
160 case Op_AddVHF:
161 case Op_SubVHF:
162 case Op_MulVHF:
163 case Op_DivVHF:
164 case Op_MaxVHF:
165 case Op_MinVHF:
166 case Op_SqrtVHF:
167 case Op_FmaVHF:
168 return false;
169 default:
170 break;
171 }
172
173 return match_rule_supported_vector(opcode, vlen, bt);
174 }
175
176 bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* vt) {
177 return false;
178 }
179
180 bool Matcher::vector_rearrange_requires_load_shuffle(BasicType elem_bt, int vlen) {
181 return false;
182 }
183
184 bool Matcher::mask_op_prefers_predicate(int opcode, const TypeVect* vt) {
185 // Prefer predicate if the mask type is "TypePVectMask".
186 return vt->isa_pvectmask() != nullptr;
187 }
188 %}
189
190 // All VEC instructions
191
192 // vector load/store
193 instruct loadV(vReg dst, vmemA mem) %{
194 match(Set dst (LoadVector mem));
195 format %{ "loadV $dst, $mem\t# vector (rvv)" %}
196 ins_encode %{
197 VectorRegister dst_reg = as_VectorRegister($dst$$reg);
198 loadStore(masm, false, dst_reg,
199 Matcher::vector_element_basic_type(this), as_Register($mem$$base), Matcher::vector_length(this));
200 %}
201 ins_pipe(pipe_slow);
202 %}
203
204 instruct storeV(vReg src, vmemA mem) %{
205 match(Set mem (StoreVector mem src));
206 format %{ "storeV $mem, $src\t# vector (rvv)" %}
207 ins_encode %{
208 VectorRegister src_reg = as_VectorRegister($src$$reg);
209 loadStore(masm, true, src_reg,
210 Matcher::vector_element_basic_type(this, $src), as_Register($mem$$base), Matcher::vector_length(this, $src));
211 %}
212 ins_pipe(pipe_slow);
213 %}
214
215 // vector load mask
216
217 instruct vloadmask(vRegMask dst, vReg src) %{
218 match(Set dst (VectorLoadMask src));
219 format %{ "vloadmask $dst, $src" %}
220 ins_encode %{
221 __ vsetvli_helper(T_BOOLEAN, Matcher::vector_length(this));
222 __ vmsne_vx(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), zr);
223 %}
224 ins_pipe(pipe_slow);
225 %}
226
227 instruct vloadmask_masked(vRegMask dst, vReg src, vRegMask_V0 v0) %{
228 match(Set dst (VectorLoadMask src v0));
229 format %{ "vloadmask_masked $dst, $src, $v0" %}
230 ins_encode %{
231 __ vsetvli_helper(T_BOOLEAN, Matcher::vector_length(this));
232 __ vmsne_vx(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), zr, Assembler::v0_t);
233 %}
234 ins_pipe(pipe_slow);
235 %}
236
237 // vector store mask
238
239 instruct vstoremask(vReg dst, vRegMask_V0 v0, immI size) %{
240 match(Set dst (VectorStoreMask v0 size));
241 format %{ "vstoremask $dst, V0 # elem size is $size byte[s]" %}
242 ins_encode %{
243 __ vsetvli_helper(T_BOOLEAN, Matcher::vector_length(this));
244 __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg));
245 __ vmerge_vim(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg), 1);
246 %}
247 ins_pipe(pipe_slow);
248 %}
249
250 // vector mask compare
251
252 instruct vmaskcmp(vRegMask dst, vReg src1, vReg src2, immI cond) %{
253 predicate(Matcher::vector_element_basic_type(n) == T_BYTE ||
254 Matcher::vector_element_basic_type(n) == T_SHORT ||
255 Matcher::vector_element_basic_type(n) == T_INT ||
256 Matcher::vector_element_basic_type(n) == T_LONG);
257 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
258 format %{ "vmaskcmp $dst, $src1, $src2, $cond" %}
259 ins_encode %{
260 BasicType bt = Matcher::vector_element_basic_type(this);
261 uint vector_length = Matcher::vector_length(this);
262 __ compare_integral_v(as_VectorRegister($dst$$reg),
263 as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg),
264 (int)($cond$$constant), bt, vector_length);
265 %}
266 ins_pipe(pipe_slow);
267 %}
268
269 instruct vmaskcmp_masked(vRegMask dst, vReg src1, vReg src2, immI cond, vRegMask_V0 v0) %{
270 predicate(Matcher::vector_element_basic_type(n) == T_BYTE ||
271 Matcher::vector_element_basic_type(n) == T_SHORT ||
272 Matcher::vector_element_basic_type(n) == T_INT ||
273 Matcher::vector_element_basic_type(n) == T_LONG);
274 match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond v0)));
275 effect(TEMP_DEF dst);
276 format %{ "vmaskcmp_masked $dst, $src1, $src2, $cond, $v0" %}
277 ins_encode %{
278 BasicType bt = Matcher::vector_element_basic_type(this);
279 uint vector_length = Matcher::vector_length(this);
280 __ compare_integral_v(as_VectorRegister($dst$$reg),
281 as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg),
282 (int)($cond$$constant), bt, vector_length, Assembler::v0_t);
283 %}
284 ins_pipe(pipe_slow);
285 %}
286
287 // vector mask float compare
288
289 instruct vmaskcmp_fp(vRegMask dst, vReg src1, vReg src2, immI cond) %{
290 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT ||
291 Matcher::vector_element_basic_type(n) == T_DOUBLE);
292 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
293 format %{ "vmaskcmp_fp $dst, $src1, $src2, $cond" %}
294 ins_encode %{
295 BasicType bt = Matcher::vector_element_basic_type(this);
296 uint vector_length = Matcher::vector_length(this);
297 __ compare_fp_v(as_VectorRegister($dst$$reg),
298 as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg),
299 (int)($cond$$constant), bt, vector_length);
300 %}
301 ins_pipe(pipe_slow);
302 %}
303
304 instruct vmaskcmp_fp_masked(vRegMask dst, vReg src1, vReg src2, immI cond, vRegMask_V0 v0) %{
305 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT ||
306 Matcher::vector_element_basic_type(n) == T_DOUBLE);
307 match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond v0)));
308 effect(TEMP_DEF dst);
309 format %{ "vmaskcmp_fp_masked $dst, $src1, $src2, $cond, $v0" %}
310 ins_encode %{
311 BasicType bt = Matcher::vector_element_basic_type(this);
312 uint vector_length = Matcher::vector_length(this);
313 __ compare_fp_v(as_VectorRegister($dst$$reg),
314 as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg),
315 (int)($cond$$constant), bt, vector_length, Assembler::v0_t);
316 %}
317 ins_pipe(pipe_slow);
318 %}
319
320 // vector abs
321
322 instruct vabs(vReg dst, vReg src, vReg tmp) %{
323 match(Set dst (AbsVB src));
324 match(Set dst (AbsVS src));
325 match(Set dst (AbsVI src));
326 match(Set dst (AbsVL src));
327 effect(TEMP tmp);
328 format %{ "vabs $dst, $src\t# KILL $tmp" %}
329 ins_encode %{
330 BasicType bt = Matcher::vector_element_basic_type(this);
331 __ vsetvli_helper(bt, Matcher::vector_length(this));
332 __ vrsub_vi(as_VectorRegister($tmp$$reg), as_VectorRegister($src$$reg), 0);
333 __ vmax_vv(as_VectorRegister($dst$$reg), as_VectorRegister($tmp$$reg), as_VectorRegister($src$$reg));
334 %}
335 ins_pipe(pipe_slow);
336 %}
337
338 instruct vabs_fp(vReg dst, vReg src) %{
339 match(Set dst (AbsVF src));
340 match(Set dst (AbsVD src));
341 format %{ "vabs_fp $dst, $src" %}
342 ins_encode %{
343 BasicType bt = Matcher::vector_element_basic_type(this);
344 __ vsetvli_helper(bt, Matcher::vector_length(this));
345 __ vfabs_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg));
346 %}
347 ins_pipe(pipe_slow);
348 %}
349
350 // vector abs - predicated
351
352 instruct vabs_masked(vReg dst_src, vRegMask_V0 v0, vReg tmp) %{
353 match(Set dst_src (AbsVB dst_src v0));
354 match(Set dst_src (AbsVS dst_src v0));
355 match(Set dst_src (AbsVI dst_src v0));
356 match(Set dst_src (AbsVL dst_src v0));
357 effect(TEMP tmp);
358 format %{ "vabs_masked $dst_src, $dst_src, $v0\t# KILL $tmp" %}
359 ins_encode %{
360 BasicType bt = Matcher::vector_element_basic_type(this);
361 __ vsetvli_helper(bt, Matcher::vector_length(this));
362 __ vrsub_vi(as_VectorRegister($tmp$$reg), as_VectorRegister($dst_src$$reg), 0,
363 Assembler::v0_t);
364 __ vmax_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($tmp$$reg),
365 as_VectorRegister($dst_src$$reg), Assembler::v0_t);
366 %}
367 ins_pipe(pipe_slow);
368 %}
369
370 instruct vabs_fp_masked(vReg dst_src, vRegMask_V0 v0) %{
371 match(Set dst_src (AbsVF dst_src v0));
372 match(Set dst_src (AbsVD dst_src v0));
373 format %{ "vabs_fp_masked $dst_src, $dst_src, $v0" %}
374 ins_encode %{
375 BasicType bt = Matcher::vector_element_basic_type(this);
376 __ vsetvli_helper(bt, Matcher::vector_length(this));
377 __ vfabs_v(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg), Assembler::v0_t);
378 %}
379 ins_pipe(pipe_slow);
380 %}
381
382 // vector add
383
384 instruct vadd(vReg dst, vReg src1, vReg src2) %{
385 match(Set dst (AddVB src1 src2));
386 match(Set dst (AddVS src1 src2));
387 match(Set dst (AddVI src1 src2));
388 match(Set dst (AddVL src1 src2));
389 format %{ "vadd $dst, $src1, $src2" %}
390 ins_encode %{
391 BasicType bt = Matcher::vector_element_basic_type(this);
392 __ vsetvli_helper(bt, Matcher::vector_length(this));
393 __ vadd_vv(as_VectorRegister($dst$$reg),
394 as_VectorRegister($src1$$reg),
395 as_VectorRegister($src2$$reg));
396 %}
397 ins_pipe(pipe_slow);
398 %}
399
400 instruct vadd_hfp(vReg dst, vReg src1, vReg src2) %{
401 match(Set dst (AddVHF src1 src2));
402 format %{ "vadd_hfp $dst, $src1, $src2" %}
403 ins_encode %{
404 assert(UseZvfh, "must");
405 assert(Matcher::vector_element_basic_type(this) == T_SHORT, "must");
406 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
407 __ vfadd_vv(as_VectorRegister($dst$$reg),
408 as_VectorRegister($src1$$reg),
409 as_VectorRegister($src2$$reg));
410 %}
411 ins_pipe(pipe_slow);
412 %}
413
414 instruct vadd_fp(vReg dst, vReg src1, vReg src2) %{
415 match(Set dst (AddVF src1 src2));
416 match(Set dst (AddVD src1 src2));
417 format %{ "vadd_fp $dst, $src1, $src2" %}
418 ins_encode %{
419 BasicType bt = Matcher::vector_element_basic_type(this);
420 __ vsetvli_helper(bt, Matcher::vector_length(this));
421 __ vfadd_vv(as_VectorRegister($dst$$reg),
422 as_VectorRegister($src1$$reg),
423 as_VectorRegister($src2$$reg));
424 %}
425 ins_pipe(pipe_slow);
426 %}
427
428 // vector add - predicated
429
430 instruct vadd_masked(vReg dst_src1, vReg src2, vRegMask_V0 v0) %{
431 match(Set dst_src1 (AddVB (Binary dst_src1 src2) v0));
432 match(Set dst_src1 (AddVS (Binary dst_src1 src2) v0));
433 match(Set dst_src1 (AddVI (Binary dst_src1 src2) v0));
434 match(Set dst_src1 (AddVL (Binary dst_src1 src2) v0));
435 format %{ "vadd_masked $dst_src1, $dst_src1, $src2, $v0" %}
436 ins_encode %{
437 BasicType bt = Matcher::vector_element_basic_type(this);
438 __ vsetvli_helper(bt, Matcher::vector_length(this));
439 __ vadd_vv(as_VectorRegister($dst_src1$$reg),
440 as_VectorRegister($dst_src1$$reg),
441 as_VectorRegister($src2$$reg), Assembler::v0_t);
442 %}
443 ins_pipe(pipe_slow);
444 %}
445
446 instruct vadd_fp_masked(vReg dst_src1, vReg src2, vRegMask_V0 v0) %{
447 match(Set dst_src1 (AddVF (Binary dst_src1 src2) v0));
448 match(Set dst_src1 (AddVD (Binary dst_src1 src2) v0));
449 format %{ "vadd_fp_masked $dst_src1, $dst_src1, $src2, $v0" %}
450 ins_encode %{
451 BasicType bt = Matcher::vector_element_basic_type(this);
452 __ vsetvli_helper(bt, Matcher::vector_length(this));
453 __ vfadd_vv(as_VectorRegister($dst_src1$$reg),
454 as_VectorRegister($dst_src1$$reg),
455 as_VectorRegister($src2$$reg), Assembler::v0_t);
456 %}
457 ins_pipe(pipe_slow);
458 %}
459
460 // vector-immediate add (unpredicated)
461
462 instruct vadd_vi(vReg dst, vReg src1, immI5 con) %{
463 match(Set dst (AddVB src1 (Replicate con)));
464 match(Set dst (AddVS src1 (Replicate con)));
465 match(Set dst (AddVI src1 (Replicate con)));
466 format %{ "vadd_vi $dst, $src1, $con" %}
467 ins_encode %{
468 BasicType bt = Matcher::vector_element_basic_type(this);
469 __ vsetvli_helper(bt, Matcher::vector_length(this));
470 __ vadd_vi(as_VectorRegister($dst$$reg),
471 as_VectorRegister($src1$$reg),
472 $con$$constant);
473 %}
474 ins_pipe(pipe_slow);
475 %}
476
477 instruct vaddL_vi(vReg dst, vReg src1, immL5 con) %{
478 match(Set dst (AddVL src1 (Replicate con)));
479 format %{ "vaddL_vi $dst, $src1, $con" %}
480 ins_encode %{
481 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
482 __ vadd_vi(as_VectorRegister($dst$$reg),
483 as_VectorRegister($src1$$reg),
484 $con$$constant);
485 %}
486 ins_pipe(pipe_slow);
487 %}
488
489 // vector-scalar add (unpredicated)
490
491 instruct vadd_vx(vReg dst, vReg src1, iRegIorL2I src2) %{
492 match(Set dst (AddVB src1 (Replicate src2)));
493 match(Set dst (AddVS src1 (Replicate src2)));
494 match(Set dst (AddVI src1 (Replicate src2)));
495 format %{ "vadd_vx $dst, $src1, $src2" %}
496 ins_encode %{
497 BasicType bt = Matcher::vector_element_basic_type(this);
498 __ vsetvli_helper(bt, Matcher::vector_length(this));
499 __ vadd_vx(as_VectorRegister($dst$$reg),
500 as_VectorRegister($src1$$reg),
501 as_Register($src2$$reg));
502 %}
503 ins_pipe(pipe_slow);
504 %}
505
506 instruct vaddL_vx(vReg dst, vReg src1, iRegL src2) %{
507 match(Set dst (AddVL src1 (Replicate src2)));
508 format %{ "vaddL_vx $dst, $src1, $src2" %}
509 ins_encode %{
510 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
511 __ vadd_vx(as_VectorRegister($dst$$reg),
512 as_VectorRegister($src1$$reg),
513 as_Register($src2$$reg));
514 %}
515 ins_pipe(pipe_slow);
516 %}
517
518 // vector-immediate add (predicated)
519
520 instruct vadd_vi_masked(vReg dst_src, immI5 con, vRegMask_V0 v0) %{
521 match(Set dst_src (AddVB (Binary dst_src (Replicate con)) v0));
522 match(Set dst_src (AddVS (Binary dst_src (Replicate con)) v0));
523 match(Set dst_src (AddVI (Binary dst_src (Replicate con)) v0));
524 format %{ "vadd_vi_masked $dst_src, $dst_src, $con, $v0" %}
525 ins_encode %{
526 BasicType bt = Matcher::vector_element_basic_type(this);
527 __ vsetvli_helper(bt, Matcher::vector_length(this));
528 __ vadd_vi(as_VectorRegister($dst_src$$reg),
529 as_VectorRegister($dst_src$$reg),
530 $con$$constant, Assembler::v0_t);
531 %}
532 ins_pipe(pipe_slow);
533 %}
534
535 instruct vaddL_vi_masked(vReg dst_src, immL5 con, vRegMask_V0 v0) %{
536 match(Set dst_src (AddVL (Binary dst_src (Replicate con)) v0));
537 format %{ "vaddL_vi_masked $dst_src, $dst_src, $con, $v0" %}
538 ins_encode %{
539 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
540 __ vadd_vi(as_VectorRegister($dst_src$$reg),
541 as_VectorRegister($dst_src$$reg),
542 $con$$constant, Assembler::v0_t);
543 %}
544 ins_pipe(pipe_slow);
545 %}
546
547 // vector-scalar add (predicated)
548
549 instruct vadd_vx_masked(vReg dst_src, iRegIorL2I src2, vRegMask_V0 v0) %{
550 match(Set dst_src (AddVB (Binary dst_src (Replicate src2)) v0));
551 match(Set dst_src (AddVS (Binary dst_src (Replicate src2)) v0));
552 match(Set dst_src (AddVI (Binary dst_src (Replicate src2)) v0));
553 format %{ "vadd_vx_masked $dst_src, $dst_src, $src2, $v0" %}
554 ins_encode %{
555 BasicType bt = Matcher::vector_element_basic_type(this);
556 __ vsetvli_helper(bt, Matcher::vector_length(this));
557 __ vadd_vx(as_VectorRegister($dst_src$$reg),
558 as_VectorRegister($dst_src$$reg),
559 as_Register($src2$$reg), Assembler::v0_t);
560 %}
561 ins_pipe(pipe_slow);
562 %}
563
564 instruct vaddL_vx_masked(vReg dst_src, iRegL src2, vRegMask_V0 v0) %{
565 match(Set dst_src (AddVL (Binary dst_src (Replicate src2)) v0));
566 format %{ "vaddL_vx_masked $dst_src, $dst_src, $src2, $v0" %}
567 ins_encode %{
568 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
569 __ vadd_vx(as_VectorRegister($dst_src$$reg),
570 as_VectorRegister($dst_src$$reg),
571 as_Register($src2$$reg), Assembler::v0_t);
572 %}
573 ins_pipe(pipe_slow);
574 %}
575
576 // vector sub
577
578 instruct vsub(vReg dst, vReg src1, vReg src2) %{
579 match(Set dst (SubVB src1 src2));
580 match(Set dst (SubVS src1 src2));
581 match(Set dst (SubVI src1 src2));
582 match(Set dst (SubVL src1 src2));
583 format %{ "vsub $dst, $src1, $src2" %}
584 ins_encode %{
585 BasicType bt = Matcher::vector_element_basic_type(this);
586 __ vsetvli_helper(bt, Matcher::vector_length(this));
587 __ vsub_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src1$$reg),
588 as_VectorRegister($src2$$reg));
589 %}
590 ins_pipe(pipe_slow);
591 %}
592
593 instruct vsub_hfp(vReg dst, vReg src1, vReg src2) %{
594 match(Set dst (SubVHF src1 src2));
595 format %{ "vsub_hfp $dst, $src1, $src2" %}
596 ins_encode %{
597 assert(UseZvfh, "must");
598 assert(Matcher::vector_element_basic_type(this) == T_SHORT, "must");
599 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
600 __ vfsub_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src1$$reg),
601 as_VectorRegister($src2$$reg));
602 %}
603 ins_pipe(pipe_slow);
604 %}
605
606 instruct vsub_fp(vReg dst, vReg src1, vReg src2) %{
607 match(Set dst (SubVF src1 src2));
608 match(Set dst (SubVD src1 src2));
609 format %{ "vsub_fp $dst, $src1, $src2" %}
610 ins_encode %{
611 BasicType bt = Matcher::vector_element_basic_type(this);
612 __ vsetvli_helper(bt, Matcher::vector_length(this));
613 __ vfsub_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src1$$reg),
614 as_VectorRegister($src2$$reg));
615 %}
616 ins_pipe(pipe_slow);
617 %}
618
619 // vector sub - predicated
620
621 instruct vsub_masked(vReg dst_src1, vReg src2, vRegMask_V0 v0) %{
622 match(Set dst_src1 (SubVB (Binary dst_src1 src2) v0));
623 match(Set dst_src1 (SubVS (Binary dst_src1 src2) v0));
624 match(Set dst_src1 (SubVI (Binary dst_src1 src2) v0));
625 match(Set dst_src1 (SubVL (Binary dst_src1 src2) v0));
626 format %{ "vsub_masked $dst_src1, $dst_src1, $src2, $v0" %}
627 ins_encode %{
628 BasicType bt = Matcher::vector_element_basic_type(this);
629 __ vsetvli_helper(bt, Matcher::vector_length(this));
630 __ vsub_vv(as_VectorRegister($dst_src1$$reg), as_VectorRegister($dst_src1$$reg),
631 as_VectorRegister($src2$$reg), Assembler::v0_t);
632 %}
633 ins_pipe(pipe_slow);
634 %}
635
636 instruct vsub_fp_masked(vReg dst_src1, vReg src2, vRegMask_V0 v0) %{
637 match(Set dst_src1 (SubVF (Binary dst_src1 src2) v0));
638 match(Set dst_src1 (SubVD (Binary dst_src1 src2) v0));
639 format %{ "vsub_fp_masked $dst_src1, $dst_src1, $src2, $v0" %}
640 ins_encode %{
641 BasicType bt = Matcher::vector_element_basic_type(this);
642 __ vsetvli_helper(bt, Matcher::vector_length(this));
643 __ vfsub_vv(as_VectorRegister($dst_src1$$reg), as_VectorRegister($dst_src1$$reg),
644 as_VectorRegister($src2$$reg), Assembler::v0_t);
645 %}
646 ins_pipe(pipe_slow);
647 %}
648
649 // vector-scalar sub (unpredicated)
650
651 instruct vsub_vx(vReg dst, vReg src1, iRegIorL2I src2) %{
652 match(Set dst (SubVB src1 (Replicate src2)));
653 match(Set dst (SubVS src1 (Replicate src2)));
654 match(Set dst (SubVI src1 (Replicate src2)));
655 format %{ "vsub_vx $dst, $src1, $src2" %}
656 ins_encode %{
657 BasicType bt = Matcher::vector_element_basic_type(this);
658 __ vsetvli_helper(bt, Matcher::vector_length(this));
659 __ vsub_vx(as_VectorRegister($dst$$reg),
660 as_VectorRegister($src1$$reg),
661 as_Register($src2$$reg));
662 %}
663 ins_pipe(pipe_slow);
664 %}
665
666 instruct vsubL_vx(vReg dst, vReg src1, iRegL src2) %{
667 match(Set dst (SubVL src1 (Replicate src2)));
668 format %{ "vsubL_vx $dst, $src1, $src2" %}
669 ins_encode %{
670 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
671 __ vsub_vx(as_VectorRegister($dst$$reg),
672 as_VectorRegister($src1$$reg),
673 as_Register($src2$$reg));
674 %}
675 ins_pipe(pipe_slow);
676 %}
677
678 // vector-scalar sub (predicated)
679
680 instruct vsub_vx_masked(vReg dst_src, iRegIorL2I src2, vRegMask_V0 v0) %{
681 match(Set dst_src (SubVB (Binary dst_src (Replicate src2)) v0));
682 match(Set dst_src (SubVS (Binary dst_src (Replicate src2)) v0));
683 match(Set dst_src (SubVI (Binary dst_src (Replicate src2)) v0));
684 format %{ "vsub_vx_masked $dst_src, $dst_src, $src2, $v0" %}
685 ins_encode %{
686 BasicType bt = Matcher::vector_element_basic_type(this);
687 __ vsetvli_helper(bt, Matcher::vector_length(this));
688 __ vsub_vx(as_VectorRegister($dst_src$$reg),
689 as_VectorRegister($dst_src$$reg),
690 as_Register($src2$$reg), Assembler::v0_t);
691 %}
692 ins_pipe(pipe_slow);
693 %}
694
695 instruct vsubL_vx_masked(vReg dst_src, iRegL src2, vRegMask_V0 v0) %{
696 match(Set dst_src (SubVL (Binary dst_src (Replicate src2)) v0));
697 format %{ "vsubL_vx_masked $dst_src, $dst_src, $src2, $v0" %}
698 ins_encode %{
699 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
700 __ vsub_vx(as_VectorRegister($dst_src$$reg),
701 as_VectorRegister($dst_src$$reg),
702 as_Register($src2$$reg), Assembler::v0_t);
703 %}
704 ins_pipe(pipe_slow);
705 %}
706
707 // -------- vector saturating integer operations
708
709 // vector saturating signed integer addition
710
711 instruct vsadd(vReg dst, vReg src1, vReg src2) %{
712 predicate(n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
713 match(Set dst (SaturatingAddV src1 src2));
714 format %{ "vsadd $dst, $src1, $src2" %}
715 ins_encode %{
716 BasicType bt = Matcher::vector_element_basic_type(this);
717 assert(is_integral_type(bt), "unsupported type");
718 __ vsetvli_helper(bt, Matcher::vector_length(this));
719 __ vsadd_vv(as_VectorRegister($dst$$reg),
720 as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg));
721 %}
722 ins_pipe(pipe_slow);
723 %}
724
725 // vector saturating unsigned integer addition
726
727 instruct vsaddu(vReg dst, vReg src1, vReg src2) %{
728 predicate(n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
729 match(Set dst (SaturatingAddV src1 src2));
730 format %{ "vsaddu $dst, $src1, $src2" %}
731 ins_encode %{
732 BasicType bt = Matcher::vector_element_basic_type(this);
733 assert(is_integral_type(bt), "unsupported type");
734 __ vsetvli_helper(bt, Matcher::vector_length(this));
735 __ vsaddu_vv(as_VectorRegister($dst$$reg),
736 as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg));
737 %}
738 ins_pipe(pipe_slow);
739 %}
740
741 // vector saturating signed integer addition (predicated)
742
743 instruct vsadd_masked(vReg dst_src, vReg src1, vRegMask_V0 v0) %{
744 predicate(n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
745 match(Set dst_src (SaturatingAddV (Binary dst_src src1) v0));
746 format %{ "vsadd_masked $dst_src, $dst_src, $src1, $v0" %}
747 ins_encode %{
748 BasicType bt = Matcher::vector_element_basic_type(this);
749 assert(is_integral_type(bt), "unsupported type");
750 __ vsetvli_helper(bt, Matcher::vector_length(this));
751 __ vsadd_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
752 as_VectorRegister($src1$$reg), Assembler::v0_t);
753 %}
754 ins_pipe(pipe_slow);
755 %}
756
757 // vector saturating unsigned integer addition (predicated)
758
759 instruct vsaddu_masked(vReg dst_src, vReg src1, vRegMask_V0 v0) %{
760 predicate(n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
761 match(Set dst_src (SaturatingAddV (Binary dst_src src1) v0));
762 format %{ "vsaddu_masked $dst_src, $dst_src, $src1, $v0" %}
763 ins_encode %{
764 BasicType bt = Matcher::vector_element_basic_type(this);
765 assert(is_integral_type(bt), "unsupported type");
766 __ vsetvli_helper(bt, Matcher::vector_length(this));
767 __ vsaddu_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
768 as_VectorRegister($src1$$reg), Assembler::v0_t);
769 %}
770 ins_pipe(pipe_slow);
771 %}
772
773 // vector saturating signed integer subtraction
774
775 instruct vssub(vReg dst, vReg src1, vReg src2) %{
776 predicate(n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
777 match(Set dst (SaturatingSubV src1 src2));
778 format %{ "vssub $dst, $src1, $src2" %}
779 ins_encode %{
780 BasicType bt = Matcher::vector_element_basic_type(this);
781 assert(is_integral_type(bt), "unsupported type");
782 __ vsetvli_helper(bt, Matcher::vector_length(this));
783 __ vssub_vv(as_VectorRegister($dst$$reg),
784 as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg));
785 %}
786 ins_pipe(pipe_slow);
787 %}
788
789 // vector saturating unsigned integer subtraction
790
791 instruct vssubu(vReg dst, vReg src1, vReg src2) %{
792 predicate(n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
793 match(Set dst (SaturatingSubV src1 src2));
794 format %{ "vssubu $dst, $src1, $src2" %}
795 ins_encode %{
796 BasicType bt = Matcher::vector_element_basic_type(this);
797 assert(is_integral_type(bt), "unsupported type");
798 __ vsetvli_helper(bt, Matcher::vector_length(this));
799 __ vssubu_vv(as_VectorRegister($dst$$reg),
800 as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg));
801 %}
802 ins_pipe(pipe_slow);
803 %}
804
805 // vector saturating signed integer subtraction (predicated)
806
807 instruct vssub_masked(vReg dst_src, vReg src1, vRegMask_V0 v0) %{
808 predicate(n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
809 match(Set dst_src (SaturatingSubV (Binary dst_src src1) v0));
810 format %{ "vssub_masked $dst_src, $dst_src, $src1, $v0" %}
811 ins_encode %{
812 BasicType bt = Matcher::vector_element_basic_type(this);
813 assert(is_integral_type(bt), "unsupported type");
814 __ vsetvli_helper(bt, Matcher::vector_length(this));
815 __ vssub_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
816 as_VectorRegister($src1$$reg), Assembler::v0_t);
817 %}
818 ins_pipe(pipe_slow);
819 %}
820
821 // vector saturating unsigned integer subtraction (predicated)
822
823 instruct vssubu_masked(vReg dst_src, vReg src1, vRegMask_V0 v0) %{
824 predicate(n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
825 match(Set dst_src (SaturatingSubV (Binary dst_src src1) v0));
826 format %{ "vssubu_masked $dst_src, $dst_src, $src1, $v0" %}
827 ins_encode %{
828 BasicType bt = Matcher::vector_element_basic_type(this);
829 assert(is_integral_type(bt), "unsupported type");
830 __ vsetvli_helper(bt, Matcher::vector_length(this));
831 __ vssubu_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
832 as_VectorRegister($src1$$reg), Assembler::v0_t);
833 %}
834 ins_pipe(pipe_slow);
835 %}
836
837 // vector and
838
839 instruct vand(vReg dst, vReg src1, vReg src2) %{
840 match(Set dst (AndV src1 src2));
841 format %{ "vand $dst, $src1, $src2" %}
842 ins_encode %{
843 BasicType bt = Matcher::vector_element_basic_type(this);
844 __ vsetvli_helper(bt, Matcher::vector_length(this));
845 __ vand_vv(as_VectorRegister($dst$$reg),
846 as_VectorRegister($src1$$reg),
847 as_VectorRegister($src2$$reg));
848 %}
849 ins_pipe(pipe_slow);
850 %}
851
852 // vector and - predicated
853
854 instruct vand_masked(vReg dst_src1, vReg src2, vRegMask_V0 v0) %{
855 match(Set dst_src1 (AndV (Binary dst_src1 src2) v0));
856 format %{ "vand_masked $dst_src1, $dst_src1, $src2, $v0" %}
857 ins_encode %{
858 BasicType bt = Matcher::vector_element_basic_type(this);
859 __ vsetvli_helper(bt, Matcher::vector_length(this));
860 __ vand_vv(as_VectorRegister($dst_src1$$reg),
861 as_VectorRegister($dst_src1$$reg),
862 as_VectorRegister($src2$$reg), Assembler::v0_t);
863 %}
864 ins_pipe(pipe_slow);
865 %}
866
867 // vector-immediate and (unpredicated)
868
869 instruct vand_vi(vReg dst, vReg src1, immI5 con) %{
870 predicate(Matcher::vector_element_basic_type(n) == T_BYTE ||
871 Matcher::vector_element_basic_type(n) == T_SHORT ||
872 Matcher::vector_element_basic_type(n) == T_INT);
873 match(Set dst (AndV src1 (Replicate con)));
874 format %{ "vand_vi $dst, $src1, $con" %}
875 ins_encode %{
876 BasicType bt = Matcher::vector_element_basic_type(this);
877 __ vsetvli_helper(bt, Matcher::vector_length(this));
878 __ vand_vi(as_VectorRegister($dst$$reg),
879 as_VectorRegister($src1$$reg),
880 $con$$constant);
881 %}
882 ins_pipe(pipe_slow);
883 %}
884
885 instruct vandL_vi(vReg dst, vReg src1, immL5 con) %{
886 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
887 match(Set dst (AndV src1 (Replicate con)));
888 format %{ "vandL_vi $dst, $src1, $con" %}
889 ins_encode %{
890 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
891 __ vand_vi(as_VectorRegister($dst$$reg),
892 as_VectorRegister($src1$$reg),
893 $con$$constant);
894 %}
895 ins_pipe(pipe_slow);
896 %}
897
898 // vector-scalar and (unpredicated)
899
900 instruct vand_vx(vReg dst, vReg src1, iRegIorL2I src2) %{
901 predicate(Matcher::vector_element_basic_type(n) == T_BYTE ||
902 Matcher::vector_element_basic_type(n) == T_SHORT ||
903 Matcher::vector_element_basic_type(n) == T_INT);
904 match(Set dst (AndV src1 (Replicate src2)));
905 format %{ "vand_vx $dst, $src1, $src2" %}
906 ins_encode %{
907 BasicType bt = Matcher::vector_element_basic_type(this);
908 __ vsetvli_helper(bt, Matcher::vector_length(this));
909 __ vand_vx(as_VectorRegister($dst$$reg),
910 as_VectorRegister($src1$$reg),
911 as_Register($src2$$reg));
912 %}
913 ins_pipe(pipe_slow);
914 %}
915
916 instruct vandL_vx(vReg dst, vReg src1, iRegL src2) %{
917 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
918 match(Set dst (AndV src1 (Replicate src2)));
919 format %{ "vandL_vx $dst, $src1, $src2" %}
920 ins_encode %{
921 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
922 __ vand_vx(as_VectorRegister($dst$$reg),
923 as_VectorRegister($src1$$reg),
924 as_Register($src2$$reg));
925 %}
926 ins_pipe(pipe_slow);
927 %}
928
929 // vector-immediate and (predicated)
930
931 instruct vand_vi_masked(vReg dst_src, immI5 con, vRegMask_V0 v0) %{
932 predicate(Matcher::vector_element_basic_type(n) == T_BYTE ||
933 Matcher::vector_element_basic_type(n) == T_SHORT ||
934 Matcher::vector_element_basic_type(n) == T_INT);
935 match(Set dst_src (AndV (Binary dst_src (Replicate con)) v0));
936 format %{ "vand_vi_masked $dst_src, $dst_src, $con, $v0" %}
937 ins_encode %{
938 BasicType bt = Matcher::vector_element_basic_type(this);
939 __ vsetvli_helper(bt, Matcher::vector_length(this));
940 __ vand_vi(as_VectorRegister($dst_src$$reg),
941 as_VectorRegister($dst_src$$reg),
942 $con$$constant, Assembler::v0_t);
943 %}
944 ins_pipe(pipe_slow);
945 %}
946
947 instruct vandL_vi_masked(vReg dst_src, immL5 con, vRegMask_V0 v0) %{
948 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
949 match(Set dst_src (AndV (Binary dst_src (Replicate con)) v0));
950 format %{ "vandL_vi_masked $dst_src, $dst_src, $con, $v0" %}
951 ins_encode %{
952 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
953 __ vand_vi(as_VectorRegister($dst_src$$reg),
954 as_VectorRegister($dst_src$$reg),
955 $con$$constant, Assembler::v0_t);
956 %}
957 ins_pipe(pipe_slow);
958 %}
959
960 // vector-scalar and (predicated)
961
962 instruct vand_vx_masked(vReg dst_src, iRegIorL2I src, vRegMask_V0 v0) %{
963 predicate(Matcher::vector_element_basic_type(n) == T_BYTE ||
964 Matcher::vector_element_basic_type(n) == T_SHORT ||
965 Matcher::vector_element_basic_type(n) == T_INT);
966 match(Set dst_src (AndV (Binary dst_src (Replicate src)) v0));
967 format %{ "vand_vx_masked $dst_src, $dst_src, $src, $v0" %}
968 ins_encode %{
969 BasicType bt = Matcher::vector_element_basic_type(this);
970 __ vsetvli_helper(bt, Matcher::vector_length(this));
971 __ vand_vx(as_VectorRegister($dst_src$$reg),
972 as_VectorRegister($dst_src$$reg),
973 as_Register($src$$reg), Assembler::v0_t);
974 %}
975 ins_pipe(pipe_slow);
976 %}
977
978 instruct vandL_vx_masked(vReg dst_src, iRegL src, vRegMask_V0 v0) %{
979 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
980 match(Set dst_src (AndV (Binary dst_src (Replicate src)) v0));
981 format %{ "vandL_vx_masked $dst_src, $dst_src, $src, $v0" %}
982 ins_encode %{
983 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
984 __ vand_vx(as_VectorRegister($dst_src$$reg),
985 as_VectorRegister($dst_src$$reg),
986 as_Register($src$$reg), Assembler::v0_t);
987 %}
988 ins_pipe(pipe_slow);
989 %}
990
991 // vector or
992
993 instruct vor(vReg dst, vReg src1, vReg src2) %{
994 match(Set dst (OrV src1 src2));
995 format %{ "vor $dst, $src1, $src2" %}
996 ins_encode %{
997 BasicType bt = Matcher::vector_element_basic_type(this);
998 __ vsetvli_helper(bt, Matcher::vector_length(this));
999 __ vor_vv(as_VectorRegister($dst$$reg),
1000 as_VectorRegister($src1$$reg),
1001 as_VectorRegister($src2$$reg));
1002 %}
1003 ins_pipe(pipe_slow);
1004 %}
1005
1006 // vector or - predicated
1007
1008 instruct vor_masked(vReg dst_src1, vReg src2, vRegMask_V0 v0) %{
1009 match(Set dst_src1 (OrV (Binary dst_src1 src2) v0));
1010 format %{ "vor_masked $dst_src1, $dst_src1, $src2, $v0" %}
1011 ins_encode %{
1012 BasicType bt = Matcher::vector_element_basic_type(this);
1013 __ vsetvli_helper(bt, Matcher::vector_length(this));
1014 __ vor_vv(as_VectorRegister($dst_src1$$reg),
1015 as_VectorRegister($dst_src1$$reg),
1016 as_VectorRegister($src2$$reg), Assembler::v0_t);
1017 %}
1018 ins_pipe(pipe_slow);
1019 %}
1020
1021 // vector-immediate or (unpredicated)
1022
1023 instruct vor_vi(vReg dst, vReg src1, immI5 con) %{
1024 predicate(Matcher::vector_element_basic_type(n) == T_BYTE ||
1025 Matcher::vector_element_basic_type(n) == T_SHORT ||
1026 Matcher::vector_element_basic_type(n) == T_INT);
1027 match(Set dst (OrV src1 (Replicate con)));
1028 format %{ "vor_vi $dst, $src1, $con" %}
1029 ins_encode %{
1030 BasicType bt = Matcher::vector_element_basic_type(this);
1031 __ vsetvli_helper(bt, Matcher::vector_length(this));
1032 __ vor_vi(as_VectorRegister($dst$$reg),
1033 as_VectorRegister($src1$$reg),
1034 $con$$constant);
1035 %}
1036 ins_pipe(pipe_slow);
1037 %}
1038
1039 instruct vorL_vi(vReg dst, vReg src1, immL5 con) %{
1040 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
1041 match(Set dst (OrV src1 (Replicate con)));
1042 format %{ "vorL_vi $dst, $src1, $con" %}
1043 ins_encode %{
1044 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
1045 __ vor_vi(as_VectorRegister($dst$$reg),
1046 as_VectorRegister($src1$$reg),
1047 $con$$constant);
1048 %}
1049 ins_pipe(pipe_slow);
1050 %}
1051
1052 // vector-scalar or (unpredicated)
1053
1054 instruct vor_vx(vReg dst, vReg src1, iRegIorL2I src2) %{
1055 predicate(Matcher::vector_element_basic_type(n) == T_BYTE ||
1056 Matcher::vector_element_basic_type(n) == T_SHORT ||
1057 Matcher::vector_element_basic_type(n) == T_INT);
1058 match(Set dst (OrV src1 (Replicate src2)));
1059 format %{ "vor_vx $dst, $src1, $src2" %}
1060 ins_encode %{
1061 BasicType bt = Matcher::vector_element_basic_type(this);
1062 __ vsetvli_helper(bt, Matcher::vector_length(this));
1063 __ vor_vx(as_VectorRegister($dst$$reg),
1064 as_VectorRegister($src1$$reg),
1065 as_Register($src2$$reg));
1066 %}
1067 ins_pipe(pipe_slow);
1068 %}
1069
1070 instruct vorL_vx(vReg dst, vReg src1, iRegL src2) %{
1071 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
1072 match(Set dst (OrV src1 (Replicate src2)));
1073 format %{ "vorL_vx $dst, $src1, $src2" %}
1074 ins_encode %{
1075 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
1076 __ vor_vx(as_VectorRegister($dst$$reg),
1077 as_VectorRegister($src1$$reg),
1078 as_Register($src2$$reg));
1079 %}
1080 ins_pipe(pipe_slow);
1081 %}
1082
1083 // vector-immediate or (predicated)
1084
1085 instruct vor_vi_masked(vReg dst_src, immI5 con, vRegMask_V0 v0) %{
1086 predicate(Matcher::vector_element_basic_type(n) == T_BYTE ||
1087 Matcher::vector_element_basic_type(n) == T_SHORT ||
1088 Matcher::vector_element_basic_type(n) == T_INT);
1089 match(Set dst_src (OrV (Binary dst_src (Replicate con)) v0));
1090 format %{ "vor_vi_masked $dst_src, $dst_src, $con, $v0" %}
1091 ins_encode %{
1092 BasicType bt = Matcher::vector_element_basic_type(this);
1093 __ vsetvli_helper(bt, Matcher::vector_length(this));
1094 __ vor_vi(as_VectorRegister($dst_src$$reg),
1095 as_VectorRegister($dst_src$$reg),
1096 $con$$constant, Assembler::v0_t);
1097 %}
1098 ins_pipe(pipe_slow);
1099 %}
1100
1101 instruct vorL_vi_masked(vReg dst_src, immL5 con, vRegMask_V0 v0) %{
1102 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
1103 match(Set dst_src (OrV (Binary dst_src (Replicate con)) v0));
1104 format %{ "vorL_vi_masked $dst_src, $dst_src, $con, $v0" %}
1105 ins_encode %{
1106 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
1107 __ vor_vi(as_VectorRegister($dst_src$$reg),
1108 as_VectorRegister($dst_src$$reg),
1109 $con$$constant, Assembler::v0_t);
1110 %}
1111 ins_pipe(pipe_slow);
1112 %}
1113
1114 // vector-scalar or (predicated)
1115
1116 instruct vor_vx_masked(vReg dst_src, iRegIorL2I src, vRegMask_V0 v0) %{
1117 predicate(Matcher::vector_element_basic_type(n) == T_BYTE ||
1118 Matcher::vector_element_basic_type(n) == T_SHORT ||
1119 Matcher::vector_element_basic_type(n) == T_INT);
1120 match(Set dst_src (OrV (Binary dst_src (Replicate src)) v0));
1121 format %{ "vor_vx_masked $dst_src, $dst_src, $src, $v0" %}
1122 ins_encode %{
1123 BasicType bt = Matcher::vector_element_basic_type(this);
1124 __ vsetvli_helper(bt, Matcher::vector_length(this));
1125 __ vor_vx(as_VectorRegister($dst_src$$reg),
1126 as_VectorRegister($dst_src$$reg),
1127 as_Register($src$$reg), Assembler::v0_t);
1128 %}
1129 ins_pipe(pipe_slow);
1130 %}
1131
1132 instruct vorL_vx_masked(vReg dst_src, iRegL src, vRegMask_V0 v0) %{
1133 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
1134 match(Set dst_src (OrV (Binary dst_src (Replicate src)) v0));
1135 format %{ "vorL_vx_masked $dst_src, $dst_src, $src, $v0" %}
1136 ins_encode %{
1137 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
1138 __ vor_vx(as_VectorRegister($dst_src$$reg),
1139 as_VectorRegister($dst_src$$reg),
1140 as_Register($src$$reg), Assembler::v0_t);
1141 %}
1142 ins_pipe(pipe_slow);
1143 %}
1144
1145 // vector xor
1146
1147 instruct vxor(vReg dst, vReg src1, vReg src2) %{
1148 match(Set dst (XorV src1 src2));
1149 format %{ "vxor $dst, $src1, $src2" %}
1150 ins_encode %{
1151 BasicType bt = Matcher::vector_element_basic_type(this);
1152 __ vsetvli_helper(bt, Matcher::vector_length(this));
1153 __ vxor_vv(as_VectorRegister($dst$$reg),
1154 as_VectorRegister($src1$$reg),
1155 as_VectorRegister($src2$$reg));
1156 %}
1157 ins_pipe(pipe_slow);
1158 %}
1159
1160 // vector xor - predicated
1161
1162 instruct vxor_masked(vReg dst_src1, vReg src2, vRegMask_V0 v0) %{
1163 match(Set dst_src1 (XorV (Binary dst_src1 src2) v0));
1164 format %{ "vxor_masked $dst_src1, $dst_src1, $src2, $v0" %}
1165 ins_encode %{
1166 BasicType bt = Matcher::vector_element_basic_type(this);
1167 __ vsetvli_helper(bt, Matcher::vector_length(this));
1168 __ vxor_vv(as_VectorRegister($dst_src1$$reg),
1169 as_VectorRegister($dst_src1$$reg),
1170 as_VectorRegister($src2$$reg), Assembler::v0_t);
1171 %}
1172 ins_pipe(pipe_slow);
1173 %}
1174
1175 // vector-immediate xor (unpredicated)
1176
1177 instruct vxor_vi(vReg dst, vReg src1, immI5 con) %{
1178 predicate(Matcher::vector_element_basic_type(n) == T_BYTE ||
1179 Matcher::vector_element_basic_type(n) == T_SHORT ||
1180 Matcher::vector_element_basic_type(n) == T_INT);
1181 match(Set dst (XorV src1 (Replicate con)));
1182 format %{ "vxor_vi $dst, $src1, $con" %}
1183 ins_encode %{
1184 BasicType bt = Matcher::vector_element_basic_type(this);
1185 __ vsetvli_helper(bt, Matcher::vector_length(this));
1186 __ vxor_vi(as_VectorRegister($dst$$reg),
1187 as_VectorRegister($src1$$reg),
1188 $con$$constant);
1189 %}
1190 ins_pipe(pipe_slow);
1191 %}
1192
1193 instruct vxorL_vi(vReg dst, vReg src1, immL5 con) %{
1194 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
1195 match(Set dst (XorV src1 (Replicate con)));
1196 format %{ "vxorL_vi $dst, $src1, $con" %}
1197 ins_encode %{
1198 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
1199 __ vxor_vi(as_VectorRegister($dst$$reg),
1200 as_VectorRegister($src1$$reg),
1201 $con$$constant);
1202 %}
1203 ins_pipe(pipe_slow);
1204 %}
1205
1206 // vector-scalar xor (unpredicated)
1207
1208 instruct vxor_vx(vReg dst, vReg src1, iRegIorL2I src2) %{
1209 predicate(Matcher::vector_element_basic_type(n) == T_BYTE ||
1210 Matcher::vector_element_basic_type(n) == T_SHORT ||
1211 Matcher::vector_element_basic_type(n) == T_INT);
1212 match(Set dst (XorV src1 (Replicate src2)));
1213 format %{ "vxor_vx $dst, $src1, $src2" %}
1214 ins_encode %{
1215 BasicType bt = Matcher::vector_element_basic_type(this);
1216 __ vsetvli_helper(bt, Matcher::vector_length(this));
1217 __ vxor_vx(as_VectorRegister($dst$$reg),
1218 as_VectorRegister($src1$$reg),
1219 as_Register($src2$$reg));
1220 %}
1221 ins_pipe(pipe_slow);
1222 %}
1223
1224 instruct vxorL_vx(vReg dst, vReg src1, iRegL src2) %{
1225 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
1226 match(Set dst (XorV src1 (Replicate src2)));
1227 format %{ "vxorL_vx $dst, $src1, $src2" %}
1228 ins_encode %{
1229 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
1230 __ vxor_vx(as_VectorRegister($dst$$reg),
1231 as_VectorRegister($src1$$reg),
1232 as_Register($src2$$reg));
1233 %}
1234 ins_pipe(pipe_slow);
1235 %}
1236
1237 // vector-immediate xor (predicated)
1238
1239 instruct vxor_vi_masked(vReg dst_src, immI5 con, vRegMask_V0 v0) %{
1240 predicate(Matcher::vector_element_basic_type(n) == T_BYTE ||
1241 Matcher::vector_element_basic_type(n) == T_SHORT ||
1242 Matcher::vector_element_basic_type(n) == T_INT);
1243 match(Set dst_src (XorV (Binary dst_src (Replicate con)) v0));
1244 format %{ "vxor_vi_masked $dst_src, $dst_src, $con, $v0" %}
1245 ins_encode %{
1246 BasicType bt = Matcher::vector_element_basic_type(this);
1247 __ vsetvli_helper(bt, Matcher::vector_length(this));
1248 __ vxor_vi(as_VectorRegister($dst_src$$reg),
1249 as_VectorRegister($dst_src$$reg),
1250 $con$$constant, Assembler::v0_t);
1251 %}
1252 ins_pipe(pipe_slow);
1253 %}
1254
1255 instruct vxorL_vi_masked(vReg dst_src, immL5 con, vRegMask_V0 v0) %{
1256 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
1257 match(Set dst_src (XorV (Binary dst_src (Replicate con)) v0));
1258 format %{ "vxorL_vi_masked $dst_src, $dst_src, $con, $v0" %}
1259 ins_encode %{
1260 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
1261 __ vxor_vi(as_VectorRegister($dst_src$$reg),
1262 as_VectorRegister($dst_src$$reg),
1263 $con$$constant, Assembler::v0_t);
1264 %}
1265 ins_pipe(pipe_slow);
1266 %}
1267
1268 // vector-scalar xor (predicated)
1269
1270 instruct vxor_vx_masked(vReg dst_src, iRegIorL2I src, vRegMask_V0 v0) %{
1271 predicate(Matcher::vector_element_basic_type(n) == T_BYTE ||
1272 Matcher::vector_element_basic_type(n) == T_SHORT ||
1273 Matcher::vector_element_basic_type(n) == T_INT);
1274 match(Set dst_src (XorV (Binary dst_src (Replicate src)) v0));
1275 format %{ "vxor_vx_masked $dst_src, $dst_src, $src, $v0" %}
1276 ins_encode %{
1277 BasicType bt = Matcher::vector_element_basic_type(this);
1278 __ vsetvli_helper(bt, Matcher::vector_length(this));
1279 __ vxor_vx(as_VectorRegister($dst_src$$reg),
1280 as_VectorRegister($dst_src$$reg),
1281 as_Register($src$$reg), Assembler::v0_t);
1282 %}
1283 ins_pipe(pipe_slow);
1284 %}
1285
1286 instruct vxorL_vx_masked(vReg dst_src, iRegL src, vRegMask_V0 v0) %{
1287 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
1288 match(Set dst_src (XorV (Binary dst_src (Replicate src)) v0));
1289 format %{ "vxorL_vx_masked $dst_src, $dst_src, $src, $v0" %}
1290 ins_encode %{
1291 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
1292 __ vxor_vx(as_VectorRegister($dst_src$$reg),
1293 as_VectorRegister($dst_src$$reg),
1294 as_Register($src$$reg), Assembler::v0_t);
1295 %}
1296 ins_pipe(pipe_slow);
1297 %}
1298
1299 // ------------------------------ Vector and not -----------------------------------
1300
1301 // vector and not
1302
1303 instruct vand_notB(vReg dst, vReg src1, vReg src2, immI_M1 m1) %{
1304 predicate(UseZvbb && Matcher::vector_element_basic_type(n) == T_BYTE);
1305 match(Set dst (AndV src1 (XorV src2 (Replicate m1))));
1306 format %{ "vand_notB $dst, $src1, $src2" %}
1307 ins_encode %{
1308 __ vsetvli_helper(T_BYTE, Matcher::vector_length(this));
1309 __ vandn_vv(as_VectorRegister($dst$$reg),
1310 as_VectorRegister($src1$$reg),
1311 as_VectorRegister($src2$$reg));
1312 %}
1313 ins_pipe(pipe_slow);
1314 %}
1315
1316 instruct vand_notS(vReg dst, vReg src1, vReg src2, immI_M1 m1) %{
1317 predicate(UseZvbb && Matcher::vector_element_basic_type(n) == T_SHORT);
1318 match(Set dst (AndV src1 (XorV src2 (Replicate m1))));
1319 format %{ "vand_notS $dst, $src1, $src2" %}
1320 ins_encode %{
1321 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
1322 __ vandn_vv(as_VectorRegister($dst$$reg),
1323 as_VectorRegister($src1$$reg),
1324 as_VectorRegister($src2$$reg));
1325 %}
1326 ins_pipe(pipe_slow);
1327 %}
1328
1329 instruct vand_notI(vReg dst, vReg src1, vReg src2, immI_M1 m1) %{
1330 predicate(UseZvbb && Matcher::vector_element_basic_type(n) == T_INT);
1331 match(Set dst (AndV src1 (XorV src2 (Replicate m1))));
1332 format %{ "vand_notI $dst, $src1, $src2" %}
1333 ins_encode %{
1334 __ vsetvli_helper(T_INT, Matcher::vector_length(this));
1335 __ vandn_vv(as_VectorRegister($dst$$reg),
1336 as_VectorRegister($src1$$reg),
1337 as_VectorRegister($src2$$reg));
1338 %}
1339 ins_pipe(pipe_slow);
1340 %}
1341
1342 instruct vand_notL(vReg dst, vReg src1, vReg src2, immL_M1 m1) %{
1343 predicate(UseZvbb && Matcher::vector_element_basic_type(n) == T_LONG);
1344 match(Set dst (AndV src1 (XorV src2 (Replicate m1))));
1345 format %{ "vand_notL $dst, $src1, $src2" %}
1346 ins_encode %{
1347 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
1348 __ vandn_vv(as_VectorRegister($dst$$reg),
1349 as_VectorRegister($src1$$reg),
1350 as_VectorRegister($src2$$reg));
1351 %}
1352 ins_pipe(pipe_slow);
1353 %}
1354
1355 instruct vand_notB_masked(vReg dst_src1, vReg src2, immI_M1 m1, vRegMask_V0 v0) %{
1356 predicate(UseZvbb && Matcher::vector_element_basic_type(n) == T_BYTE);
1357 match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (Replicate m1))) v0));
1358 format %{ "vand_notB_masked $dst_src1, $dst_src1, $src2, $v0" %}
1359 ins_encode %{
1360 __ vsetvli_helper(T_BYTE, Matcher::vector_length(this));
1361 __ vandn_vv(as_VectorRegister($dst_src1$$reg),
1362 as_VectorRegister($dst_src1$$reg),
1363 as_VectorRegister($src2$$reg),
1364 Assembler::v0_t);
1365 %}
1366 ins_pipe(pipe_slow);
1367 %}
1368
1369 instruct vand_notS_masked(vReg dst_src1, vReg src2, immI_M1 m1, vRegMask_V0 v0) %{
1370 predicate(UseZvbb && Matcher::vector_element_basic_type(n) == T_SHORT);
1371 match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (Replicate m1))) v0));
1372 format %{ "vand_notS_masked $dst_src1, $dst_src1, $src2, $v0" %}
1373 ins_encode %{
1374 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
1375 __ vandn_vv(as_VectorRegister($dst_src1$$reg),
1376 as_VectorRegister($dst_src1$$reg),
1377 as_VectorRegister($src2$$reg),
1378 Assembler::v0_t);
1379 %}
1380 ins_pipe(pipe_slow);
1381 %}
1382
1383 instruct vand_notI_masked(vReg dst_src1, vReg src2, immI_M1 m1, vRegMask_V0 v0) %{
1384 predicate(UseZvbb && Matcher::vector_element_basic_type(n) == T_INT);
1385 match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (Replicate m1))) v0));
1386 format %{ "vand_notI_masked $dst_src1, $dst_src1, $src2, $v0" %}
1387 ins_encode %{
1388 __ vsetvli_helper(T_INT, Matcher::vector_length(this));
1389 __ vandn_vv(as_VectorRegister($dst_src1$$reg),
1390 as_VectorRegister($dst_src1$$reg),
1391 as_VectorRegister($src2$$reg),
1392 Assembler::v0_t);
1393 %}
1394 ins_pipe(pipe_slow);
1395 %}
1396
1397 instruct vand_notL_masked(vReg dst_src1, vReg src2, immL_M1 m1, vRegMask_V0 v0) %{
1398 predicate(UseZvbb && Matcher::vector_element_basic_type(n) == T_LONG);
1399 match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (Replicate m1))) v0));
1400 format %{ "vand_notL_masked $dst_src1, $dst_src1, $src2, $v0" %}
1401 ins_encode %{
1402 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
1403 __ vandn_vv(as_VectorRegister($dst_src1$$reg),
1404 as_VectorRegister($dst_src1$$reg),
1405 as_VectorRegister($src2$$reg),
1406 Assembler::v0_t);
1407 %}
1408 ins_pipe(pipe_slow);
1409 %}
1410
1411 instruct vand_notB_vx(vReg dst, vReg src1, iRegIorL2I src2, immI_M1 m1) %{
1412 predicate(UseZvbb && Matcher::vector_element_basic_type(n) == T_BYTE);
1413 match(Set dst (AndV src1 (Replicate (XorI src2 m1))));
1414 format %{ "vand_notB_vx $dst, $src1, $src2" %}
1415 ins_encode %{
1416 __ vsetvli_helper(T_BYTE, Matcher::vector_length(this));
1417 __ vandn_vx(as_VectorRegister($dst$$reg),
1418 as_VectorRegister($src1$$reg),
1419 as_Register($src2$$reg));
1420 %}
1421 ins_pipe(pipe_slow);
1422 %}
1423
1424 instruct vand_notS_vx(vReg dst, vReg src1, iRegIorL2I src2, immI_M1 m1) %{
1425 predicate(UseZvbb && Matcher::vector_element_basic_type(n) == T_SHORT);
1426 match(Set dst (AndV src1 (Replicate (XorI src2 m1))));
1427 format %{ "vand_notS_vx $dst, $src1, $src2" %}
1428 ins_encode %{
1429 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
1430 __ vandn_vx(as_VectorRegister($dst$$reg),
1431 as_VectorRegister($src1$$reg),
1432 as_Register($src2$$reg));
1433 %}
1434 ins_pipe(pipe_slow);
1435 %}
1436
1437 instruct vand_notI_vx(vReg dst, vReg src1, iRegIorL2I src2, immI_M1 m1) %{
1438 predicate(UseZvbb && Matcher::vector_element_basic_type(n) == T_INT);
1439 match(Set dst (AndV src1 (Replicate (XorI src2 m1))));
1440 format %{ "vand_notI_vx $dst, $src1, $src2" %}
1441 ins_encode %{
1442 __ vsetvli_helper(T_INT, Matcher::vector_length(this));
1443 __ vandn_vx(as_VectorRegister($dst$$reg),
1444 as_VectorRegister($src1$$reg),
1445 as_Register($src2$$reg));
1446 %}
1447 ins_pipe(pipe_slow);
1448 %}
1449
1450 instruct vand_notL_vx(vReg dst, vReg src1, iRegL src2, immL_M1 m1) %{
1451 predicate(UseZvbb && Matcher::vector_element_basic_type(n) == T_LONG);
1452 match(Set dst (AndV src1 (Replicate (XorL src2 m1))));
1453 format %{ "vand_notL_vx $dst, $src1, $src2" %}
1454 ins_encode %{
1455 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
1456 __ vandn_vx(as_VectorRegister($dst$$reg),
1457 as_VectorRegister($src1$$reg),
1458 as_Register($src2$$reg));
1459 %}
1460 ins_pipe(pipe_slow);
1461 %}
1462
1463 instruct vand_notB_vx_masked(vReg dst_src1, iRegIorL2I src2, immI_M1 m1, vRegMask_V0 v0) %{
1464 predicate(UseZvbb && Matcher::vector_element_basic_type(n) == T_BYTE);
1465 match(Set dst_src1 (AndV (Binary dst_src1 (Replicate (XorI src2 m1))) v0));
1466 format %{ "vand_notB_vx_masked $dst_src1, $dst_src1, $src2, $v0" %}
1467 ins_encode %{
1468 __ vsetvli_helper(T_BYTE, Matcher::vector_length(this));
1469 __ vandn_vx(as_VectorRegister($dst_src1$$reg),
1470 as_VectorRegister($dst_src1$$reg),
1471 as_Register($src2$$reg),
1472 Assembler::v0_t);
1473 %}
1474 ins_pipe(pipe_slow);
1475 %}
1476
1477 instruct vand_notS_vx_masked(vReg dst_src1, iRegIorL2I src2, immI_M1 m1, vRegMask_V0 v0) %{
1478 predicate(UseZvbb && Matcher::vector_element_basic_type(n) == T_SHORT);
1479 match(Set dst_src1 (AndV (Binary dst_src1 (Replicate (XorI src2 m1))) v0));
1480 format %{ "vand_notS_vx_masked $dst_src1, $dst_src1, $src2, $v0" %}
1481 ins_encode %{
1482 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
1483 __ vandn_vx(as_VectorRegister($dst_src1$$reg),
1484 as_VectorRegister($dst_src1$$reg),
1485 as_Register($src2$$reg),
1486 Assembler::v0_t);
1487 %}
1488 ins_pipe(pipe_slow);
1489 %}
1490
1491 instruct vand_notI_vx_masked(vReg dst_src1, iRegIorL2I src2, immI_M1 m1, vRegMask_V0 v0) %{
1492 predicate(UseZvbb && Matcher::vector_element_basic_type(n) == T_INT);
1493 match(Set dst_src1 (AndV (Binary dst_src1 (Replicate (XorI src2 m1))) v0));
1494 format %{ "vand_notI_vx_masked $dst_src1, $dst_src1, $src2, $v0" %}
1495 ins_encode %{
1496 __ vsetvli_helper(T_INT, Matcher::vector_length(this));
1497 __ vandn_vx(as_VectorRegister($dst_src1$$reg),
1498 as_VectorRegister($dst_src1$$reg),
1499 as_Register($src2$$reg),
1500 Assembler::v0_t);
1501 %}
1502 ins_pipe(pipe_slow);
1503 %}
1504
1505 instruct vand_notL_vx_masked(vReg dst_src1, iRegL src2, immL_M1 m1, vRegMask_V0 v0) %{
1506 predicate(UseZvbb && Matcher::vector_element_basic_type(n) == T_LONG);
1507 match(Set dst_src1 (AndV (Binary dst_src1 (Replicate (XorL src2 m1))) v0));
1508 format %{ "vand_notL_vx_masked $dst_src1, $dst_src1, $src2, $v0" %}
1509 ins_encode %{
1510 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
1511 __ vandn_vx(as_VectorRegister($dst_src1$$reg),
1512 as_VectorRegister($dst_src1$$reg),
1513 as_Register($src2$$reg),
1514 Assembler::v0_t);
1515 %}
1516 ins_pipe(pipe_slow);
1517 %}
1518
1519 // ------------------------------ Vector not -----------------------------------
1520
1521 // vector not
1522
1523 instruct vnot(vReg dst, vReg src, immI_M1 m1) %{
1524 predicate(Matcher::vector_element_basic_type(n) == T_BYTE ||
1525 Matcher::vector_element_basic_type(n) == T_SHORT ||
1526 Matcher::vector_element_basic_type(n) == T_INT);
1527 match(Set dst (XorV src (Replicate m1)));
1528 format %{ "vnot $dst, $src" %}
1529 ins_encode %{
1530 BasicType bt = Matcher::vector_element_basic_type(this);
1531 __ vsetvli_helper(bt, Matcher::vector_length(this));
1532 __ vxor_vi(as_VectorRegister($dst$$reg),
1533 as_VectorRegister($src$$reg),
1534 -1);
1535 %}
1536 ins_pipe(pipe_slow);
1537 %}
1538
1539 instruct vnotL(vReg dst, vReg src, immL_M1 m1) %{
1540 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
1541 match(Set dst (XorV src (Replicate m1)));
1542 format %{ "vnotL $dst, $src" %}
1543 ins_encode %{
1544 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
1545 __ vxor_vi(as_VectorRegister($dst$$reg),
1546 as_VectorRegister($src$$reg),
1547 -1);
1548 %}
1549 ins_pipe(pipe_slow);
1550 %}
1551
1552 // vector not - predicated
1553
1554 instruct vnot_masked(vReg dst_src, immI_M1 m1, vRegMask_V0 v0) %{
1555 predicate(Matcher::vector_element_basic_type(n) == T_BYTE ||
1556 Matcher::vector_element_basic_type(n) == T_SHORT ||
1557 Matcher::vector_element_basic_type(n) == T_INT);
1558 match(Set dst_src (XorV (Binary dst_src (Replicate m1)) v0));
1559 format %{ "vnot_masked $dst_src, $dst_src, $v0" %}
1560 ins_encode %{
1561 BasicType bt = Matcher::vector_element_basic_type(this);
1562 __ vsetvli_helper(bt, Matcher::vector_length(this));
1563 __ vxor_vi(as_VectorRegister($dst_src$$reg),
1564 as_VectorRegister($dst_src$$reg),
1565 -1, Assembler::v0_t);
1566 %}
1567 ins_pipe(pipe_slow);
1568 %}
1569
1570 instruct vnotL_masked(vReg dst_src, immI_M1 m1, vRegMask_V0 v0) %{
1571 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
1572 match(Set dst_src (XorV (Binary dst_src (Replicate m1)) v0));
1573 format %{ "vnotL_masked $dst_src, $dst_src, $v0" %}
1574 ins_encode %{
1575 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
1576 __ vxor_vi(as_VectorRegister($dst_src$$reg),
1577 as_VectorRegister($dst_src$$reg),
1578 -1, Assembler::v0_t);
1579 %}
1580 ins_pipe(pipe_slow);
1581 %}
1582
1583 // vector float div
1584
1585 instruct vdiv_hfp(vReg dst, vReg src1, vReg src2) %{
1586 match(Set dst (DivVHF src1 src2));
1587 format %{ "vdiv_hfp $dst, $src1, $src2" %}
1588 ins_encode %{
1589 assert(UseZvfh, "must");
1590 assert(Matcher::vector_element_basic_type(this) == T_SHORT, "must");
1591 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
1592 __ vfdiv_vv(as_VectorRegister($dst$$reg),
1593 as_VectorRegister($src1$$reg),
1594 as_VectorRegister($src2$$reg));
1595 %}
1596 ins_pipe(pipe_slow);
1597 %}
1598
1599 instruct vdiv_fp(vReg dst, vReg src1, vReg src2) %{
1600 match(Set dst (DivVF src1 src2));
1601 match(Set dst (DivVD src1 src2));
1602 format %{ "vdiv_fp $dst, $src1, $src2" %}
1603 ins_encode %{
1604 BasicType bt = Matcher::vector_element_basic_type(this);
1605 __ vsetvli_helper(bt, Matcher::vector_length(this));
1606 __ vfdiv_vv(as_VectorRegister($dst$$reg),
1607 as_VectorRegister($src1$$reg),
1608 as_VectorRegister($src2$$reg));
1609 %}
1610 ins_pipe(pipe_slow);
1611 %}
1612
1613 // vector float div - predicated
1614
1615 instruct vdiv_fp_masked(vReg dst_src1, vReg src2, vRegMask_V0 v0) %{
1616 match(Set dst_src1 (DivVF (Binary dst_src1 src2) v0));
1617 match(Set dst_src1 (DivVD (Binary dst_src1 src2) v0));
1618 format %{ "vdiv_fp_masked $dst_src1, $dst_src1, $src2, $v0" %}
1619 ins_encode %{
1620 BasicType bt = Matcher::vector_element_basic_type(this);
1621 __ vsetvli_helper(bt, Matcher::vector_length(this));
1622 __ vfdiv_vv(as_VectorRegister($dst_src1$$reg),
1623 as_VectorRegister($dst_src1$$reg),
1624 as_VectorRegister($src2$$reg), Assembler::v0_t);
1625 %}
1626 ins_pipe(pipe_slow);
1627 %}
1628
1629 // vector integer max/min
1630
1631 instruct vmax(vReg dst, vReg src1, vReg src2) %{
1632 predicate(Matcher::vector_element_basic_type(n) != T_FLOAT &&
1633 Matcher::vector_element_basic_type(n) != T_DOUBLE);
1634 match(Set dst (MaxV src1 src2));
1635 format %{ "vmax $dst, $src1, $src2" %}
1636 ins_encode %{
1637 BasicType bt = Matcher::vector_element_basic_type(this);
1638 __ vsetvli_helper(bt, Matcher::vector_length(this));
1639 __ vmax_vv(as_VectorRegister($dst$$reg),
1640 as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg));
1641 %}
1642 ins_pipe(pipe_slow);
1643 %}
1644
1645 instruct vmin(vReg dst, vReg src1, vReg src2) %{
1646 predicate(Matcher::vector_element_basic_type(n) != T_FLOAT &&
1647 Matcher::vector_element_basic_type(n) != T_DOUBLE);
1648 match(Set dst (MinV src1 src2));
1649 format %{ "vmin $dst, $src1, $src2" %}
1650 ins_encode %{
1651 BasicType bt = Matcher::vector_element_basic_type(this);
1652 __ vsetvli_helper(bt, Matcher::vector_length(this));
1653 __ vmin_vv(as_VectorRegister($dst$$reg),
1654 as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg));
1655 %}
1656 ins_pipe(pipe_slow);
1657 %}
1658
1659 // vector integer max/min - predicated
1660
1661 instruct vmax_masked(vReg dst_src1, vReg src2, vRegMask_V0 v0) %{
1662 predicate(Matcher::vector_element_basic_type(n) != T_FLOAT &&
1663 Matcher::vector_element_basic_type(n) != T_DOUBLE);
1664 match(Set dst_src1 (MaxV (Binary dst_src1 src2) v0));
1665 format %{ "vmax_masked $dst_src1, $dst_src1, $src2, $v0" %}
1666 ins_encode %{
1667 BasicType bt = Matcher::vector_element_basic_type(this);
1668 __ vsetvli_helper(bt, Matcher::vector_length(this));
1669 __ vmax_vv(as_VectorRegister($dst_src1$$reg), as_VectorRegister($dst_src1$$reg),
1670 as_VectorRegister($src2$$reg), Assembler::v0_t);
1671 %}
1672 ins_pipe(pipe_slow);
1673 %}
1674
1675 instruct vmin_masked(vReg dst_src1, vReg src2, vRegMask_V0 v0) %{
1676 predicate(Matcher::vector_element_basic_type(n) != T_FLOAT &&
1677 Matcher::vector_element_basic_type(n) != T_DOUBLE);
1678 match(Set dst_src1 (MinV (Binary dst_src1 src2) v0));
1679 format %{ "vmin_masked $dst_src1, $dst_src1, $src2, $v0" %}
1680 ins_encode %{
1681 BasicType bt = Matcher::vector_element_basic_type(this);
1682 __ vsetvli_helper(bt, Matcher::vector_length(this));
1683 __ vmin_vv(as_VectorRegister($dst_src1$$reg), as_VectorRegister($dst_src1$$reg),
1684 as_VectorRegister($src2$$reg), Assembler::v0_t);
1685 %}
1686 ins_pipe(pipe_slow);
1687 %}
1688
1689 // vector unsigned integer max/min
1690
1691 instruct vmaxu(vReg dst, vReg src1, vReg src2) %{
1692 match(Set dst (UMaxV src1 src2));
1693 format %{ "vmaxu $dst, $src1, $src2" %}
1694 ins_encode %{
1695 BasicType bt = Matcher::vector_element_basic_type(this);
1696 assert(is_integral_type(bt), "unsupported type");
1697 __ vsetvli_helper(bt, Matcher::vector_length(this));
1698 __ vmaxu_vv(as_VectorRegister($dst$$reg),
1699 as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg));
1700 %}
1701 ins_pipe(pipe_slow);
1702 %}
1703
1704 instruct vminu(vReg dst, vReg src1, vReg src2) %{
1705 match(Set dst (UMinV src1 src2));
1706 format %{ "vminu $dst, $src1, $src2" %}
1707 ins_encode %{
1708 BasicType bt = Matcher::vector_element_basic_type(this);
1709 assert(is_integral_type(bt), "unsupported type");
1710 __ vsetvli_helper(bt, Matcher::vector_length(this));
1711 __ vminu_vv(as_VectorRegister($dst$$reg),
1712 as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg));
1713 %}
1714 ins_pipe(pipe_slow);
1715 %}
1716
1717 // vector unsigned integer max/min - predicated
1718
1719 instruct vmaxu_masked(vReg dst_src1, vReg src2, vRegMask_V0 v0) %{
1720 match(Set dst_src1 (UMaxV (Binary dst_src1 src2) v0));
1721 format %{ "vmaxu_masked $dst_src1, $dst_src1, $src2, $v0" %}
1722 ins_encode %{
1723 BasicType bt = Matcher::vector_element_basic_type(this);
1724 assert(is_integral_type(bt), "unsupported type");
1725 __ vsetvli_helper(bt, Matcher::vector_length(this));
1726 __ vmaxu_vv(as_VectorRegister($dst_src1$$reg), as_VectorRegister($dst_src1$$reg),
1727 as_VectorRegister($src2$$reg), Assembler::v0_t);
1728 %}
1729 ins_pipe(pipe_slow);
1730 %}
1731
1732 instruct vminu_masked(vReg dst_src1, vReg src2, vRegMask_V0 v0) %{
1733 match(Set dst_src1 (UMinV (Binary dst_src1 src2) v0));
1734 format %{ "vminu_masked $dst_src1, $dst_src1, $src2, $v0" %}
1735 ins_encode %{
1736 BasicType bt = Matcher::vector_element_basic_type(this);
1737 assert(is_integral_type(bt), "unsupported type");
1738 __ vsetvli_helper(bt, Matcher::vector_length(this));
1739 __ vminu_vv(as_VectorRegister($dst_src1$$reg), as_VectorRegister($dst_src1$$reg),
1740 as_VectorRegister($src2$$reg), Assembler::v0_t);
1741 %}
1742 ins_pipe(pipe_slow);
1743 %}
1744
1745 // vector float-point max/min (half precision)
1746
1747 instruct vmax_hfp(vReg dst, vReg src1, vReg src2, vRegMask_V0 v0) %{
1748 match(Set dst (MaxVHF src1 src2));
1749 effect(TEMP_DEF dst, TEMP v0);
1750 format %{ "vmax_hfp $dst, $src1, $src2" %}
1751 ins_encode %{
1752 assert(UseZvfh, "must");
1753 assert(Matcher::vector_element_basic_type(this) == T_SHORT, "must");
1754 __ minmax_fp_v(as_VectorRegister($dst$$reg),
1755 as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg),
1756 T_SHORT, false /* is_min */, Matcher::vector_length(this));
1757 %}
1758 ins_pipe(pipe_slow);
1759 %}
1760
1761 instruct vmin_hfp(vReg dst, vReg src1, vReg src2, vRegMask_V0 v0) %{
1762 match(Set dst (MinVHF src1 src2));
1763 effect(TEMP_DEF dst, TEMP v0);
1764 format %{ "vmin_hfp $dst, $src1, $src2" %}
1765 ins_encode %{
1766 assert(UseZvfh, "must");
1767 assert(Matcher::vector_element_basic_type(this) == T_SHORT, "must");
1768 __ minmax_fp_v(as_VectorRegister($dst$$reg),
1769 as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg),
1770 T_SHORT, true /* is_min */, Matcher::vector_length(this));
1771 %}
1772 ins_pipe(pipe_slow);
1773 %}
1774
1775 // vector float-point max/min
1776
1777 instruct vmax_fp(vReg dst, vReg src1, vReg src2, vRegMask_V0 v0) %{
1778 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT ||
1779 Matcher::vector_element_basic_type(n) == T_DOUBLE);
1780 match(Set dst (MaxV src1 src2));
1781 effect(TEMP_DEF dst, TEMP v0);
1782 format %{ "vmax_fp $dst, $src1, $src2" %}
1783 ins_encode %{
1784 BasicType bt = Matcher::vector_element_basic_type(this);
1785 __ minmax_fp_v(as_VectorRegister($dst$$reg),
1786 as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg),
1787 bt, false /* is_min */, Matcher::vector_length(this));
1788 %}
1789 ins_pipe(pipe_slow);
1790 %}
1791
1792 instruct vmin_fp(vReg dst, vReg src1, vReg src2, vRegMask_V0 v0) %{
1793 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT ||
1794 Matcher::vector_element_basic_type(n) == T_DOUBLE);
1795 match(Set dst (MinV src1 src2));
1796 effect(TEMP_DEF dst, TEMP v0);
1797 format %{ "vmin_fp $dst, $src1, $src2" %}
1798 ins_encode %{
1799 BasicType bt = Matcher::vector_element_basic_type(this);
1800 __ minmax_fp_v(as_VectorRegister($dst$$reg),
1801 as_VectorRegister($src1$$reg), as_VectorRegister($src2$$reg),
1802 bt, true /* is_min */, Matcher::vector_length(this));
1803 %}
1804 ins_pipe(pipe_slow);
1805 %}
1806
1807 // vector float-point max/min - predicated
1808
1809 instruct vmax_fp_masked(vReg dst_src1, vReg src2, vRegMask vmask, vReg tmp1, vReg tmp2, vRegMask_V0 v0) %{
1810 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT ||
1811 Matcher::vector_element_basic_type(n) == T_DOUBLE);
1812 match(Set dst_src1 (MaxV (Binary dst_src1 src2) vmask));
1813 effect(TEMP_DEF dst_src1, TEMP tmp1, TEMP tmp2, TEMP v0);
1814 format %{ "vmax_fp_masked $dst_src1, $dst_src1, $src2, $vmask\t# KILL $tmp1, $tmp2, $v0" %}
1815 ins_encode %{
1816 BasicType bt = Matcher::vector_element_basic_type(this);
1817 __ minmax_fp_masked_v(as_VectorRegister($dst_src1$$reg), as_VectorRegister($dst_src1$$reg),
1818 as_VectorRegister($src2$$reg), as_VectorRegister($vmask$$reg),
1819 as_VectorRegister($tmp1$$reg), as_VectorRegister($tmp2$$reg),
1820 bt, false /* is_min */, Matcher::vector_length(this));
1821 %}
1822 ins_pipe(pipe_slow);
1823 %}
1824
1825 instruct vmin_fp_masked(vReg dst_src1, vReg src2, vRegMask vmask, vReg tmp1, vReg tmp2, vRegMask_V0 v0) %{
1826 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT ||
1827 Matcher::vector_element_basic_type(n) == T_DOUBLE);
1828 match(Set dst_src1 (MinV (Binary dst_src1 src2) vmask));
1829 effect(TEMP_DEF dst_src1, TEMP tmp1, TEMP tmp2, TEMP v0);
1830 format %{ "vmin_fp_masked $dst_src1, $dst_src1, $src2, $vmask\t# KILL $tmp1, $tmp2, $v0" %}
1831 ins_encode %{
1832 BasicType bt = Matcher::vector_element_basic_type(this);
1833 __ minmax_fp_masked_v(as_VectorRegister($dst_src1$$reg), as_VectorRegister($dst_src1$$reg),
1834 as_VectorRegister($src2$$reg), as_VectorRegister($vmask$$reg),
1835 as_VectorRegister($tmp1$$reg), as_VectorRegister($tmp2$$reg),
1836 bt, true /* is_min */, Matcher::vector_length(this));
1837 %}
1838 ins_pipe(pipe_slow);
1839 %}
1840
1841 // vector fmla
1842
1843 // dst_src1 = src2 * src3 + dst_src1 (half precision)
1844 instruct vhfmla(vReg dst_src1, vReg src2, vReg src3) %{
1845 match(Set dst_src1 (FmaVHF dst_src1 (Binary src2 src3)));
1846 format %{ "vhfmla $dst_src1, $dst_src1, $src2, $src3" %}
1847 ins_encode %{
1848 assert(UseFMA, "Needs FMA instructions support.");
1849 assert(UseZvfh, "must");
1850 assert(Matcher::vector_element_basic_type(this) == T_SHORT, "must");
1851 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
1852 __ vfmacc_vv(as_VectorRegister($dst_src1$$reg),
1853 as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg));
1854 %}
1855 ins_pipe(pipe_slow);
1856 %}
1857
1858 // dst_src1 = src2 * src3 + dst_src1
1859 instruct vfmla(vReg dst_src1, vReg src2, vReg src3) %{
1860 match(Set dst_src1 (FmaVF dst_src1 (Binary src2 src3)));
1861 match(Set dst_src1 (FmaVD dst_src1 (Binary src2 src3)));
1862 format %{ "vfmla $dst_src1, $dst_src1, $src2, $src3" %}
1863 ins_encode %{
1864 assert(UseFMA, "Needs FMA instructions support.");
1865 BasicType bt = Matcher::vector_element_basic_type(this);
1866 __ vsetvli_helper(bt, Matcher::vector_length(this));
1867 __ vfmacc_vv(as_VectorRegister($dst_src1$$reg),
1868 as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg));
1869 %}
1870 ins_pipe(pipe_slow);
1871 %}
1872
1873 // vector fmadd - predicated
1874 // dst_src1 = dst_src1 * src2 + src3
1875
1876 instruct vfmadd_masked(vReg dst_src1, vReg src2, vReg src3, vRegMask_V0 v0) %{
1877 match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary src3 v0)));
1878 match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary src3 v0)));
1879 format %{ "vfmadd_masked $dst_src1, $dst_src1, $src2, $src3, $v0" %}
1880 ins_encode %{
1881 assert(UseFMA, "Needs FMA instructions support.");
1882 BasicType bt = Matcher::vector_element_basic_type(this);
1883 __ vsetvli_helper(bt, Matcher::vector_length(this));
1884 __ vfmadd_vv(as_VectorRegister($dst_src1$$reg), as_VectorRegister($src2$$reg),
1885 as_VectorRegister($src3$$reg), Assembler::v0_t);
1886 %}
1887 ins_pipe(pipe_slow);
1888 %}
1889
1890 // vector fmls
1891
1892 // dst_src1 = src2 * (-src3) + dst_src1
1893 // "(-src2) * src3 + dst_src1" has been idealized to "src3 * (-src2) + dst_src1"
1894 instruct vfmlsF(vReg dst_src1, vReg src2, vReg src3) %{
1895 match(Set dst_src1 (FmaVF dst_src1 (Binary src2 (NegVF src3))));
1896 format %{ "vfmlsF $dst_src1, $dst_src1, $src2, $src3" %}
1897 ins_encode %{
1898 assert(UseFMA, "Needs FMA instructions support.");
1899 __ vsetvli_helper(T_FLOAT, Matcher::vector_length(this));
1900 __ vfnmsac_vv(as_VectorRegister($dst_src1$$reg),
1901 as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg));
1902 %}
1903 ins_pipe(pipe_slow);
1904 %}
1905
1906 // dst_src1 = src2 * (-src3) + dst_src1
1907 // "(-src2) * src3 + dst_src1" has been idealized to "src3 * (-src2) + dst_src1"
1908 instruct vfmlsD(vReg dst_src1, vReg src2, vReg src3) %{
1909 match(Set dst_src1 (FmaVD dst_src1 (Binary src2 (NegVD src3))));
1910 format %{ "vfmlsD $dst_src1, $dst_src1, $src2, $src3" %}
1911 ins_encode %{
1912 assert(UseFMA, "Needs FMA instructions support.");
1913 __ vsetvli_helper(T_DOUBLE, Matcher::vector_length(this));
1914 __ vfnmsac_vv(as_VectorRegister($dst_src1$$reg),
1915 as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg));
1916 %}
1917 ins_pipe(pipe_slow);
1918 %}
1919
1920 // vector fnmsub - predicated
1921
1922 // dst_src1 = dst_src1 * (-src2) + src3
1923 instruct vfnmsub_masked(vReg dst_src1, vReg src2, vReg src3, vRegMask_V0 v0) %{
1924 match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary src3 v0)));
1925 match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary src3 v0)));
1926 format %{ "vfnmsub_masked $dst_src1, $dst_src1, $src2, $src3, $v0" %}
1927 ins_encode %{
1928 assert(UseFMA, "Needs FMA instructions support.");
1929 BasicType bt = Matcher::vector_element_basic_type(this);
1930 __ vsetvli_helper(bt, Matcher::vector_length(this));
1931 __ vfnmsub_vv(as_VectorRegister($dst_src1$$reg), as_VectorRegister($src2$$reg),
1932 as_VectorRegister($src3$$reg), Assembler::v0_t);
1933 %}
1934 ins_pipe(pipe_slow);
1935 %}
1936
1937 // vector fnmla
1938
1939 // dst_src1 = src2 * (-src3) - dst_src1
1940 // "(-src2) * src3 - dst_src1" has been idealized to "src3 * (-src2) - dst_src1"
1941 instruct vfnmlaF(vReg dst_src1, vReg src2, vReg src3) %{
1942 match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 (NegVF src3))));
1943 format %{ "vfnmlaF $dst_src1, $dst_src1, $src2, $src3" %}
1944 ins_encode %{
1945 assert(UseFMA, "Needs FMA instructions support.");
1946 __ vsetvli_helper(T_FLOAT, Matcher::vector_length(this));
1947 __ vfnmacc_vv(as_VectorRegister($dst_src1$$reg),
1948 as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg));
1949 %}
1950 ins_pipe(pipe_slow);
1951 %}
1952
1953 // dst_src1 = src2 * (-src3) - dst_src1
1954 // "(-src2) * src3 - dst_src1" has been idealized to "src3 * (-src2) - dst_src1"
1955 instruct vfnmlaD(vReg dst_src1, vReg src2, vReg src3) %{
1956 match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 (NegVD src3))));
1957 format %{ "vfnmlaD $dst_src1, $dst_src1, $src2, $src3" %}
1958 ins_encode %{
1959 assert(UseFMA, "Needs FMA instructions support.");
1960 __ vsetvli_helper(T_DOUBLE, Matcher::vector_length(this));
1961 __ vfnmacc_vv(as_VectorRegister($dst_src1$$reg),
1962 as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg));
1963 %}
1964 ins_pipe(pipe_slow);
1965 %}
1966
1967 // vector fnmadd - predicated
1968
1969 // dst_src1 = dst_src1 * (-src2) - src3
1970 instruct vfnmadd_masked(vReg dst_src1, vReg src2, vReg src3, vRegMask_V0 v0) %{
1971 match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary (NegVF src3) v0)));
1972 match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary (NegVD src3) v0)));
1973 format %{ "vfnmadd_masked $dst_src1, $dst_src1, $src2, $src3, $v0" %}
1974 ins_encode %{
1975 assert(UseFMA, "Needs FMA instructions support.");
1976 BasicType bt = Matcher::vector_element_basic_type(this);
1977 __ vsetvli_helper(bt, Matcher::vector_length(this));
1978 __ vfnmadd_vv(as_VectorRegister($dst_src1$$reg), as_VectorRegister($src2$$reg),
1979 as_VectorRegister($src3$$reg), Assembler::v0_t);
1980 %}
1981 ins_pipe(pipe_slow);
1982 %}
1983
1984 // vector fnmls
1985
1986 // dst_src1 = src2 * src3 - dst_src1
1987 instruct vfnmlsF(vReg dst_src1, vReg src2, vReg src3) %{
1988 match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 src3)));
1989 format %{ "vfnmlsF $dst_src1, $dst_src1, $src2, $src3" %}
1990 ins_encode %{
1991 assert(UseFMA, "Needs FMA instructions support.");
1992 __ vsetvli_helper(T_FLOAT, Matcher::vector_length(this));
1993 __ vfmsac_vv(as_VectorRegister($dst_src1$$reg),
1994 as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg));
1995 %}
1996 ins_pipe(pipe_slow);
1997 %}
1998
1999 // dst_src1 = -dst_src1 + src2 * src3
2000 instruct vfnmlsD(vReg dst_src1, vReg src2, vReg src3) %{
2001 match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 src3)));
2002 format %{ "vfnmlsD $dst_src1, $dst_src1, $src2, $src3" %}
2003 ins_encode %{
2004 assert(UseFMA, "Needs FMA instructions support.");
2005 __ vsetvli_helper(T_DOUBLE, Matcher::vector_length(this));
2006 __ vfmsac_vv(as_VectorRegister($dst_src1$$reg),
2007 as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg));
2008 %}
2009 ins_pipe(pipe_slow);
2010 %}
2011
2012 // vector vfmsub - predicated
2013
2014 // dst_src1 = dst_src1 * src2 - src3
2015 instruct vfmsub_masked(vReg dst_src1, vReg src2, vReg src3, vRegMask_V0 v0) %{
2016 match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary (NegVF src3) v0)));
2017 match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary (NegVD src3) v0)));
2018 format %{ "vfmsub_masked $dst_src1, $dst_src1, $src2, $src3, $v0" %}
2019 ins_encode %{
2020 assert(UseFMA, "Needs FMA instructions support.");
2021 BasicType bt = Matcher::vector_element_basic_type(this);
2022 __ vsetvli_helper(bt, Matcher::vector_length(this));
2023 __ vfmsub_vv(as_VectorRegister($dst_src1$$reg), as_VectorRegister($src2$$reg),
2024 as_VectorRegister($src3$$reg), Assembler::v0_t);
2025 %}
2026 ins_pipe(pipe_slow);
2027 %}
2028
2029 // vector mla
2030
2031 // dst_src1 = dst_src1 + src2 * src3
2032 instruct vmla(vReg dst_src1, vReg src2, vReg src3) %{
2033 match(Set dst_src1 (AddVB dst_src1 (MulVB src2 src3)));
2034 match(Set dst_src1 (AddVS dst_src1 (MulVS src2 src3)));
2035 match(Set dst_src1 (AddVI dst_src1 (MulVI src2 src3)));
2036 match(Set dst_src1 (AddVL dst_src1 (MulVL src2 src3)));
2037 format %{ "vmla $dst_src1, $dst_src1, $src2, $src3" %}
2038 ins_encode %{
2039 BasicType bt = Matcher::vector_element_basic_type(this);
2040 __ vsetvli_helper(bt, Matcher::vector_length(this));
2041 __ vmacc_vv(as_VectorRegister($dst_src1$$reg),
2042 as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg));
2043 %}
2044 ins_pipe(pipe_slow);
2045 %}
2046
2047 // vector mla - predicated
2048
2049 instruct vmla_masked(vReg dst_src1, vReg src2, vReg src3, vRegMask_V0 v0) %{
2050 match(Set dst_src1 (AddVB (Binary dst_src1 (MulVB src2 src3)) v0));
2051 match(Set dst_src1 (AddVS (Binary dst_src1 (MulVS src2 src3)) v0));
2052 match(Set dst_src1 (AddVI (Binary dst_src1 (MulVI src2 src3)) v0));
2053 match(Set dst_src1 (AddVL (Binary dst_src1 (MulVL src2 src3)) v0));
2054 format %{ "vmla_masked $dst_src1, $dst_src1, $src2, $src3, $v0" %}
2055 ins_encode %{
2056 BasicType bt = Matcher::vector_element_basic_type(this);
2057 __ vsetvli_helper(bt, Matcher::vector_length(this));
2058 __ vmacc_vv(as_VectorRegister($dst_src1$$reg), as_VectorRegister($src2$$reg),
2059 as_VectorRegister($src3$$reg), Assembler::v0_t);
2060 %}
2061 ins_pipe(pipe_slow);
2062 %}
2063
2064 // vector mls
2065
2066 // dst_src1 = dst_src1 - src2 * src3
2067 instruct vmls(vReg dst_src1, vReg src2, vReg src3) %{
2068 match(Set dst_src1 (SubVB dst_src1 (MulVB src2 src3)));
2069 match(Set dst_src1 (SubVS dst_src1 (MulVS src2 src3)));
2070 match(Set dst_src1 (SubVI dst_src1 (MulVI src2 src3)));
2071 match(Set dst_src1 (SubVL dst_src1 (MulVL src2 src3)));
2072 format %{ "vmls $dst_src1, $dst_src1, $src2, $src3" %}
2073 ins_encode %{
2074 BasicType bt = Matcher::vector_element_basic_type(this);
2075 __ vsetvli_helper(bt, Matcher::vector_length(this));
2076 __ vnmsac_vv(as_VectorRegister($dst_src1$$reg),
2077 as_VectorRegister($src2$$reg), as_VectorRegister($src3$$reg));
2078 %}
2079 ins_pipe(pipe_slow);
2080 %}
2081
2082 // vector mls - predicated
2083
2084 instruct vmls_masked(vReg dst_src1, vReg src2, vReg src3, vRegMask_V0 v0) %{
2085 match(Set dst_src1 (SubVB (Binary dst_src1 (MulVB src2 src3)) v0));
2086 match(Set dst_src1 (SubVS (Binary dst_src1 (MulVS src2 src3)) v0));
2087 match(Set dst_src1 (SubVI (Binary dst_src1 (MulVI src2 src3)) v0));
2088 match(Set dst_src1 (SubVL (Binary dst_src1 (MulVL src2 src3)) v0));
2089 format %{ "vmls_masked $dst_src1, $dst_src1, $src2, $src3, $v0" %}
2090 ins_encode %{
2091 BasicType bt = Matcher::vector_element_basic_type(this);
2092 __ vsetvli_helper(bt, Matcher::vector_length(this));
2093 __ vnmsac_vv(as_VectorRegister($dst_src1$$reg), as_VectorRegister($src2$$reg),
2094 as_VectorRegister($src3$$reg), Assembler::v0_t);
2095 %}
2096 ins_pipe(pipe_slow);
2097 %}
2098
2099 // vector mul
2100
2101 instruct vmul(vReg dst, vReg src1, vReg src2) %{
2102 match(Set dst (MulVB src1 src2));
2103 match(Set dst (MulVS src1 src2));
2104 match(Set dst (MulVI src1 src2));
2105 match(Set dst (MulVL src1 src2));
2106 format %{ "vmul $dst, $src1, $src2" %}
2107 ins_encode %{
2108 BasicType bt = Matcher::vector_element_basic_type(this);
2109 __ vsetvli_helper(bt, Matcher::vector_length(this));
2110 __ vmul_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src1$$reg),
2111 as_VectorRegister($src2$$reg));
2112 %}
2113 ins_pipe(pipe_slow);
2114 %}
2115
2116 instruct vmul_hfp(vReg dst, vReg src1, vReg src2) %{
2117 match(Set dst (MulVHF src1 src2));
2118 format %{ "vmul_hfp $dst, $src1, $src2" %}
2119 ins_encode %{
2120 assert(UseZvfh, "must");
2121 assert(Matcher::vector_element_basic_type(this) == T_SHORT, "must");
2122 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
2123 __ vfmul_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src1$$reg),
2124 as_VectorRegister($src2$$reg));
2125 %}
2126 ins_pipe(pipe_slow);
2127 %}
2128
2129 instruct vmul_fp(vReg dst, vReg src1, vReg src2) %{
2130 match(Set dst (MulVF src1 src2));
2131 match(Set dst (MulVD src1 src2));
2132 format %{ "vmul_fp $dst, $src1, $src2" %}
2133 ins_encode %{
2134 BasicType bt = Matcher::vector_element_basic_type(this);
2135 __ vsetvli_helper(bt, Matcher::vector_length(this));
2136 __ vfmul_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src1$$reg),
2137 as_VectorRegister($src2$$reg));
2138 %}
2139 ins_pipe(pipe_slow);
2140 %}
2141
2142 // vector mul - predicated
2143
2144 instruct vmul_masked(vReg dst_src1, vReg src2, vRegMask_V0 v0) %{
2145 match(Set dst_src1 (MulVB (Binary dst_src1 src2) v0));
2146 match(Set dst_src1 (MulVS (Binary dst_src1 src2) v0));
2147 match(Set dst_src1 (MulVI (Binary dst_src1 src2) v0));
2148 match(Set dst_src1 (MulVL (Binary dst_src1 src2) v0));
2149 format %{ "vmul_masked $dst_src1, $dst_src1, $src2, $v0" %}
2150 ins_encode %{
2151 BasicType bt = Matcher::vector_element_basic_type(this);
2152 __ vsetvli_helper(bt, Matcher::vector_length(this));
2153 __ vmul_vv(as_VectorRegister($dst_src1$$reg), as_VectorRegister($dst_src1$$reg),
2154 as_VectorRegister($src2$$reg), Assembler::v0_t);
2155 %}
2156 ins_pipe(pipe_slow);
2157 %}
2158
2159 instruct vmul_fp_masked(vReg dst_src1, vReg src2, vRegMask_V0 v0) %{
2160 match(Set dst_src1 (MulVF (Binary dst_src1 src2) v0));
2161 match(Set dst_src1 (MulVD (Binary dst_src1 src2) v0));
2162 format %{ "vmul_fp_masked $dst_src1, $dst_src1, $src2, $v0" %}
2163 ins_encode %{
2164 BasicType bt = Matcher::vector_element_basic_type(this);
2165 __ vsetvli_helper(bt, Matcher::vector_length(this));
2166 __ vfmul_vv(as_VectorRegister($dst_src1$$reg), as_VectorRegister($dst_src1$$reg),
2167 as_VectorRegister($src2$$reg), Assembler::v0_t);
2168 %}
2169 ins_pipe(pipe_slow);
2170 %}
2171
2172 // vector-scalar mul (unpredicated)
2173
2174 instruct vmul_vx(vReg dst, vReg src1, iRegIorL2I src2) %{
2175 match(Set dst (MulVB src1 (Replicate src2)));
2176 match(Set dst (MulVS src1 (Replicate src2)));
2177 match(Set dst (MulVI src1 (Replicate src2)));
2178 format %{ "vmul_vx $dst, $src1, $src2" %}
2179 ins_encode %{
2180 BasicType bt = Matcher::vector_element_basic_type(this);
2181 __ vsetvli_helper(bt, Matcher::vector_length(this));
2182 __ vmul_vx(as_VectorRegister($dst$$reg),
2183 as_VectorRegister($src1$$reg),
2184 as_Register($src2$$reg));
2185 %}
2186 ins_pipe(pipe_slow);
2187 %}
2188
2189 instruct vmulL_vx(vReg dst, vReg src1, iRegL src2) %{
2190 match(Set dst (MulVL src1 (Replicate src2)));
2191 format %{ "vmulL_vx $dst, $src1, $src2" %}
2192 ins_encode %{
2193 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
2194 __ vmul_vx(as_VectorRegister($dst$$reg),
2195 as_VectorRegister($src1$$reg),
2196 as_Register($src2$$reg));
2197 %}
2198 ins_pipe(pipe_slow);
2199 %}
2200
2201 // vector-scalar mul (predicated)
2202
2203 instruct vmul_vx_masked(vReg dst_src, iRegIorL2I src2, vRegMask_V0 v0) %{
2204 match(Set dst_src (MulVB (Binary dst_src (Replicate src2)) v0));
2205 match(Set dst_src (MulVS (Binary dst_src (Replicate src2)) v0));
2206 match(Set dst_src (MulVI (Binary dst_src (Replicate src2)) v0));
2207 format %{ "vmul_vx_masked $dst_src, $dst_src, $src2, $v0" %}
2208 ins_encode %{
2209 BasicType bt = Matcher::vector_element_basic_type(this);
2210 __ vsetvli_helper(bt, Matcher::vector_length(this));
2211 __ vmul_vx(as_VectorRegister($dst_src$$reg),
2212 as_VectorRegister($dst_src$$reg),
2213 as_Register($src2$$reg), Assembler::v0_t);
2214 %}
2215 ins_pipe(pipe_slow);
2216 %}
2217
2218 instruct vmulL_vx_masked(vReg dst_src, iRegL src2, vRegMask_V0 v0) %{
2219 match(Set dst_src (MulVL (Binary dst_src (Replicate src2)) v0));
2220 format %{ "vmulL_vx_masked $dst_src, $dst_src, $src2, $v0" %}
2221 ins_encode %{
2222 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
2223 __ vmul_vx(as_VectorRegister($dst_src$$reg),
2224 as_VectorRegister($dst_src$$reg),
2225 as_Register($src2$$reg), Assembler::v0_t);
2226 %}
2227 ins_pipe(pipe_slow);
2228 %}
2229
2230 // vector neg
2231
2232 instruct vneg(vReg dst, vReg src) %{
2233 match(Set dst (NegVI src));
2234 match(Set dst (NegVL src));
2235 format %{ "vneg $dst, $src" %}
2236 ins_encode %{
2237 BasicType bt = Matcher::vector_element_basic_type(this);
2238 __ vsetvli_helper(bt, Matcher::vector_length(this));
2239 __ vneg_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg));
2240 %}
2241 ins_pipe(pipe_slow);
2242 %}
2243
2244 // vector neg - predicated
2245
2246 instruct vneg_masked(vReg dst_src, vRegMask_V0 v0) %{
2247 match(Set dst_src (NegVI dst_src v0));
2248 match(Set dst_src (NegVL dst_src v0));
2249 format %{ "vneg_masked $dst_src, $dst_src, $v0" %}
2250 ins_encode %{
2251 BasicType bt = Matcher::vector_element_basic_type(this);
2252 __ vsetvli_helper(bt, Matcher::vector_length(this));
2253 __ vneg_v(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
2254 Assembler::v0_t);
2255 %}
2256 ins_pipe(pipe_slow);
2257 %}
2258
2259 // vector fneg
2260
2261 instruct vfneg(vReg dst, vReg src) %{
2262 match(Set dst (NegVF src));
2263 match(Set dst (NegVD src));
2264 format %{ "vfneg $dst, $src" %}
2265 ins_encode %{
2266 BasicType bt = Matcher::vector_element_basic_type(this);
2267 __ vsetvli_helper(bt, Matcher::vector_length(this));
2268 __ vfneg_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg));
2269 %}
2270 ins_pipe(pipe_slow);
2271 %}
2272
2273 // vector fneg - predicated
2274
2275 instruct vfneg_masked(vReg dst_src, vRegMask_V0 v0) %{
2276 match(Set dst_src (NegVF dst_src v0));
2277 match(Set dst_src (NegVD dst_src v0));
2278 format %{ "vfneg_masked $dst_src, $dst_src, $v0" %}
2279 ins_encode %{
2280 BasicType bt = Matcher::vector_element_basic_type(this);
2281 __ vsetvli_helper(bt, Matcher::vector_length(this));
2282 __ vfneg_v(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
2283 Assembler::v0_t);
2284 %}
2285 ins_pipe(pipe_slow);
2286 %}
2287
2288 // vector and reduction
2289
2290 instruct reduce_and(iRegINoSp dst, iRegIorL2I src1, vReg src2, vReg tmp) %{
2291 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
2292 Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
2293 Matcher::vector_element_basic_type(n->in(2)) == T_INT);
2294 match(Set dst (AndReductionV src1 src2));
2295 effect(TEMP tmp);
2296 format %{ "reduce_and $dst, $src1, $src2\t# KILL $tmp" %}
2297 ins_encode %{
2298 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2299 __ reduce_integral_v($dst$$Register, $src1$$Register,
2300 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2301 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2));
2302 %}
2303 ins_pipe(pipe_slow);
2304 %}
2305
2306 instruct reduce_andL(iRegLNoSp dst, iRegL src1, vReg src2, vReg tmp) %{
2307 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
2308 match(Set dst (AndReductionV src1 src2));
2309 effect(TEMP tmp);
2310 format %{ "reduce_andL $dst, $src1, $src2\t# KILL $tmp" %}
2311 ins_encode %{
2312 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2313 __ reduce_integral_v($dst$$Register, $src1$$Register,
2314 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2315 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2));
2316 %}
2317 ins_pipe(pipe_slow);
2318 %}
2319
2320 // vector and reduction - predicated
2321
2322 instruct reduce_and_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegMask_V0 v0, vReg tmp) %{
2323 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
2324 Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
2325 Matcher::vector_element_basic_type(n->in(2)) == T_INT);
2326 match(Set dst (AndReductionV (Binary src1 src2) v0));
2327 effect(TEMP tmp);
2328 format %{ "reduce_and_masked $dst, $src1, $src2, $v0\t# KILL $tmp" %}
2329 ins_encode %{
2330 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2331 __ reduce_integral_v($dst$$Register, $src1$$Register,
2332 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2333 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2),
2334 Assembler::v0_t);
2335 %}
2336 ins_pipe(pipe_slow);
2337 %}
2338
2339 instruct reduce_andL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegMask_V0 v0, vReg tmp) %{
2340 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
2341 match(Set dst (AndReductionV (Binary src1 src2) v0));
2342 effect(TEMP tmp);
2343 format %{ "reduce_andL_masked $dst, $src1, $src2, $v0\t# KILL $tmp" %}
2344 ins_encode %{
2345 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2346 __ reduce_integral_v($dst$$Register, $src1$$Register,
2347 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2348 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2),
2349 Assembler::v0_t);
2350 %}
2351 ins_pipe(pipe_slow);
2352 %}
2353
2354 // vector or reduction
2355
2356 instruct reduce_or(iRegINoSp dst, iRegIorL2I src1, vReg src2, vReg tmp) %{
2357 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
2358 Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
2359 Matcher::vector_element_basic_type(n->in(2)) == T_INT);
2360 match(Set dst (OrReductionV src1 src2));
2361 effect(TEMP tmp);
2362 format %{ "reduce_or $dst, $src1, $src2\t# KILL $tmp" %}
2363 ins_encode %{
2364 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2365 __ reduce_integral_v($dst$$Register, $src1$$Register,
2366 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2367 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2));
2368 %}
2369 ins_pipe(pipe_slow);
2370 %}
2371
2372 instruct reduce_orL(iRegLNoSp dst, iRegL src1, vReg src2, vReg tmp) %{
2373 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
2374 match(Set dst (OrReductionV src1 src2));
2375 effect(TEMP tmp);
2376 format %{ "reduce_orL $dst, $src1, $src2\t# KILL $tmp" %}
2377 ins_encode %{
2378 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2379 __ reduce_integral_v($dst$$Register, $src1$$Register,
2380 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2381 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2));
2382 %}
2383 ins_pipe(pipe_slow);
2384 %}
2385
2386 // vector or reduction - predicated
2387
2388 instruct reduce_or_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegMask_V0 v0, vReg tmp) %{
2389 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
2390 Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
2391 Matcher::vector_element_basic_type(n->in(2)) == T_INT);
2392 match(Set dst (OrReductionV (Binary src1 src2) v0));
2393 effect(TEMP tmp);
2394 format %{ "reduce_or_masked $dst, $src1, $src2, $v0\t# KILL $tmp" %}
2395 ins_encode %{
2396 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2397 __ reduce_integral_v($dst$$Register, $src1$$Register,
2398 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2399 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2),
2400 Assembler::v0_t);
2401 %}
2402 ins_pipe(pipe_slow);
2403 %}
2404
2405 instruct reduce_orL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegMask_V0 v0, vReg tmp) %{
2406 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
2407 match(Set dst (OrReductionV (Binary src1 src2) v0));
2408 effect(TEMP tmp);
2409 format %{ "reduce_orL_masked $dst, $src1, $src2, $v0\t# KILL $tmp" %}
2410 ins_encode %{
2411 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2412 __ reduce_integral_v($dst$$Register, $src1$$Register,
2413 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2414 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2),
2415 Assembler::v0_t);
2416 %}
2417 ins_pipe(pipe_slow);
2418 %}
2419
2420 // vector xor reduction
2421
2422 instruct reduce_xor(iRegINoSp dst, iRegIorL2I src1, vReg src2, vReg tmp) %{
2423 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
2424 Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
2425 Matcher::vector_element_basic_type(n->in(2)) == T_INT);
2426 match(Set dst (XorReductionV src1 src2));
2427 effect(TEMP tmp);
2428 format %{ "reduce_xor $dst, $src1, $src2\t# KILL $tmp" %}
2429 ins_encode %{
2430 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2431 __ reduce_integral_v($dst$$Register, $src1$$Register,
2432 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2433 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2));
2434 %}
2435 ins_pipe(pipe_slow);
2436 %}
2437
2438 instruct reduce_xorL(iRegLNoSp dst, iRegL src1, vReg src2, vReg tmp) %{
2439 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
2440 match(Set dst (XorReductionV src1 src2));
2441 effect(TEMP tmp);
2442 format %{ "reduce_xorL $dst, $src1, $src2\t# KILL $tmp" %}
2443 ins_encode %{
2444 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2445 __ reduce_integral_v($dst$$Register, $src1$$Register,
2446 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2447 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2));
2448 %}
2449 ins_pipe(pipe_slow);
2450 %}
2451
2452 // vector xor reduction - predicated
2453
2454 instruct reduce_xor_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegMask_V0 v0, vReg tmp) %{
2455 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
2456 Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
2457 Matcher::vector_element_basic_type(n->in(2)) == T_INT);
2458 match(Set dst (XorReductionV (Binary src1 src2) v0));
2459 effect(TEMP tmp);
2460 format %{ "reduce_xor_masked $dst, $src1, $src2, $v0\t# KILL $tmp" %}
2461 ins_encode %{
2462 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2463 __ reduce_integral_v($dst$$Register, $src1$$Register,
2464 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2465 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2),
2466 Assembler::v0_t);
2467 %}
2468 ins_pipe(pipe_slow);
2469 %}
2470
2471 instruct reduce_xorL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegMask_V0 v0, vReg tmp) %{
2472 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
2473 match(Set dst (XorReductionV (Binary src1 src2) v0));
2474 effect(TEMP tmp);
2475 format %{ "reduce_xorL_masked $dst, $src1, $src2, $v0\t# KILL $tmp" %}
2476 ins_encode %{
2477 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2478 __ reduce_integral_v($dst$$Register, $src1$$Register,
2479 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2480 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2),
2481 Assembler::v0_t);
2482 %}
2483 ins_pipe(pipe_slow);
2484 %}
2485
2486 // vector add reduction
2487
2488 instruct reduce_add(iRegINoSp dst, iRegIorL2I src1, vReg src2, vReg tmp) %{
2489 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
2490 Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
2491 Matcher::vector_element_basic_type(n->in(2)) == T_INT);
2492 match(Set dst (AddReductionVI src1 src2));
2493 effect(TEMP tmp);
2494 format %{ "reduce_add $dst, $src1, $src2\t# KILL $tmp" %}
2495 ins_encode %{
2496 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2497 __ reduce_integral_v($dst$$Register, $src1$$Register,
2498 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2499 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2));
2500 %}
2501 ins_pipe(pipe_slow);
2502 %}
2503
2504 instruct reduce_addL(iRegLNoSp dst, iRegL src1, vReg src2, vReg tmp) %{
2505 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
2506 match(Set dst (AddReductionVL src1 src2));
2507 effect(TEMP tmp);
2508 format %{ "reduce_addL $dst, $src1, $src2\t# KILL $tmp" %}
2509 ins_encode %{
2510 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2511 __ reduce_integral_v($dst$$Register, $src1$$Register,
2512 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2513 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2));
2514 %}
2515 ins_pipe(pipe_slow);
2516 %}
2517
2518 // Distinguish two cases based on requires_strict_order
2519 // 1. Non strictly-ordered AddReductionVF/D. For example, AddReductionVF/D
2520 // generated by Vector API. It is more beneficial performance-wise to do
2521 // an unordered FP reduction sum (vfredusum.vs).
2522 // 2. Strictly-ordered AddReductionVF/D. For example, AddReductionVF/D
2523 // generated by auto-vectorization. Must do an ordered FP reduction sum
2524 // (vfredosum.vs).
2525
2526 instruct reduce_addF_ordered(fRegF dst, fRegF src1, vReg src2, vReg tmp) %{
2527 predicate(n->as_Reduction()->requires_strict_order());
2528 match(Set dst (AddReductionVF src1 src2));
2529 effect(TEMP tmp);
2530 format %{ "reduce_addF_ordered $dst, $src1, $src2\t# KILL $tmp" %}
2531 ins_encode %{
2532 __ vsetvli_helper(T_FLOAT, Matcher::vector_length(this, $src2));
2533 __ vfmv_s_f(as_VectorRegister($tmp$$reg), $src1$$FloatRegister);
2534 __ vfredosum_vs(as_VectorRegister($tmp$$reg), as_VectorRegister($src2$$reg),
2535 as_VectorRegister($tmp$$reg));
2536 __ vfmv_f_s($dst$$FloatRegister, as_VectorRegister($tmp$$reg));
2537 %}
2538 ins_pipe(pipe_slow);
2539 %}
2540
2541 instruct reduce_addF_unordered(fRegF dst, fRegF src1, vReg src2, vReg tmp) %{
2542 predicate(!n->as_Reduction()->requires_strict_order());
2543 match(Set dst (AddReductionVF src1 src2));
2544 effect(TEMP tmp);
2545 format %{ "reduce_addF_unordered $dst, $src1, $src2\t# KILL $tmp" %}
2546 ins_encode %{
2547 __ vsetvli_helper(T_FLOAT, Matcher::vector_length(this, $src2));
2548 __ vfmv_s_f(as_VectorRegister($tmp$$reg), $src1$$FloatRegister);
2549 __ vfredusum_vs(as_VectorRegister($tmp$$reg), as_VectorRegister($src2$$reg),
2550 as_VectorRegister($tmp$$reg));
2551 __ vfmv_f_s($dst$$FloatRegister, as_VectorRegister($tmp$$reg));
2552 %}
2553 ins_pipe(pipe_slow);
2554 %}
2555
2556 instruct reduce_addD_ordered(fRegD dst, fRegD src1, vReg src2, vReg tmp) %{
2557 predicate(n->as_Reduction()->requires_strict_order());
2558 match(Set dst (AddReductionVD src1 src2));
2559 effect(TEMP tmp);
2560 format %{ "reduce_addD_ordered $dst, $src1, $src2\t# KILL $tmp" %}
2561 ins_encode %{
2562 __ vsetvli_helper(T_DOUBLE, Matcher::vector_length(this, $src2));
2563 __ vfmv_s_f(as_VectorRegister($tmp$$reg), $src1$$FloatRegister);
2564 __ vfredosum_vs(as_VectorRegister($tmp$$reg), as_VectorRegister($src2$$reg),
2565 as_VectorRegister($tmp$$reg));
2566 __ vfmv_f_s($dst$$FloatRegister, as_VectorRegister($tmp$$reg));
2567 %}
2568 ins_pipe(pipe_slow);
2569 %}
2570
2571 instruct reduce_addD_unordered(fRegD dst, fRegD src1, vReg src2, vReg tmp) %{
2572 predicate(!n->as_Reduction()->requires_strict_order());
2573 match(Set dst (AddReductionVD src1 src2));
2574 effect(TEMP tmp);
2575 format %{ "reduce_addD_unordered $dst, $src1, $src2\t# KILL $tmp" %}
2576 ins_encode %{
2577 __ vsetvli_helper(T_DOUBLE, Matcher::vector_length(this, $src2));
2578 __ vfmv_s_f(as_VectorRegister($tmp$$reg), $src1$$FloatRegister);
2579 __ vfredusum_vs(as_VectorRegister($tmp$$reg), as_VectorRegister($src2$$reg),
2580 as_VectorRegister($tmp$$reg));
2581 __ vfmv_f_s($dst$$FloatRegister, as_VectorRegister($tmp$$reg));
2582 %}
2583 ins_pipe(pipe_slow);
2584 %}
2585
2586 // vector add reduction - predicated
2587
2588 instruct reduce_add_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegMask_V0 v0, vReg tmp) %{
2589 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
2590 Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
2591 Matcher::vector_element_basic_type(n->in(2)) == T_INT);
2592 match(Set dst (AddReductionVI (Binary src1 src2) v0));
2593 effect(TEMP tmp);
2594 format %{ "reduce_add_masked $dst, $src1, $src2, $v0\t# KILL $tmp" %}
2595 ins_encode %{
2596 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2597 __ reduce_integral_v($dst$$Register, $src1$$Register,
2598 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2599 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2),
2600 Assembler::v0_t);
2601 %}
2602 ins_pipe(pipe_slow);
2603 %}
2604
2605 instruct reduce_addL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegMask_V0 v0, vReg tmp) %{
2606 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
2607 match(Set dst (AddReductionVL (Binary src1 src2) v0));
2608 effect(TEMP tmp);
2609 format %{ "reduce_addL_masked $dst, $src1, $src2, $v0\t# KILL $tmp" %}
2610 ins_encode %{
2611 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2612 __ reduce_integral_v($dst$$Register, $src1$$Register,
2613 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2614 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2),
2615 Assembler::v0_t);
2616 %}
2617 ins_pipe(pipe_slow);
2618 %}
2619
2620 instruct reduce_addF_masked(fRegF dst, fRegF src1, vReg src2, vRegMask_V0 v0, vReg tmp) %{
2621 match(Set dst (AddReductionVF (Binary src1 src2) v0));
2622 effect(TEMP tmp);
2623 format %{ "reduce_addF_masked $dst, $src1, $src2, $v0\t# KILL $tmp" %}
2624 ins_encode %{
2625 __ vsetvli_helper(T_FLOAT, Matcher::vector_length(this, $src2));
2626 __ vfmv_s_f(as_VectorRegister($tmp$$reg), $src1$$FloatRegister);
2627 __ vfredosum_vs(as_VectorRegister($tmp$$reg), as_VectorRegister($src2$$reg),
2628 as_VectorRegister($tmp$$reg), Assembler::v0_t);
2629 __ vfmv_f_s($dst$$FloatRegister, as_VectorRegister($tmp$$reg));
2630 %}
2631 ins_pipe(pipe_slow);
2632 %}
2633
2634 instruct reduce_addD_masked(fRegD dst, fRegD src1, vReg src2, vRegMask_V0 v0, vReg tmp) %{
2635 match(Set dst (AddReductionVD (Binary src1 src2) v0));
2636 effect(TEMP tmp);
2637 format %{ "reduce_addD_masked $dst, $src1, $src2, $v0\t# KILL $tmp" %}
2638 ins_encode %{
2639 __ vsetvli_helper(T_DOUBLE, Matcher::vector_length(this, $src2));
2640 __ vfmv_s_f(as_VectorRegister($tmp$$reg), $src1$$FloatRegister);
2641 __ vfredosum_vs(as_VectorRegister($tmp$$reg), as_VectorRegister($src2$$reg),
2642 as_VectorRegister($tmp$$reg), Assembler::v0_t);
2643 __ vfmv_f_s($dst$$FloatRegister, as_VectorRegister($tmp$$reg));
2644 %}
2645 ins_pipe(pipe_slow);
2646 %}
2647
2648 // vector integer max reduction
2649
2650 instruct vreduce_max(iRegINoSp dst, iRegIorL2I src1, vReg src2, vReg tmp) %{
2651 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
2652 Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
2653 Matcher::vector_element_basic_type(n->in(2)) == T_INT);
2654 match(Set dst (MaxReductionV src1 src2));
2655 effect(TEMP tmp);
2656 format %{ "vreduce_max $dst, $src1, $src2\t# KILL $tmp" %}
2657 ins_encode %{
2658 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2659 __ reduce_integral_v($dst$$Register, $src1$$Register,
2660 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2661 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2));
2662 %}
2663 ins_pipe(pipe_slow);
2664 %}
2665
2666 instruct vreduce_maxL(iRegLNoSp dst, iRegL src1, vReg src2, vReg tmp) %{
2667 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
2668 match(Set dst (MaxReductionV src1 src2));
2669 effect(TEMP tmp);
2670 format %{ "vreduce_maxL $dst, $src1, $src2\t# KILL $tmp" %}
2671 ins_encode %{
2672 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2673 __ reduce_integral_v($dst$$Register, $src1$$Register,
2674 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2675 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2));
2676 %}
2677 ins_pipe(pipe_slow);
2678 %}
2679
2680 // vector integer max reduction - predicated
2681
2682 instruct vreduce_max_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegMask_V0 v0, vReg tmp) %{
2683 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
2684 Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
2685 Matcher::vector_element_basic_type(n->in(2)) == T_INT);
2686 match(Set dst (MaxReductionV (Binary src1 src2) v0));
2687 effect(TEMP tmp);
2688 format %{ "vreduce_max_masked $dst, $src1, $src2, $v0\t# KILL $tmp" %}
2689 ins_encode %{
2690 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2691 __ reduce_integral_v($dst$$Register, $src1$$Register,
2692 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2693 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2),
2694 Assembler::v0_t);
2695 %}
2696 ins_pipe(pipe_slow);
2697 %}
2698
2699 instruct vreduce_maxL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegMask_V0 v0, vReg tmp) %{
2700 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
2701 match(Set dst (MaxReductionV (Binary src1 src2) v0));
2702 effect(TEMP tmp);
2703 format %{ "vreduce_maxL_masked $dst, $src1, $src2, $v0\t# KILL $tmp" %}
2704 ins_encode %{
2705 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2706 __ reduce_integral_v($dst$$Register, $src1$$Register,
2707 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2708 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2),
2709 Assembler::v0_t);
2710 %}
2711 ins_pipe(pipe_slow);
2712 %}
2713
2714 // vector integer min reduction
2715
2716 instruct vreduce_min(iRegINoSp dst, iRegIorL2I src1, vReg src2, vReg tmp) %{
2717 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
2718 Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
2719 Matcher::vector_element_basic_type(n->in(2)) == T_INT);
2720 match(Set dst (MinReductionV src1 src2));
2721 effect(TEMP tmp);
2722 format %{ "vreduce_min $dst, $src1, $src2\t# KILL $tmp" %}
2723 ins_encode %{
2724 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2725 __ reduce_integral_v($dst$$Register, $src1$$Register,
2726 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2727 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2));
2728 %}
2729 ins_pipe(pipe_slow);
2730 %}
2731
2732 instruct vreduce_minL(iRegLNoSp dst, iRegL src1, vReg src2, vReg tmp) %{
2733 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
2734 match(Set dst (MinReductionV src1 src2));
2735 effect(TEMP tmp);
2736 format %{ "vreduce_minL $dst, $src1, $src2\t# KILL $tmp" %}
2737 ins_encode %{
2738 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2739 __ reduce_integral_v($dst$$Register, $src1$$Register,
2740 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2741 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2));
2742 %}
2743 ins_pipe(pipe_slow);
2744 %}
2745
2746 // vector integer min reduction - predicated
2747
2748 instruct vreduce_min_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegMask_V0 v0, vReg tmp) %{
2749 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
2750 Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
2751 Matcher::vector_element_basic_type(n->in(2)) == T_INT);
2752 match(Set dst (MinReductionV (Binary src1 src2) v0));
2753 effect(TEMP tmp);
2754 format %{ "vreduce_min_masked $dst, $src1, $src2, $v0\t# KILL $tmp" %}
2755 ins_encode %{
2756 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2757 __ reduce_integral_v($dst$$Register, $src1$$Register,
2758 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2759 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2),
2760 Assembler::v0_t);
2761 %}
2762 ins_pipe(pipe_slow);
2763 %}
2764
2765 instruct vreduce_minL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegMask_V0 v0, vReg tmp) %{
2766 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
2767 match(Set dst (MinReductionV (Binary src1 src2) v0));
2768 effect(TEMP tmp);
2769 format %{ "vreduce_minL_masked $dst, $src1, $src2, $v0\t# KILL $tmp" %}
2770 ins_encode %{
2771 BasicType bt = Matcher::vector_element_basic_type(this, $src2);
2772 __ reduce_integral_v($dst$$Register, $src1$$Register,
2773 as_VectorRegister($src2$$reg), as_VectorRegister($tmp$$reg),
2774 this->ideal_Opcode(), bt, Matcher::vector_length(this, $src2),
2775 Assembler::v0_t);
2776 %}
2777 ins_pipe(pipe_slow);
2778 %}
2779
2780 // vector float max reduction
2781
2782 instruct vreduce_maxF(fRegF dst, fRegF src1, vReg src2, vReg tmp1, vReg tmp2) %{
2783 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT);
2784 match(Set dst (MaxReductionV src1 src2));
2785 effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
2786 format %{ "vreduce_maxF $dst, $src1, $src2, $tmp1, $tmp2" %}
2787 ins_encode %{
2788 __ reduce_minmax_fp_v($dst$$FloatRegister,
2789 $src1$$FloatRegister, as_VectorRegister($src2$$reg),
2790 as_VectorRegister($tmp1$$reg), as_VectorRegister($tmp2$$reg),
2791 false /* is_double */, false /* is_min */, Matcher::vector_length(this, $src2));
2792 %}
2793 ins_pipe(pipe_slow);
2794 %}
2795
2796 instruct vreduce_maxD(fRegD dst, fRegD src1, vReg src2, vReg tmp1, vReg tmp2) %{
2797 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE);
2798 match(Set dst (MaxReductionV src1 src2));
2799 effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
2800 format %{ "vreduce_maxD $dst, $src1, $src2, $tmp1, $tmp2" %}
2801 ins_encode %{
2802 __ reduce_minmax_fp_v($dst$$FloatRegister,
2803 $src1$$FloatRegister, as_VectorRegister($src2$$reg),
2804 as_VectorRegister($tmp1$$reg), as_VectorRegister($tmp2$$reg),
2805 true /* is_double */, false /* is_min */, Matcher::vector_length(this, $src2));
2806 %}
2807 ins_pipe(pipe_slow);
2808 %}
2809
2810 // vector float max reduction - predicated
2811
2812 instruct vreduce_maxF_masked(fRegF dst, fRegF src1, vReg src2, vRegMask_V0 v0, vReg tmp1, vReg tmp2) %{
2813 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT);
2814 match(Set dst (MaxReductionV (Binary src1 src2) v0));
2815 effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
2816 format %{ "vreduce_maxF_masked $dst, $src1, $src2, $v0\t# KILL $tmp1, $tmp2" %}
2817 ins_encode %{
2818 __ reduce_minmax_fp_v($dst$$FloatRegister,
2819 $src1$$FloatRegister, as_VectorRegister($src2$$reg),
2820 as_VectorRegister($tmp1$$reg), as_VectorRegister($tmp2$$reg),
2821 false /* is_double */, false /* is_min */,
2822 Matcher::vector_length(this, $src2), Assembler::v0_t);
2823 %}
2824 ins_pipe(pipe_slow);
2825 %}
2826
2827 instruct vreduce_maxD_masked(fRegD dst, fRegD src1, vReg src2, vRegMask_V0 v0, vReg tmp1, vReg tmp2) %{
2828 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE);
2829 match(Set dst (MaxReductionV (Binary src1 src2) v0));
2830 effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
2831 format %{ "vreduce_maxD_masked $dst, $src1, $src2, $v0\t# KILL $tmp1, $tmp2" %}
2832 ins_encode %{
2833 __ reduce_minmax_fp_v($dst$$FloatRegister,
2834 $src1$$FloatRegister, as_VectorRegister($src2$$reg),
2835 as_VectorRegister($tmp1$$reg), as_VectorRegister($tmp2$$reg),
2836 true /* is_double */, false /* is_min */,
2837 Matcher::vector_length(this, $src2), Assembler::v0_t);
2838 %}
2839 ins_pipe(pipe_slow);
2840 %}
2841
2842 // vector float min reduction
2843
2844 instruct vreduce_minF(fRegF dst, fRegF src1, vReg src2, vReg tmp1, vReg tmp2) %{
2845 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT);
2846 match(Set dst (MinReductionV src1 src2));
2847 effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
2848 format %{ "vreduce_minF $dst, $src1, $src2, $tmp1, $tmp2" %}
2849 ins_encode %{
2850 __ reduce_minmax_fp_v($dst$$FloatRegister,
2851 $src1$$FloatRegister, as_VectorRegister($src2$$reg),
2852 as_VectorRegister($tmp1$$reg), as_VectorRegister($tmp2$$reg),
2853 false /* is_double */, true /* is_min */, Matcher::vector_length(this, $src2));
2854 %}
2855 ins_pipe(pipe_slow);
2856 %}
2857
2858 instruct vreduce_minD(fRegD dst, fRegD src1, vReg src2, vReg tmp1, vReg tmp2) %{
2859 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE);
2860 match(Set dst (MinReductionV src1 src2));
2861 effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
2862 format %{ "vreduce_minD $dst, $src1, $src2, $tmp1, $tmp2" %}
2863 ins_encode %{
2864 __ reduce_minmax_fp_v($dst$$FloatRegister,
2865 $src1$$FloatRegister, as_VectorRegister($src2$$reg),
2866 as_VectorRegister($tmp1$$reg), as_VectorRegister($tmp2$$reg),
2867 true /* is_double */, true /* is_min */, Matcher::vector_length(this, $src2));
2868 %}
2869 ins_pipe(pipe_slow);
2870 %}
2871
2872 // vector float min reduction - predicated
2873
2874 instruct vreduce_minF_masked(fRegF dst, fRegF src1, vReg src2, vRegMask_V0 v0, vReg tmp1, vReg tmp2) %{
2875 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT);
2876 match(Set dst (MinReductionV (Binary src1 src2) v0));
2877 effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
2878 format %{ "vreduce_minF_masked $dst, $src1, $src2, $v0\t# KILL $tmp1, $tmp2" %}
2879 ins_encode %{
2880 __ reduce_minmax_fp_v($dst$$FloatRegister,
2881 $src1$$FloatRegister, as_VectorRegister($src2$$reg),
2882 as_VectorRegister($tmp1$$reg), as_VectorRegister($tmp2$$reg),
2883 false /* is_double */, true /* is_min */,
2884 Matcher::vector_length(this, $src2), Assembler::v0_t);
2885 %}
2886 ins_pipe(pipe_slow);
2887 %}
2888
2889 instruct vreduce_minD_masked(fRegD dst, fRegD src1, vReg src2, vRegMask_V0 v0, vReg tmp1, vReg tmp2) %{
2890 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE);
2891 match(Set dst (MinReductionV (Binary src1 src2) v0));
2892 effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
2893 format %{ "vreduce_minD_masked $dst, $src1, $src2, $v0\t# KILL $tmp1, $tmp2" %}
2894 ins_encode %{
2895 __ reduce_minmax_fp_v($dst$$FloatRegister,
2896 $src1$$FloatRegister, as_VectorRegister($src2$$reg),
2897 as_VectorRegister($tmp1$$reg), as_VectorRegister($tmp2$$reg),
2898 true /* is_double */, true /* is_min */,
2899 Matcher::vector_length(this, $src2), Assembler::v0_t);
2900 %}
2901 ins_pipe(pipe_slow);
2902 %}
2903
2904
2905 // ------------------------------ Vector reduction mul -------------------------
2906
2907 instruct reduce_mulI(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc,
2908 vReg tmp1, vReg tmp2) %{
2909 match(Set dst (MulReductionVI isrc vsrc));
2910 effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
2911 format %{ "reduce_mulI $dst, $isrc, $vsrc\t" %}
2912
2913 ins_encode %{
2914 __ reduce_mul_integral_v($dst$$Register, $isrc$$Register, as_VectorRegister($vsrc$$reg),
2915 as_VectorRegister($tmp1$$reg), as_VectorRegister($tmp2$$reg),
2916 Matcher::vector_element_basic_type(this, $vsrc), Matcher::vector_length(this, $vsrc));
2917 %}
2918 ins_pipe(pipe_slow);
2919 %}
2920
2921 instruct reduce_mulI_masked(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc,
2922 vRegMask_V0 v0, vReg tmp1, vReg tmp2) %{
2923 match(Set dst (MulReductionVI (Binary isrc vsrc) v0));
2924 effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
2925 format %{ "reduce_mulI_masked $dst, $isrc, $vsrc, $v0\t" %}
2926
2927 ins_encode %{
2928 __ reduce_mul_integral_v($dst$$Register, $isrc$$Register, as_VectorRegister($vsrc$$reg),
2929 as_VectorRegister($tmp1$$reg), as_VectorRegister($tmp2$$reg),
2930 Matcher::vector_element_basic_type(this, $vsrc), Matcher::vector_length(this, $vsrc),
2931 Assembler::v0_t);
2932 %}
2933 ins_pipe(pipe_slow);
2934 %}
2935
2936 instruct reduce_mulL(iRegLNoSp dst, iRegL isrc, vReg vsrc,
2937 vReg tmp1, vReg tmp2) %{
2938 match(Set dst (MulReductionVL isrc vsrc));
2939 effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
2940 format %{ "reduce_mulL $dst, $isrc, $vsrc\t" %}
2941
2942 ins_encode %{
2943 __ reduce_mul_integral_v($dst$$Register, $isrc$$Register, as_VectorRegister($vsrc$$reg),
2944 as_VectorRegister($tmp1$$reg), as_VectorRegister($tmp2$$reg),
2945 Matcher::vector_element_basic_type(this, $vsrc), Matcher::vector_length(this, $vsrc));
2946 %}
2947 ins_pipe(pipe_slow);
2948 %}
2949
2950 instruct reduce_mulL_masked(iRegLNoSp dst, iRegL isrc, vReg vsrc,
2951 vRegMask_V0 v0, vReg tmp1, vReg tmp2) %{
2952 match(Set dst (MulReductionVL (Binary isrc vsrc) v0));
2953 effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
2954 format %{ "reduce_mulL_masked $dst, $isrc, $vsrc, $v0\t" %}
2955
2956 ins_encode %{
2957 __ reduce_mul_integral_v($dst$$Register, $isrc$$Register, as_VectorRegister($vsrc$$reg),
2958 as_VectorRegister($tmp1$$reg), as_VectorRegister($tmp2$$reg),
2959 Matcher::vector_element_basic_type(this, $vsrc), Matcher::vector_length(this, $vsrc),
2960 Assembler::v0_t);
2961 %}
2962 ins_pipe(pipe_slow);
2963 %}
2964
2965 // vector replicate
2966
2967 instruct replicate(vReg dst, iRegIorL2I src) %{
2968 predicate(Matcher::is_non_long_integral_vector(n));
2969 match(Set dst (Replicate src));
2970 format %{ "replicate $dst, $src" %}
2971 ins_encode %{
2972 BasicType bt = Matcher::vector_element_basic_type(this);
2973 __ vsetvli_helper(bt, Matcher::vector_length(this));
2974 __ vmv_v_x(as_VectorRegister($dst$$reg), as_Register($src$$reg));
2975 %}
2976 ins_pipe(pipe_slow);
2977 %}
2978
2979 instruct replicateL(vReg dst, iRegL src) %{
2980 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
2981 match(Set dst (Replicate src));
2982 format %{ "replicateL $dst, $src" %}
2983 ins_encode %{
2984 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
2985 __ vmv_v_x(as_VectorRegister($dst$$reg), as_Register($src$$reg));
2986 %}
2987 ins_pipe(pipe_slow);
2988 %}
2989
2990 instruct replicate_imm5(vReg dst, immI5 con) %{
2991 predicate(Matcher::is_non_long_integral_vector(n));
2992 match(Set dst (Replicate con));
2993 format %{ "replicate_imm5 $dst, $con" %}
2994 ins_encode %{
2995 BasicType bt = Matcher::vector_element_basic_type(this);
2996 __ vsetvli_helper(bt, Matcher::vector_length_in_bytes(this));
2997 __ vmv_v_i(as_VectorRegister($dst$$reg), $con$$constant);
2998 %}
2999 ins_pipe(pipe_slow);
3000 %}
3001
3002 instruct replicateL_imm5(vReg dst, immL5 con) %{
3003 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
3004 match(Set dst (Replicate con));
3005 format %{ "replicateL_imm5 $dst, $con" %}
3006 ins_encode %{
3007 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
3008 __ vmv_v_i(as_VectorRegister($dst$$reg), $con$$constant);
3009 %}
3010 ins_pipe(pipe_slow);
3011 %}
3012
3013 instruct replicateHF(vReg dst, fRegF src) %{
3014 predicate(Matcher::vector_element_basic_type(n) == T_SHORT);
3015 match(Set dst (Replicate src));
3016 format %{ "replicateHF $dst, $src" %}
3017 ins_encode %{
3018 assert(UseZvfh, "must");
3019 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
3020 __ vfmv_v_f(as_VectorRegister($dst$$reg), $src$$FloatRegister);
3021 %}
3022 ins_pipe(pipe_slow);
3023 %}
3024
3025 instruct replicateF(vReg dst, fRegF src) %{
3026 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
3027 match(Set dst (Replicate src));
3028 format %{ "replicateF $dst, $src" %}
3029 ins_encode %{
3030 __ vsetvli_helper(T_FLOAT, Matcher::vector_length(this));
3031 __ vfmv_v_f(as_VectorRegister($dst$$reg), $src$$FloatRegister);
3032 %}
3033 ins_pipe(pipe_slow);
3034 %}
3035
3036 instruct replicateD(vReg dst, fRegD src) %{
3037 predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
3038 match(Set dst (Replicate src));
3039 format %{ "replicateD $dst, $src" %}
3040 ins_encode %{
3041 __ vsetvli_helper(T_DOUBLE, Matcher::vector_length(this));
3042 __ vfmv_v_f(as_VectorRegister($dst$$reg), $src$$FloatRegister);
3043 %}
3044 ins_pipe(pipe_slow);
3045 %}
3046
3047 // vector shift
3048 //
3049 // Following shift instruct's are shared by vectorization (in SLP, superword.cpp) and Vector API.
3050 //
3051 // Shift behaviour in vectorization is defined by java language spec, which includes:
3052 // 1. "If the promoted type of the left-hand operand is int, then only the five lowest-order bits of
3053 // the right-hand operand are used as the shift distance. It is as if the right-hand operand were
3054 // subjected to a bitwise logical AND operator & (§15.22.1) with the mask value 0x1f (0b11111).
3055 // The shift distance actually used is therefore always in the range 0 to 31, inclusive."
3056 // 2. similarly, for long "with the mask value 0x3f (0b111111)"
3057 // check https://docs.oracle.com/javase/specs/jls/se21/html/jls-15.html#jls-15.19 for details.
3058 //
3059 // Shift behaviour in Vector API is defined as:
3060 // e.g. for ASHR, "a>>(n&(ESIZE*8-1))"
3061 // this behaviour is the same as shift instrunction's in riscv vector extension.
3062 // check https://docs.oracle.com/en/java/javase/21/docs/api/jdk.incubator.vector/jdk/incubator/vector/VectorOperators.html#ASHR
3063 // and https://github.com/riscv/riscv-v-spec/blob/master/v-spec.adoc#116-vector-single-width-shift-instructions for details.
3064 //
3065 // Despite the difference between these 2 behaviours, the same shift instruct's of byte and short are
3066 // still shared between vectorization and Vector API. The way it works is hidden inside the implementation
3067 // of vectorization and Vector API:
3068 // 1. when doing optimization vectorization masks the shift value with "(BitsPerInt - 1)" or "(BitsPerLong - 1)"
3069 // 2. in Vector API, shift value is masked with SHIFT_MASK (e.g. for ByteVector it's "Byte.SIZE - 1")
3070 //
3071 // If not because of this pre-processing of shift value respectively in vectorization and Vector API, then
3072 // e.g. for a byte shift value 16, the intrinsic behaviour will be different, and they can not share the same
3073 // instruct here, as vectorization requires x >> 16, but Vector API requires x >> (16 & 7).
3074
3075 instruct vasrB(vReg dst, vReg src, vReg shift, vRegMask_V0 v0) %{
3076 match(Set dst (RShiftVB src shift));
3077 effect(TEMP_DEF dst, TEMP v0);
3078 format %{ "vasrB $dst, $src, $shift" %}
3079 ins_encode %{
3080 __ vsetvli_helper(T_BYTE, Matcher::vector_length(this));
3081 // if shift > BitsPerByte - 1, clear the low BitsPerByte - 1 bits
3082 __ vmsgtu_vi(as_VectorRegister($v0$$reg), as_VectorRegister($shift$$reg), BitsPerByte - 1);
3083 __ vsra_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3084 BitsPerByte - 1, Assembler::v0_t);
3085 // otherwise, shift
3086 __ vmnot_m(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg));
3087 __ vsra_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3088 as_VectorRegister($shift$$reg), Assembler::v0_t);
3089 %}
3090 ins_pipe(pipe_slow);
3091 %}
3092
3093 instruct vasrS(vReg dst, vReg src, vReg shift, vRegMask_V0 v0) %{
3094 match(Set dst (RShiftVS src shift));
3095 effect(TEMP_DEF dst, TEMP v0);
3096 format %{ "vasrS $dst, $src, $shift" %}
3097 ins_encode %{
3098 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
3099 // if shift > BitsPerShort - 1, clear the low BitsPerShort - 1 bits
3100 __ vmsgtu_vi(as_VectorRegister($v0$$reg), as_VectorRegister($shift$$reg), BitsPerShort - 1);
3101 __ vsra_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3102 BitsPerShort - 1, Assembler::v0_t);
3103 // otherwise, shift
3104 __ vmnot_m(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg));
3105 __ vsra_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3106 as_VectorRegister($shift$$reg), Assembler::v0_t);
3107 %}
3108 ins_pipe(pipe_slow);
3109 %}
3110
3111 instruct vasrI(vReg dst, vReg src, vReg shift) %{
3112 match(Set dst (RShiftVI src shift));
3113 format %{ "vasrI $dst, $src, $shift" %}
3114 ins_encode %{
3115 __ vsetvli_helper(T_INT, Matcher::vector_length(this));
3116 __ vsra_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3117 as_VectorRegister($shift$$reg));
3118 %}
3119 ins_pipe(pipe_slow);
3120 %}
3121
3122 instruct vasrL(vReg dst, vReg src, vReg shift) %{
3123 match(Set dst (RShiftVL src shift));
3124 format %{ "vasrL $dst, $src, $shift" %}
3125 ins_encode %{
3126 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
3127 __ vsra_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3128 as_VectorRegister($shift$$reg));
3129 %}
3130 ins_pipe(pipe_slow);
3131 %}
3132
3133 instruct vasrB_masked(vReg dst_src, vReg shift, vRegMask vmask, vRegMask_V0 v0) %{
3134 match(Set dst_src (RShiftVB (Binary dst_src shift) vmask));
3135 effect(TEMP_DEF dst_src, TEMP v0);
3136 format %{ "vasrB_masked $dst_src, $dst_src, $shift, $vmask\t# KILL $v0" %}
3137 ins_encode %{
3138 __ vsetvli_helper(T_BYTE, Matcher::vector_length(this));
3139 __ vmsgtu_vi(as_VectorRegister($v0$$reg), as_VectorRegister($shift$$reg), BitsPerByte - 1);
3140 // if shift > BitsPerByte - 1, clear the low BitsPerByte - 1 bits
3141 __ vmerge_vim(as_VectorRegister($shift$$reg), as_VectorRegister($shift$$reg), BitsPerByte - 1);
3142 // otherwise, shift
3143 __ vmv1r_v(as_VectorRegister($v0$$reg), as_VectorRegister($vmask$$reg));
3144 __ vsra_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3145 as_VectorRegister($shift$$reg), Assembler::v0_t);
3146 %}
3147 ins_pipe(pipe_slow);
3148 %}
3149
3150 instruct vasrS_masked(vReg dst_src, vReg shift, vRegMask vmask, vRegMask_V0 v0) %{
3151 match(Set dst_src (RShiftVS (Binary dst_src shift) vmask));
3152 effect(TEMP_DEF dst_src, TEMP v0);
3153 format %{ "vasrS_masked $dst_src, $dst_src, $shift, $vmask\t# KILL $v0" %}
3154 ins_encode %{
3155 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
3156 __ vmsgtu_vi(as_VectorRegister($v0$$reg), as_VectorRegister($shift$$reg), BitsPerShort - 1);
3157 // if shift > BitsPerShort - 1, clear the low BitsPerShort - 1 bits
3158 __ vmerge_vim(as_VectorRegister($shift$$reg), as_VectorRegister($shift$$reg), BitsPerShort - 1);
3159 // otherwise, shift
3160 __ vmv1r_v(as_VectorRegister($v0$$reg), as_VectorRegister($vmask$$reg));
3161 __ vsra_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3162 as_VectorRegister($shift$$reg), Assembler::v0_t);
3163 %}
3164 ins_pipe(pipe_slow);
3165 %}
3166
3167 instruct vasrI_masked(vReg dst_src, vReg shift, vRegMask_V0 v0) %{
3168 match(Set dst_src (RShiftVI (Binary dst_src shift) v0));
3169 effect(TEMP_DEF dst_src);
3170 format %{ "vasrI_masked $dst_src, $dst_src, $shift, $v0" %}
3171 ins_encode %{
3172 __ vsetvli_helper(T_INT, Matcher::vector_length(this));
3173 __ vsra_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3174 as_VectorRegister($shift$$reg), Assembler::v0_t);
3175 %}
3176 ins_pipe(pipe_slow);
3177 %}
3178
3179 instruct vasrL_masked(vReg dst_src, vReg shift, vRegMask_V0 v0) %{
3180 match(Set dst_src (RShiftVL (Binary dst_src shift) v0));
3181 effect(TEMP_DEF dst_src);
3182 format %{ "vasrL_masked $dst_src, $dst_src, $shift, $v0" %}
3183 ins_encode %{
3184 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
3185 __ vsra_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3186 as_VectorRegister($shift$$reg), Assembler::v0_t);
3187 %}
3188 ins_pipe(pipe_slow);
3189 %}
3190
3191 instruct vlslB(vReg dst, vReg src, vReg shift, vRegMask_V0 v0) %{
3192 match(Set dst (LShiftVB src shift));
3193 effect(TEMP_DEF dst, TEMP v0);
3194 format %{ "vlslB $dst, $src, $shift" %}
3195 ins_encode %{
3196 __ vsetvli_helper(T_BYTE, Matcher::vector_length(this));
3197 // if shift > BitsPerByte - 1, clear the element
3198 __ vmsgtu_vi(as_VectorRegister($v0$$reg), as_VectorRegister($shift$$reg), BitsPerByte - 1);
3199 __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3200 as_VectorRegister($src$$reg), Assembler::v0_t);
3201 // otherwise, shift
3202 __ vmnot_m(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg));
3203 __ vsll_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3204 as_VectorRegister($shift$$reg), Assembler::v0_t);
3205 %}
3206 ins_pipe(pipe_slow);
3207 %}
3208
3209 instruct vlslS(vReg dst, vReg src, vReg shift, vRegMask_V0 v0) %{
3210 match(Set dst (LShiftVS src shift));
3211 effect(TEMP_DEF dst, TEMP v0);
3212 format %{ "vlslS $dst, $src, $shift" %}
3213 ins_encode %{
3214 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
3215 // if shift > BitsPerShort - 1, clear the element
3216 __ vmsgtu_vi(as_VectorRegister($v0$$reg), as_VectorRegister($shift$$reg), BitsPerShort - 1);
3217 __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3218 as_VectorRegister($src$$reg), Assembler::v0_t);
3219 // otherwise, shift
3220 __ vmnot_m(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg));
3221 __ vsll_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3222 as_VectorRegister($shift$$reg), Assembler::v0_t);
3223 %}
3224 ins_pipe(pipe_slow);
3225 %}
3226
3227 instruct vlslI(vReg dst, vReg src, vReg shift) %{
3228 match(Set dst (LShiftVI src shift));
3229 format %{ "vlslI $dst, $src, $shift" %}
3230 ins_encode %{
3231 __ vsetvli_helper(T_INT, Matcher::vector_length(this));
3232 __ vsll_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3233 as_VectorRegister($shift$$reg));
3234 %}
3235 ins_pipe(pipe_slow);
3236 %}
3237
3238 instruct vlslL(vReg dst, vReg src, vReg shift) %{
3239 match(Set dst (LShiftVL src shift));
3240 format %{ "vlslL $dst, $src, $shift" %}
3241 ins_encode %{
3242 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
3243 __ vsll_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3244 as_VectorRegister($shift$$reg));
3245 %}
3246 ins_pipe(pipe_slow);
3247 %}
3248
3249 instruct vlslB_masked(vReg dst_src, vReg shift, vRegMask vmask, vRegMask_V0 v0) %{
3250 match(Set dst_src (LShiftVB (Binary dst_src shift) vmask));
3251 effect(TEMP_DEF dst_src, TEMP v0);
3252 format %{ "vlslB_masked $dst_src, $dst_src, $shift, $vmask\t# KILL $v0" %}
3253 ins_encode %{
3254 __ vsetvli_helper(T_BYTE, Matcher::vector_length(this));
3255 // if shift > BitsPerByte - 1, clear the element
3256 __ vmsgtu_vi(as_VectorRegister($v0$$reg), as_VectorRegister($shift$$reg), BitsPerByte - 1);
3257 __ vmand_mm(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg),
3258 as_VectorRegister($vmask$$reg));
3259 __ vxor_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3260 as_VectorRegister($dst_src$$reg), Assembler::v0_t);
3261 // otherwise, shift
3262 __ vmv1r_v(as_VectorRegister($v0$$reg), as_VectorRegister($vmask$$reg));
3263 __ vsll_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3264 as_VectorRegister($shift$$reg), Assembler::v0_t);
3265 %}
3266 ins_pipe(pipe_slow);
3267 %}
3268
3269 instruct vlslS_masked(vReg dst_src, vReg shift, vRegMask vmask, vRegMask_V0 v0) %{
3270 match(Set dst_src (LShiftVS (Binary dst_src shift) vmask));
3271 effect(TEMP_DEF dst_src, TEMP v0);
3272 format %{ "vlslS_masked $dst_src, $dst_src, $shift, $vmask\t# KILL $v0" %}
3273 ins_encode %{
3274 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
3275 // if shift > BitsPerShort - 1, clear the element
3276 __ vmsgtu_vi(as_VectorRegister($v0$$reg), as_VectorRegister($shift$$reg), BitsPerShort - 1);
3277 __ vmand_mm(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg),
3278 as_VectorRegister($vmask$$reg));
3279 __ vxor_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3280 as_VectorRegister($dst_src$$reg), Assembler::v0_t);
3281 // otherwise, shift
3282 __ vmv1r_v(as_VectorRegister($v0$$reg), as_VectorRegister($vmask$$reg));
3283 __ vsll_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3284 as_VectorRegister($shift$$reg), Assembler::v0_t);
3285 %}
3286 ins_pipe(pipe_slow);
3287 %}
3288
3289 instruct vlslI_masked(vReg dst_src, vReg shift, vRegMask_V0 v0) %{
3290 match(Set dst_src (LShiftVI (Binary dst_src shift) v0));
3291 effect(TEMP_DEF dst_src);
3292 format %{ "vlslI_masked $dst_src, $dst_src, $shift, $v0" %}
3293 ins_encode %{
3294 __ vsetvli_helper(T_INT, Matcher::vector_length(this));
3295 __ vsll_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3296 as_VectorRegister($shift$$reg), Assembler::v0_t);
3297 %}
3298 ins_pipe(pipe_slow);
3299 %}
3300
3301 instruct vlslL_masked(vReg dst_src, vReg shift, vRegMask_V0 v0) %{
3302 match(Set dst_src (LShiftVL (Binary dst_src shift) v0));
3303 effect(TEMP_DEF dst_src);
3304 format %{ "vlslL_masked $dst_src, $dst_src, $shift, $v0" %}
3305 ins_encode %{
3306 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
3307 __ vsll_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3308 as_VectorRegister($shift$$reg), Assembler::v0_t);
3309 %}
3310 ins_pipe(pipe_slow);
3311 %}
3312
3313 instruct vlsrB(vReg dst, vReg src, vReg shift, vRegMask_V0 v0) %{
3314 match(Set dst (URShiftVB src shift));
3315 effect(TEMP_DEF dst, TEMP v0);
3316 format %{ "vlsrB $dst, $src, $shift" %}
3317 ins_encode %{
3318 __ vsetvli_helper(T_BYTE, Matcher::vector_length(this));
3319 // if shift > BitsPerByte - 1, clear the element
3320 __ vmsgtu_vi(as_VectorRegister($v0$$reg), as_VectorRegister($shift$$reg), BitsPerByte - 1);
3321 __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3322 as_VectorRegister($src$$reg), Assembler::v0_t);
3323 // otherwise, shift
3324 __ vmnot_m(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg));
3325 __ vsrl_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3326 as_VectorRegister($shift$$reg), Assembler::v0_t);
3327 %}
3328 ins_pipe(pipe_slow);
3329 %}
3330
3331 instruct vlsrS(vReg dst, vReg src, vReg shift, vRegMask_V0 v0) %{
3332 match(Set dst (URShiftVS src shift));
3333 effect(TEMP_DEF dst, TEMP v0);
3334 format %{ "vlsrS $dst, $src, $shift" %}
3335 ins_encode %{
3336 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
3337 // if shift > BitsPerShort - 1, clear the element
3338 __ vmsgtu_vi(as_VectorRegister($v0$$reg), as_VectorRegister($shift$$reg), BitsPerShort - 1);
3339 __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3340 as_VectorRegister($src$$reg), Assembler::v0_t);
3341 // otherwise, shift
3342 __ vmnot_m(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg));
3343 __ vsrl_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3344 as_VectorRegister($shift$$reg), Assembler::v0_t);
3345 %}
3346 ins_pipe(pipe_slow);
3347 %}
3348
3349 instruct vlsrI(vReg dst, vReg src, vReg shift) %{
3350 match(Set dst (URShiftVI src shift));
3351 format %{ "vlsrI $dst, $src, $shift" %}
3352 ins_encode %{
3353 __ vsetvli_helper(T_INT, Matcher::vector_length(this));
3354 __ vsrl_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3355 as_VectorRegister($shift$$reg));
3356 %}
3357 ins_pipe(pipe_slow);
3358 %}
3359
3360 instruct vlsrL(vReg dst, vReg src, vReg shift) %{
3361 match(Set dst (URShiftVL src shift));
3362 format %{ "vlsrL $dst, $src, $shift" %}
3363 ins_encode %{
3364 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
3365 __ vsrl_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3366 as_VectorRegister($shift$$reg));
3367 %}
3368 ins_pipe(pipe_slow);
3369 %}
3370
3371 instruct vlsrB_masked(vReg dst_src, vReg shift, vRegMask vmask, vRegMask_V0 v0) %{
3372 match(Set dst_src (URShiftVB (Binary dst_src shift) vmask));
3373 effect(TEMP_DEF dst_src, TEMP v0);
3374 format %{ "vlsrB_masked $dst_src, $dst_src, $shift, $vmask\t# KILL $v0" %}
3375 ins_encode %{
3376 __ vsetvli_helper(T_BYTE, Matcher::vector_length(this));
3377 // if shift > BitsPerByte - 1, clear the element
3378 __ vmsgtu_vi(as_VectorRegister($v0$$reg), as_VectorRegister($shift$$reg), BitsPerByte - 1);
3379 __ vmand_mm(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg),
3380 as_VectorRegister($vmask$$reg));
3381 __ vxor_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3382 as_VectorRegister($dst_src$$reg), Assembler::v0_t);
3383 // otherwise, shift
3384 __ vmv1r_v(as_VectorRegister($v0$$reg), as_VectorRegister($vmask$$reg));
3385 __ vsrl_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3386 as_VectorRegister($shift$$reg), Assembler::v0_t);
3387 %}
3388 ins_pipe(pipe_slow);
3389 %}
3390
3391 instruct vlsrS_masked(vReg dst_src, vReg shift, vRegMask vmask, vRegMask_V0 v0) %{
3392 match(Set dst_src (URShiftVS (Binary dst_src shift) vmask));
3393 effect(TEMP_DEF dst_src, TEMP v0);
3394 format %{ "vlsrS_masked $dst_src, $dst_src, $shift, $vmask\t# KILL $v0" %}
3395 ins_encode %{
3396 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
3397 // if shift > BitsPerShort - 1, clear the element
3398 __ vmsgtu_vi(as_VectorRegister($v0$$reg), as_VectorRegister($shift$$reg), BitsPerShort - 1);
3399 __ vmand_mm(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg),
3400 as_VectorRegister($vmask$$reg));
3401 __ vxor_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3402 as_VectorRegister($dst_src$$reg), Assembler::v0_t);
3403 // otherwise, shift
3404 __ vmv1r_v(as_VectorRegister($v0$$reg), as_VectorRegister($vmask$$reg));
3405 __ vsrl_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3406 as_VectorRegister($shift$$reg), Assembler::v0_t);
3407 %}
3408 ins_pipe(pipe_slow);
3409 %}
3410
3411 instruct vlsrI_masked(vReg dst_src, vReg shift, vRegMask_V0 v0) %{
3412 match(Set dst_src (URShiftVI (Binary dst_src shift) v0));
3413 effect(TEMP_DEF dst_src);
3414 format %{ "vlsrI_masked $dst_src, $dst_src, $shift, $v0" %}
3415 ins_encode %{
3416 __ vsetvli_helper(T_INT, Matcher::vector_length(this));
3417 __ vsrl_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3418 as_VectorRegister($shift$$reg), Assembler::v0_t);
3419 %}
3420 ins_pipe(pipe_slow);
3421 %}
3422
3423 instruct vlsrL_masked(vReg dst_src, vReg shift, vRegMask_V0 v0) %{
3424 match(Set dst_src (URShiftVL (Binary dst_src shift) v0));
3425 effect(TEMP_DEF dst_src);
3426 format %{ "vlsrL_masked $dst_src, $dst_src, $shift, $v0" %}
3427 ins_encode %{
3428 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
3429 __ vsrl_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3430 as_VectorRegister($shift$$reg), Assembler::v0_t);
3431 %}
3432 ins_pipe(pipe_slow);
3433 %}
3434
3435 instruct vasrB_vi(vReg dst, vReg src, immI shift) %{
3436 match(Set dst (RShiftVB src (RShiftCntV shift)));
3437 format %{ "vasrB_vi $dst, $src, $shift" %}
3438 ins_encode %{
3439 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3440 __ vsetvli_helper(T_BYTE, Matcher::vector_length(this));
3441 if (con == 0) {
3442 __ vor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3443 as_VectorRegister($src$$reg));
3444 return;
3445 }
3446 if (con >= BitsPerByte) con = BitsPerByte - 1;
3447 __ vsra_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con);
3448 %}
3449 ins_pipe(pipe_slow);
3450 %}
3451
3452 instruct vasrS_vi(vReg dst, vReg src, immI shift) %{
3453 match(Set dst (RShiftVS src (RShiftCntV shift)));
3454 format %{ "vasrS_vi $dst, $src, $shift" %}
3455 ins_encode %{
3456 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3457 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
3458 if (con == 0) {
3459 __ vor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3460 as_VectorRegister($src$$reg));
3461 return;
3462 }
3463 if (con >= BitsPerShort) con = BitsPerShort - 1;
3464 __ vsra_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con);
3465 %}
3466 ins_pipe(pipe_slow);
3467 %}
3468
3469 instruct vasrI_vi(vReg dst, vReg src, immI shift) %{
3470 match(Set dst (RShiftVI src (RShiftCntV shift)));
3471 format %{ "vasrI_vi $dst, $src, $shift" %}
3472 ins_encode %{
3473 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3474 __ vsetvli_helper(T_INT, Matcher::vector_length(this));
3475 if (con == 0) {
3476 __ vor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3477 as_VectorRegister($src$$reg));
3478 return;
3479 }
3480 __ vsra_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con);
3481 %}
3482 ins_pipe(pipe_slow);
3483 %}
3484
3485 instruct vasrL_vi(vReg dst, vReg src, immI shift) %{
3486 predicate((n->in(2)->in(1)->get_int() & 0x3f) < 32);
3487 match(Set dst (RShiftVL src (RShiftCntV shift)));
3488 format %{ "vasrL_vi $dst, $src, $shift" %}
3489 ins_encode %{
3490 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3491 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
3492 if (con == 0) {
3493 __ vor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3494 as_VectorRegister($src$$reg));
3495 return;
3496 }
3497 __ vsra_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con);
3498 %}
3499 ins_pipe(pipe_slow);
3500 %}
3501
3502 instruct vasrB_vi_masked(vReg dst_src, immI shift, vRegMask_V0 v0) %{
3503 match(Set dst_src (RShiftVB (Binary dst_src (RShiftCntV shift)) v0));
3504 format %{ "vasrB_vi_masked $dst_src, $dst_src, $shift, $v0" %}
3505 ins_encode %{
3506 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3507 if (con == 0) {
3508 return;
3509 }
3510 if (con >= BitsPerByte) con = BitsPerByte - 1;
3511 __ vsetvli_helper(T_BYTE, Matcher::vector_length(this));
3512 __ vsra_vi(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg), con,
3513 Assembler::v0_t);
3514 %}
3515 ins_pipe(pipe_slow);
3516 %}
3517
3518 instruct vasrS_vi_masked(vReg dst_src, immI shift, vRegMask_V0 v0) %{
3519 match(Set dst_src (RShiftVS (Binary dst_src (RShiftCntV shift)) v0));
3520 format %{ "vasrS_vi_masked $dst_src, $dst_src, $shift, $v0" %}
3521 ins_encode %{
3522 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3523 if (con == 0) {
3524 return;
3525 }
3526 if (con >= BitsPerShort) con = BitsPerShort - 1;
3527 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
3528 __ vsra_vi(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg), con,
3529 Assembler::v0_t);
3530 %}
3531 ins_pipe(pipe_slow);
3532 %}
3533
3534 instruct vasrI_vi_masked(vReg dst_src, immI shift, vRegMask_V0 v0) %{
3535 match(Set dst_src (RShiftVI (Binary dst_src (RShiftCntV shift)) v0));
3536 format %{ "vasrI_vi_masked $dst_src, $dst_src, $shift, $v0" %}
3537 ins_encode %{
3538 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3539 if (con == 0) {
3540 return;
3541 }
3542 __ vsetvli_helper(T_INT, Matcher::vector_length(this));
3543 __ vsra_vi(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg), con,
3544 Assembler::v0_t);
3545 %}
3546 ins_pipe(pipe_slow);
3547 %}
3548
3549 instruct vasrL_vi_masked(vReg dst_src, immI shift, vRegMask_V0 v0) %{
3550 predicate((n->in(1)->in(2)->in(1)->get_int() & 0x3f) < 32);
3551 match(Set dst_src (RShiftVL (Binary dst_src (RShiftCntV shift)) v0));
3552 format %{ "vasrL_vi_masked $dst_src, $dst_src, $shift, $v0" %}
3553 ins_encode %{
3554 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3555 if (con == 0) {
3556 return;
3557 }
3558 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
3559 __ vsra_vi(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg), con,
3560 Assembler::v0_t);
3561 %}
3562 ins_pipe(pipe_slow);
3563 %}
3564
3565 instruct vlsrB_vi(vReg dst, vReg src, immI shift) %{
3566 match(Set dst (URShiftVB src (RShiftCntV shift)));
3567 format %{ "vlsrB_vi $dst, $src, $shift" %}
3568 ins_encode %{
3569 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3570 __ vsetvli_helper(T_BYTE, Matcher::vector_length(this));
3571 if (con == 0) {
3572 __ vor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3573 as_VectorRegister($src$$reg));
3574 return;
3575 }
3576 if (con >= BitsPerByte) {
3577 __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3578 as_VectorRegister($src$$reg));
3579 return;
3580 }
3581 __ vsrl_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con);
3582 %}
3583 ins_pipe(pipe_slow);
3584 %}
3585
3586 instruct vlsrS_vi(vReg dst, vReg src, immI shift) %{
3587 match(Set dst (URShiftVS src (RShiftCntV shift)));
3588 format %{ "vlsrS_vi $dst, $src, $shift" %}
3589 ins_encode %{
3590 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3591 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
3592 if (con == 0) {
3593 __ vor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3594 as_VectorRegister($src$$reg));
3595 return;
3596 }
3597 if (con >= BitsPerShort) {
3598 __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3599 as_VectorRegister($src$$reg));
3600 return;
3601 }
3602 __ vsrl_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con);
3603 %}
3604 ins_pipe(pipe_slow);
3605 %}
3606
3607 instruct vlsrI_vi(vReg dst, vReg src, immI shift) %{
3608 match(Set dst (URShiftVI src (RShiftCntV shift)));
3609 format %{ "vlsrI_vi $dst, $src, $shift" %}
3610 ins_encode %{
3611 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3612 __ vsetvli_helper(T_INT, Matcher::vector_length(this));
3613 if (con == 0) {
3614 __ vor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3615 as_VectorRegister($src$$reg));
3616 return;
3617 }
3618 __ vsrl_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con);
3619 %}
3620 ins_pipe(pipe_slow);
3621 %}
3622
3623 instruct vlsrL_vi(vReg dst, vReg src, immI shift) %{
3624 predicate((n->in(2)->in(1)->get_int() & 0x3f) < 32);
3625 match(Set dst (URShiftVL src (RShiftCntV shift)));
3626 format %{ "vlsrL_vi $dst, $src, $shift" %}
3627 ins_encode %{
3628 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3629 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
3630 if (con == 0) {
3631 __ vor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3632 as_VectorRegister($src$$reg));
3633 return;
3634 }
3635 __ vsrl_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con);
3636 %}
3637 ins_pipe(pipe_slow);
3638 %}
3639
3640 instruct vlsrB_vi_masked(vReg dst_src, immI shift, vRegMask_V0 v0) %{
3641 match(Set dst_src (URShiftVB (Binary dst_src (RShiftCntV shift)) v0));
3642 format %{ "vlsrB_vi_masked $dst_src, $dst_src, $shift, $v0" %}
3643 ins_encode %{
3644 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3645 if (con == 0) {
3646 return;
3647 }
3648 __ vsetvli_helper(T_BYTE, Matcher::vector_length(this));
3649 if (con >= BitsPerByte) {
3650 __ vxor_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3651 as_VectorRegister($dst_src$$reg), Assembler::v0_t);
3652 return;
3653 }
3654 __ vsrl_vi(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg), con,
3655 Assembler::v0_t);
3656 %}
3657 ins_pipe(pipe_slow);
3658 %}
3659
3660 instruct vlsrS_vi_masked(vReg dst_src, immI shift, vRegMask_V0 v0) %{
3661 match(Set dst_src (URShiftVS (Binary dst_src (RShiftCntV shift)) v0));
3662 format %{ "vlsrS_vi_masked $dst_src, $dst_src, $shift, $v0" %}
3663 ins_encode %{
3664 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3665 if (con == 0) {
3666 return;
3667 }
3668 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
3669 if (con >= BitsPerShort) {
3670 __ vxor_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3671 as_VectorRegister($dst_src$$reg), Assembler::v0_t);
3672 return;
3673 }
3674 __ vsrl_vi(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg), con,
3675 Assembler::v0_t);
3676 %}
3677 ins_pipe(pipe_slow);
3678 %}
3679
3680 instruct vlsrI_vi_masked(vReg dst_src, immI shift, vRegMask_V0 v0) %{
3681 match(Set dst_src (URShiftVI (Binary dst_src (RShiftCntV shift)) v0));
3682 format %{ "vlsrI_vi_masked $dst_src, $dst_src, $shift, $v0" %}
3683 ins_encode %{
3684 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3685 if (con == 0) {
3686 return;
3687 }
3688 __ vsetvli_helper(T_INT, Matcher::vector_length(this));
3689 __ vsrl_vi(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg), con,
3690 Assembler::v0_t);
3691 %}
3692 ins_pipe(pipe_slow);
3693 %}
3694
3695 instruct vlsrL_vi_masked(vReg dst_src, immI shift, vRegMask_V0 v0) %{
3696 predicate((n->in(1)->in(2)->in(1)->get_int() & 0x3f) < 32);
3697 match(Set dst_src (URShiftVL (Binary dst_src (RShiftCntV shift)) v0));
3698 format %{ "vlsrL_vi_masked $dst_src, $dst_src, $shift, $v0" %}
3699 ins_encode %{
3700 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3701 if (con == 0) {
3702 return;
3703 }
3704 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
3705 __ vsrl_vi(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg), con,
3706 Assembler::v0_t);
3707 %}
3708 ins_pipe(pipe_slow);
3709 %}
3710
3711 instruct vlslB_vi(vReg dst, vReg src, immI shift) %{
3712 match(Set dst (LShiftVB src (LShiftCntV shift)));
3713 format %{ "vlslB_vi $dst, $src, $shift" %}
3714 ins_encode %{
3715 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3716 __ vsetvli_helper(T_BYTE, Matcher::vector_length(this));
3717 if (con >= BitsPerByte) {
3718 __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3719 as_VectorRegister($src$$reg));
3720 return;
3721 }
3722 __ vsll_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con);
3723 %}
3724 ins_pipe(pipe_slow);
3725 %}
3726
3727 instruct vlslS_vi(vReg dst, vReg src, immI shift) %{
3728 match(Set dst (LShiftVS src (LShiftCntV shift)));
3729 format %{ "vlslS_vi $dst, $src, $shift" %}
3730 ins_encode %{
3731 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3732 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
3733 if (con >= BitsPerShort) {
3734 __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3735 as_VectorRegister($src$$reg));
3736 return;
3737 }
3738 __ vsll_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con);
3739 %}
3740 ins_pipe(pipe_slow);
3741 %}
3742
3743 instruct vlslI_vi(vReg dst, vReg src, immI shift) %{
3744 match(Set dst (LShiftVI src (LShiftCntV shift)));
3745 format %{ "vlslI_vi $dst, $src, $shift" %}
3746 ins_encode %{
3747 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3748 __ vsetvli_helper(T_INT, Matcher::vector_length(this));
3749 __ vsll_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con);
3750 %}
3751 ins_pipe(pipe_slow);
3752 %}
3753
3754 instruct vlslL_vi(vReg dst, vReg src, immI shift) %{
3755 predicate((n->in(2)->in(1)->get_int() & 0x3f) < 32);
3756 match(Set dst (LShiftVL src (LShiftCntV shift)));
3757 format %{ "vlslL_vi $dst, $src, $shift" %}
3758 ins_encode %{
3759 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3760 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
3761 __ vsll_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con);
3762 %}
3763 ins_pipe(pipe_slow);
3764 %}
3765
3766 instruct vlslB_vi_masked(vReg dst_src, immI shift, vRegMask_V0 v0) %{
3767 match(Set dst_src (LShiftVB (Binary dst_src (LShiftCntV shift)) v0));
3768 format %{ "vlslB_vi_masked $dst_src, $dst_src, $shift, $v0" %}
3769 ins_encode %{
3770 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3771 __ vsetvli_helper(T_BYTE, Matcher::vector_length(this));
3772 if (con >= BitsPerByte) {
3773 __ vxor_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3774 as_VectorRegister($dst_src$$reg), Assembler::v0_t);
3775 return;
3776 }
3777 __ vsll_vi(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg), con,
3778 Assembler::v0_t);
3779 %}
3780 ins_pipe(pipe_slow);
3781 %}
3782
3783 instruct vlslS_vi_masked(vReg dst_src, immI shift, vRegMask_V0 v0) %{
3784 match(Set dst_src (LShiftVS (Binary dst_src (LShiftCntV shift)) v0));
3785 format %{ "vlslS_vi_masked $dst_src, $dst_src, $shift, $v0" %}
3786 ins_encode %{
3787 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3788 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
3789 if (con >= BitsPerShort) {
3790 __ vxor_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3791 as_VectorRegister($dst_src$$reg), Assembler::v0_t);
3792 return;
3793 }
3794 __ vsll_vi(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg), con,
3795 Assembler::v0_t);
3796 %}
3797 ins_pipe(pipe_slow);
3798 %}
3799
3800 instruct vlslI_vi_masked(vReg dst_src, immI shift, vRegMask_V0 v0) %{
3801 match(Set dst_src (LShiftVI (Binary dst_src (LShiftCntV shift)) v0));
3802 format %{ "vlslI_vi_masked $dst_src, $dst_src, $shift, $v0" %}
3803 ins_encode %{
3804 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3805 __ vsetvli_helper(T_INT, Matcher::vector_length(this));
3806 __ vsll_vi(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg), con,
3807 Assembler::v0_t);
3808 %}
3809 ins_pipe(pipe_slow);
3810 %}
3811
3812 instruct vlslL_vi_masked(vReg dst_src, immI shift, vRegMask_V0 v0) %{
3813 predicate((n->in(1)->in(2)->in(1)->get_int() & 0x3f) < 32);
3814 match(Set dst_src (LShiftVL (Binary dst_src (LShiftCntV shift)) v0));
3815 format %{ "vlslL_vi_masked $dst_src, $dst_src, $shift, $v0" %}
3816 ins_encode %{
3817 uint32_t con = (unsigned)$shift$$constant & 0x1f;
3818 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
3819 __ vsll_vi(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg), con,
3820 Assembler::v0_t);
3821 %}
3822 ins_pipe(pipe_slow);
3823 %}
3824
3825 // vector shift count
3826
3827 instruct vshiftcnt(vReg dst, iRegIorL2I cnt) %{
3828 match(Set dst (LShiftCntV cnt));
3829 match(Set dst (RShiftCntV cnt));
3830 format %{ "vshiftcnt $dst, $cnt" %}
3831 ins_encode %{
3832 BasicType bt = Matcher::vector_element_basic_type(this);
3833 __ vsetvli_helper(bt, Matcher::vector_length(this));
3834 __ vmv_v_x(as_VectorRegister($dst$$reg), as_Register($cnt$$reg));
3835 %}
3836 ins_pipe(pipe_slow);
3837 %}
3838
3839 // --------------------------------- Vector Rotation ----------------------------------
3840 // Rotate right
3841
3842 instruct vrotate_right(vReg dst, vReg src, vReg shift) %{
3843 match(Set dst (RotateRightV src shift));
3844 format %{ "vrotate_right $dst, $src, $shift\t" %}
3845 ins_encode %{
3846 BasicType bt = Matcher::vector_element_basic_type(this);
3847 __ vsetvli_helper(bt, Matcher::vector_length(this));
3848 __ vror_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3849 as_VectorRegister($shift$$reg));
3850 %}
3851 ins_pipe(pipe_slow);
3852 %}
3853
3854 // Only the low log2(SEW) bits of shift value are used, all other bits are ignored.
3855 instruct vrotate_right_vx(vReg dst, vReg src, iRegIorL2I shift) %{
3856 match(Set dst (RotateRightV src (Replicate shift)));
3857 format %{ "vrotate_right_vx $dst, $src, $shift\t" %}
3858 ins_encode %{
3859 BasicType bt = Matcher::vector_element_basic_type(this);
3860 __ vsetvli_helper(bt, Matcher::vector_length(this));
3861 __ vror_vx(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3862 as_Register($shift$$reg));
3863 %}
3864 ins_pipe(pipe_slow);
3865 %}
3866
3867 instruct vrotate_right_vi(vReg dst, vReg src, immI shift) %{
3868 match(Set dst (RotateRightV src shift));
3869 format %{ "vrotate_right_vi $dst, $src, $shift\t" %}
3870 ins_encode %{
3871 BasicType bt = Matcher::vector_element_basic_type(this);
3872 uint32_t bits = type2aelembytes(bt) * 8;
3873 uint32_t con = (unsigned)$shift$$constant & (bits - 1);
3874 if (con == 0) {
3875 return;
3876 }
3877 __ vsetvli_helper(bt, Matcher::vector_length(this));
3878 __ vror_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con);
3879 %}
3880 ins_pipe(pipe_slow);
3881 %}
3882
3883 // Rotate right - masked
3884
3885 instruct vrotate_right_masked(vReg dst_src, vReg shift, vRegMask_V0 v0) %{
3886 match(Set dst_src (RotateRightV (Binary dst_src shift) v0));
3887 format %{ "vrotate_right_masked $dst_src, $dst_src, $shift, $v0\t" %}
3888 ins_encode %{
3889 BasicType bt = Matcher::vector_element_basic_type(this);
3890 __ vsetvli_helper(bt, Matcher::vector_length(this));
3891 __ vror_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3892 as_VectorRegister($shift$$reg), Assembler::v0_t);
3893 %}
3894 ins_pipe(pipe_slow);
3895 %}
3896
3897 // Only the low log2(SEW) bits of shift value are used, all other bits are ignored.
3898 instruct vrotate_right_vx_masked(vReg dst_src, iRegIorL2I shift, vRegMask_V0 v0) %{
3899 match(Set dst_src (RotateRightV (Binary dst_src (Replicate shift)) v0));
3900 format %{ "vrotate_right_vx_masked $dst_src, $dst_src, $shift, $v0\t" %}
3901 ins_encode %{
3902 BasicType bt = Matcher::vector_element_basic_type(this);
3903 __ vsetvli_helper(bt, Matcher::vector_length(this));
3904 __ vror_vx(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3905 as_Register($shift$$reg), Assembler::v0_t);
3906 %}
3907 ins_pipe(pipe_slow);
3908 %}
3909
3910 instruct vrotate_right_vi_masked(vReg dst_src, immI shift, vRegMask_V0 v0) %{
3911 match(Set dst_src (RotateRightV (Binary dst_src shift) v0));
3912 format %{ "vrotate_right_vi_masked $dst_src, $dst_src, $shift, $v0\t" %}
3913 ins_encode %{
3914 BasicType bt = Matcher::vector_element_basic_type(this);
3915 uint32_t bits = type2aelembytes(bt) * 8;
3916 uint32_t con = (unsigned)$shift$$constant & (bits - 1);
3917 if (con == 0) {
3918 return;
3919 }
3920 __ vsetvli_helper(bt, Matcher::vector_length(this));
3921 __ vror_vi(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3922 con, Assembler::v0_t);
3923 %}
3924 ins_pipe(pipe_slow);
3925 %}
3926
3927 // Rotate left
3928
3929 instruct vrotate_left(vReg dst, vReg src, vReg shift) %{
3930 match(Set dst (RotateLeftV src shift));
3931 format %{ "vrotate_left $dst, $src, $shift\t" %}
3932 ins_encode %{
3933 BasicType bt = Matcher::vector_element_basic_type(this);
3934 __ vsetvli_helper(bt, Matcher::vector_length(this));
3935 __ vrol_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3936 as_VectorRegister($shift$$reg));
3937 %}
3938 ins_pipe(pipe_slow);
3939 %}
3940
3941 // Only the low log2(SEW) bits of shift value are used, all other bits are ignored.
3942 instruct vrotate_left_vx(vReg dst, vReg src, iRegIorL2I shift) %{
3943 match(Set dst (RotateLeftV src (Replicate shift)));
3944 format %{ "vrotate_left_vx $dst, $src, $shift\t" %}
3945 ins_encode %{
3946 BasicType bt = Matcher::vector_element_basic_type(this);
3947 __ vsetvli_helper(bt, Matcher::vector_length(this));
3948 __ vrol_vx(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
3949 as_Register($shift$$reg));
3950 %}
3951 ins_pipe(pipe_slow);
3952 %}
3953
3954 instruct vrotate_left_vi(vReg dst, vReg src, immI shift) %{
3955 match(Set dst (RotateLeftV src shift));
3956 format %{ "vrotate_left_vi $dst, $src, $shift\t" %}
3957 ins_encode %{
3958 BasicType bt = Matcher::vector_element_basic_type(this);
3959 uint32_t bits = type2aelembytes(bt) * 8;
3960 uint32_t con = (unsigned)$shift$$constant & (bits - 1);
3961 if (con == 0) {
3962 return;
3963 }
3964 __ vsetvli_helper(bt, Matcher::vector_length(this));
3965 con = bits - con;
3966 __ vror_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con);
3967 %}
3968 ins_pipe(pipe_slow);
3969 %}
3970
3971 // Rotate left - masked
3972
3973 instruct vrotate_left_masked(vReg dst_src, vReg shift, vRegMask_V0 v0) %{
3974 match(Set dst_src (RotateLeftV (Binary dst_src shift) v0));
3975 format %{ "vrotate_left_masked $dst_src, $dst_src, $shift, $v0\t" %}
3976 ins_encode %{
3977 BasicType bt = Matcher::vector_element_basic_type(this);
3978 __ vsetvli_helper(bt, Matcher::vector_length(this));
3979 __ vrol_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3980 as_VectorRegister($shift$$reg), Assembler::v0_t);
3981 %}
3982 ins_pipe(pipe_slow);
3983 %}
3984
3985 // Only the low log2(SEW) bits of shift value are used, all other bits are ignored.
3986 instruct vrotate_left_vx_masked(vReg dst_src, iRegIorL2I shift, vRegMask_V0 v0) %{
3987 match(Set dst_src (RotateLeftV (Binary dst_src (Replicate shift)) v0));
3988 format %{ "vrotate_left_vx_masked $dst_src, $dst_src, $shift, $v0\t" %}
3989 ins_encode %{
3990 BasicType bt = Matcher::vector_element_basic_type(this);
3991 __ vsetvli_helper(bt, Matcher::vector_length(this));
3992 __ vrol_vx(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
3993 as_Register($shift$$reg), Assembler::v0_t);
3994 %}
3995 ins_pipe(pipe_slow);
3996 %}
3997
3998 instruct vrotate_left_vi_masked(vReg dst_src, immI shift, vRegMask_V0 v0) %{
3999 match(Set dst_src (RotateLeftV (Binary dst_src shift) v0));
4000 format %{ "vrotate_left_vi_masked $dst_src, $dst_src, $shift, $v0\t" %}
4001 ins_encode %{
4002 BasicType bt = Matcher::vector_element_basic_type(this);
4003 uint32_t bits = type2aelembytes(bt) * 8;
4004 uint32_t con = (unsigned)$shift$$constant & (bits - 1);
4005 if (con == 0) {
4006 return;
4007 }
4008 __ vsetvli_helper(bt, Matcher::vector_length(this));
4009 con = bits - con;
4010 __ vror_vi(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
4011 con, Assembler::v0_t);
4012 %}
4013 ins_pipe(pipe_slow);
4014 %}
4015
4016 // vector sqrt
4017
4018 instruct vsqrt_hfp(vReg dst, vReg src) %{
4019 match(Set dst (SqrtVHF src));
4020 format %{ "vsqrt_hfp $dst, $src" %}
4021 ins_encode %{
4022 assert(UseZvfh, "must");
4023 assert(Matcher::vector_element_basic_type(this) == T_SHORT, "must");
4024 __ vsetvli_helper(T_SHORT, Matcher::vector_length(this));
4025 __ vfsqrt_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg));
4026 %}
4027 ins_pipe(pipe_slow);
4028 %}
4029
4030 instruct vsqrt_fp(vReg dst, vReg src) %{
4031 match(Set dst (SqrtVF src));
4032 match(Set dst (SqrtVD src));
4033 format %{ "vsqrt_fp $dst, $src" %}
4034 ins_encode %{
4035 BasicType bt = Matcher::vector_element_basic_type(this);
4036 __ vsetvli_helper(bt, Matcher::vector_length(this));
4037 __ vfsqrt_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg));
4038 %}
4039 ins_pipe(pipe_slow);
4040 %}
4041
4042 // vector sqrt - predicated
4043
4044 instruct vsqrt_fp_masked(vReg dst_src, vRegMask_V0 v0) %{
4045 match(Set dst_src (SqrtVF dst_src v0));
4046 match(Set dst_src (SqrtVD dst_src v0));
4047 format %{ "vsqrt_fp_masked $dst_src, $dst_src, $v0" %}
4048 ins_encode %{
4049 BasicType bt = Matcher::vector_element_basic_type(this);
4050 __ vsetvli_helper(bt, Matcher::vector_length(this));
4051 __ vfsqrt_v(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
4052 Assembler::v0_t);
4053 %}
4054 ins_pipe(pipe_slow);
4055 %}
4056
4057 instruct vstring_equalsL(iRegP_R11 str1, iRegP_R13 str2, iRegI_R14 cnt,
4058 iRegI_R10 result, vReg_V2 v2,
4059 vReg_V3 v3, vReg_V4 v4, vReg_V5 v5, rFlagsReg cr)
4060 %{
4061 predicate(UseRVV && ((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
4062 match(Set result (StrEquals (Binary str1 str2) cnt));
4063 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, TEMP v2, TEMP v3, TEMP v4, TEMP v5, KILL cr);
4064
4065 format %{ "String Equals $str1, $str2, $cnt -> $result\t#@string_equalsL" %}
4066 ins_encode %{
4067 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
4068 __ string_equals_v($str1$$Register, $str2$$Register,
4069 $result$$Register, $cnt$$Register);
4070 %}
4071 ins_pipe(pipe_class_memory);
4072 %}
4073
4074 instruct varray_equalsB(iRegP_R11 ary1, iRegP_R12 ary2, iRegI_R10 result,
4075 vReg_V2 v2, vReg_V3 v3, vReg_V4 v4, vReg_V5 v5, iRegP_R28 tmp, rFlagsReg cr)
4076 %{
4077 predicate(UseRVV && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
4078 match(Set result (AryEq ary1 ary2));
4079 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP v2, TEMP v3, TEMP v4, TEMP v5, KILL cr);
4080
4081 format %{ "Array Equals $ary1, ary2 -> $result\t#@array_equalsB // KILL $tmp" %}
4082 ins_encode %{
4083 __ arrays_equals_v($ary1$$Register, $ary2$$Register,
4084 $result$$Register, $tmp$$Register, 1);
4085 %}
4086 ins_pipe(pipe_class_memory);
4087 %}
4088
4089 instruct varray_equalsC(iRegP_R11 ary1, iRegP_R12 ary2, iRegI_R10 result,
4090 vReg_V2 v2, vReg_V3 v3, vReg_V4 v4, vReg_V5 v5, iRegP_R28 tmp, rFlagsReg cr)
4091 %{
4092 predicate(UseRVV && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
4093 match(Set result (AryEq ary1 ary2));
4094 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP v2, TEMP v3, TEMP v4, TEMP v5, KILL cr);
4095
4096 format %{ "Array Equals $ary1, ary2 -> $result\t#@array_equalsC // KILL $tmp" %}
4097 ins_encode %{
4098 __ arrays_equals_v($ary1$$Register, $ary2$$Register,
4099 $result$$Register, $tmp$$Register, 2);
4100 %}
4101 ins_pipe(pipe_class_memory);
4102 %}
4103
4104 // fast ArraysSupport.vectorizedHashCode
4105 instruct varrays_hashcode(iRegP_R11 ary, iRegI_R12 cnt, iRegI_R10 result, immI basic_type,
4106 vReg_V2 v2, vReg_V3 v3, vReg_V4 v4, vReg_V5 v5,
4107 vReg_V6 v6, vReg_V7 v7, vReg_V8 v8, vReg_V9 v9,
4108 iRegLNoSp tmp1, iRegLNoSp tmp2, iRegLNoSp tmp3,
4109 rFlagsReg cr)
4110 %{
4111 predicate(UseRVV);
4112 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
4113 effect(USE_KILL ary, USE_KILL cnt, USE basic_type,
4114 TEMP v2, TEMP v3, TEMP v4, TEMP v5, TEMP v6, TEMP v7, TEMP v8, TEMP v9,
4115 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
4116
4117 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
4118 ins_encode %{
4119 __ arrays_hashcode_v($ary$$Register, $cnt$$Register, $result$$Register,
4120 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
4121 (BasicType)$basic_type$$constant);
4122 %}
4123 ins_pipe(pipe_class_memory);
4124 %}
4125
4126 instruct vstring_compareU_128b(iRegP_R11 str1, iRegI_R12 cnt1, iRegP_R13 str2, iRegI_R14 cnt2,
4127 iRegI_R10 result, vReg_V4 v4, vReg_V5 v5, vReg_V6 v6, vReg_V7 v7,
4128 vReg_V8 v8, vReg_V9 v9, vReg_V10 v10, vReg_V11 v11,
4129 iRegP_R28 tmp1, iRegL_R29 tmp2)
4130 %{
4131 predicate(UseRVV && ((StrCompNode *)n)->encoding() == StrIntrinsicNode::UU &&
4132 MaxVectorSize == 16);
4133 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
4134 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
4135 TEMP v4, TEMP v5, TEMP v6, TEMP v7, TEMP v8, TEMP v9, TEMP v10, TEMP v11);
4136
4137 format %{ "String Compare $str1, $cnt1, $str2, $cnt2 -> $result\t#@string_compareU" %}
4138 ins_encode %{
4139 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
4140 __ string_compare_v($str1$$Register, $str2$$Register,
4141 $cnt1$$Register, $cnt2$$Register, $result$$Register,
4142 $tmp1$$Register, $tmp2$$Register,
4143 StrIntrinsicNode::UU);
4144 %}
4145 ins_pipe(pipe_class_memory);
4146 %}
4147
4148 instruct vstring_compareU(iRegP_R11 str1, iRegI_R12 cnt1, iRegP_R13 str2, iRegI_R14 cnt2,
4149 iRegI_R10 result, vReg_V2 v2, vReg_V3 v3, vReg_V4 v4, vReg_V5 v5,
4150 iRegP_R28 tmp1, iRegL_R29 tmp2)
4151 %{
4152 predicate(UseRVV && ((StrCompNode *)n)->encoding() == StrIntrinsicNode::UU &&
4153 MaxVectorSize > 16);
4154 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
4155 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
4156 TEMP v2, TEMP v3, TEMP v4, TEMP v5);
4157
4158 format %{ "String Compare $str1, $cnt1, $str2, $cnt2 -> $result\t#@string_compareU" %}
4159 ins_encode %{
4160 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
4161 __ string_compare_v($str1$$Register, $str2$$Register,
4162 $cnt1$$Register, $cnt2$$Register, $result$$Register,
4163 $tmp1$$Register, $tmp2$$Register,
4164 StrIntrinsicNode::UU);
4165 %}
4166 ins_pipe(pipe_class_memory);
4167 %}
4168
4169 instruct vstring_compareL(iRegP_R11 str1, iRegI_R12 cnt1, iRegP_R13 str2, iRegI_R14 cnt2,
4170 iRegI_R10 result, vReg_V2 v2, vReg_V3 v3, vReg_V4 v4, vReg_V5 v5,
4171 iRegP_R28 tmp1, iRegL_R29 tmp2)
4172 %{
4173 predicate(UseRVV && ((StrCompNode *)n)->encoding() == StrIntrinsicNode::LL);
4174 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
4175 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
4176 TEMP v2, TEMP v3, TEMP v4, TEMP v5);
4177
4178 format %{ "String Compare $str1, $cnt1, $str2, $cnt2 -> $result\t#@string_compareL" %}
4179 ins_encode %{
4180 __ string_compare_v($str1$$Register, $str2$$Register,
4181 $cnt1$$Register, $cnt2$$Register, $result$$Register,
4182 $tmp1$$Register, $tmp2$$Register,
4183 StrIntrinsicNode::LL);
4184 %}
4185 ins_pipe(pipe_class_memory);
4186 %}
4187
4188 instruct vstring_compareUL(iRegP_R11 str1, iRegI_R12 cnt1, iRegP_R13 str2, iRegI_R14 cnt2,
4189 iRegI_R10 result, vReg_V4 v4, vReg_V5 v5, vReg_V6 v6, vReg_V7 v7,
4190 vReg_V8 v8, vReg_V9 v9, vReg_V10 v10, vReg_V11 v11,
4191 iRegP_R28 tmp1, iRegL_R29 tmp2)
4192 %{
4193 predicate(UseRVV && ((StrCompNode *)n)->encoding() == StrIntrinsicNode::UL);
4194 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
4195 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
4196 TEMP v4, TEMP v5, TEMP v6, TEMP v7, TEMP v8, TEMP v9, TEMP v10, TEMP v11);
4197
4198 format %{"String Compare $str1, $cnt1, $str2, $cnt2 -> $result\t#@string_compareUL" %}
4199 ins_encode %{
4200 __ string_compare_v($str1$$Register, $str2$$Register,
4201 $cnt1$$Register, $cnt2$$Register, $result$$Register,
4202 $tmp1$$Register, $tmp2$$Register,
4203 StrIntrinsicNode::UL);
4204 %}
4205 ins_pipe(pipe_class_memory);
4206 %}
4207 instruct vstring_compareLU(iRegP_R11 str1, iRegI_R12 cnt1, iRegP_R13 str2, iRegI_R14 cnt2,
4208 iRegI_R10 result, vReg_V4 v4, vReg_V5 v5, vReg_V6 v6, vReg_V7 v7,
4209 vReg_V8 v8, vReg_V9 v9, vReg_V10 v10, vReg_V11 v11,
4210 iRegP_R28 tmp1, iRegL_R29 tmp2)
4211 %{
4212 predicate(UseRVV && ((StrCompNode *)n)->encoding() == StrIntrinsicNode::LU);
4213 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
4214 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
4215 TEMP v4, TEMP v5, TEMP v6, TEMP v7, TEMP v8, TEMP v9, TEMP v10, TEMP v11);
4216
4217 format %{ "String Compare $str1, $cnt1, $str2, $cnt2 -> $result\t#@string_compareLU" %}
4218 ins_encode %{
4219 __ string_compare_v($str1$$Register, $str2$$Register,
4220 $cnt1$$Register, $cnt2$$Register, $result$$Register,
4221 $tmp1$$Register, $tmp2$$Register,
4222 StrIntrinsicNode::LU);
4223 %}
4224 ins_pipe(pipe_class_memory);
4225 %}
4226
4227 // fast byte[] to char[] inflation
4228 instruct vstring_inflate(Universe dummy, iRegP_R10 src, iRegP_R11 dst, iRegI_R12 len,
4229 vReg_V4 v4, vReg_V5 v5, vReg_V6 v6, vReg_V7 v7, iRegLNoSp tmp)
4230 %{
4231 predicate(UseRVV);
4232 match(Set dummy (StrInflatedCopy src (Binary dst len)));
4233 effect(TEMP v4, TEMP v5, TEMP v6, TEMP v7, TEMP tmp, USE_KILL src, USE_KILL dst, USE_KILL len);
4234
4235 format %{ "String Inflate $src,$dst" %}
4236 ins_encode %{
4237 __ byte_array_inflate_v($src$$Register, $dst$$Register, $len$$Register, $tmp$$Register);
4238 %}
4239 ins_pipe(pipe_class_memory);
4240 %}
4241
4242 // encode char[] to byte[] in ISO_8859_1
4243 instruct vencode_iso_array(iRegP_R12 src, iRegP_R11 dst, iRegI_R13 len, iRegI_R10 result,
4244 vReg_V1 v1, vReg_V2 v2, vReg_V3 v3, vRegMask_V0 v0, iRegLNoSp tmp)
4245 %{
4246 predicate(UseRVV && !((EncodeISOArrayNode*)n)->is_ascii());
4247 match(Set result (EncodeISOArray src (Binary dst len)));
4248 effect(TEMP_DEF result, USE_KILL src, USE_KILL dst, USE_KILL len,
4249 TEMP v0, TEMP v1, TEMP v2, TEMP v3, TEMP tmp);
4250
4251 format %{ "Encode ISO array $src, $dst, $len -> $result # KILL $src, $dst, $len, $tmp, V0-V3" %}
4252 ins_encode %{
4253 __ encode_iso_array_v($src$$Register, $dst$$Register, $len$$Register,
4254 $result$$Register, $tmp$$Register, false /* ascii */);
4255 %}
4256 ins_pipe(pipe_class_memory);
4257 %}
4258
4259 instruct vencode_ascii_array(iRegP_R12 src, iRegP_R11 dst, iRegI_R13 len, iRegI_R10 result,
4260 vReg_V1 v1, vReg_V2 v2, vReg_V3 v3, vRegMask_V0 v0, iRegLNoSp tmp)
4261 %{
4262 predicate(UseRVV && ((EncodeISOArrayNode*)n)->is_ascii());
4263 match(Set result (EncodeISOArray src (Binary dst len)));
4264 effect(TEMP_DEF result, USE_KILL src, USE_KILL dst, USE_KILL len,
4265 TEMP v0, TEMP v1, TEMP v2, TEMP v3, TEMP tmp);
4266
4267 format %{ "Encode ASCII array $src, $dst, $len -> $result # KILL $src, $dst, $len, $tmp, V0-V3" %}
4268 ins_encode %{
4269 __ encode_iso_array_v($src$$Register, $dst$$Register, $len$$Register,
4270 $result$$Register, $tmp$$Register, true /* ascii */);
4271 %}
4272 ins_pipe(pipe_class_memory);
4273 %}
4274
4275 // fast char[] to byte[] compression
4276 instruct vstring_compress(iRegP_R12 src, iRegP_R11 dst, iRegI_R13 len, iRegI_R10 result,
4277 vReg_V1 v1, vReg_V2 v2, vReg_V3 v3, vRegMask_V0 v0, iRegLNoSp tmp)
4278 %{
4279 predicate(UseRVV);
4280 match(Set result (StrCompressedCopy src (Binary dst len)));
4281 effect(TEMP_DEF result, USE_KILL src, USE_KILL dst, USE_KILL len,
4282 TEMP v0, TEMP v1, TEMP v2, TEMP v3, TEMP tmp);
4283
4284 format %{ "String Compress $src,$dst -> $result // KILL R11, R12, R13" %}
4285 ins_encode %{
4286 __ char_array_compress_v($src$$Register, $dst$$Register, $len$$Register,
4287 $result$$Register, $tmp$$Register);
4288 %}
4289 ins_pipe(pipe_class_memory);
4290 %}
4291
4292 instruct vcount_positives(iRegP_R11 ary, iRegI_R12 len, iRegI_R10 result,
4293 vReg_V4 v4, vReg_V5 v5, vReg_V6 v6, vReg_V7 v7, iRegLNoSp tmp)
4294 %{
4295 predicate(UseRVV);
4296 match(Set result (CountPositives ary len));
4297 effect(TEMP_DEF result, USE_KILL ary, USE_KILL len, TEMP v4, TEMP v5, TEMP v6, TEMP v7, TEMP tmp);
4298
4299 format %{ "count positives byte[] $ary, $len -> $result" %}
4300 ins_encode %{
4301 __ count_positives_v($ary$$Register, $len$$Register, $result$$Register, $tmp$$Register);
4302 %}
4303
4304 ins_pipe(pipe_class_memory);
4305 %}
4306
4307 instruct vstringU_indexof_char(iRegP_R11 str1, iRegI_R12 cnt1, iRegI_R13 ch,
4308 iRegI_R10 result, iRegINoSp tmp1, iRegINoSp tmp2,
4309 vReg_V4 v4, vReg_V5 v5, vReg_V6 v6, vReg_V7 v7)
4310 %{
4311 predicate(UseRVV && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
4312 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
4313 effect(TEMP_DEF result, USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
4314 TEMP tmp1, TEMP tmp2, TEMP v4, TEMP v5, TEMP v6, TEMP v7);
4315
4316 format %{ "StringUTF16 IndexOf char[] $str1, $cnt1, $ch -> $result" %}
4317
4318 ins_encode %{
4319 __ string_indexof_char_v($str1$$Register, $cnt1$$Register, $ch$$Register,
4320 $result$$Register, $tmp1$$Register, $tmp2$$Register,
4321 false /* isL */);
4322 %}
4323
4324 ins_pipe(pipe_class_memory);
4325 %}
4326
4327 instruct vstringL_indexof_char(iRegP_R11 str1, iRegI_R12 cnt1, iRegI_R13 ch,
4328 iRegI_R10 result, iRegINoSp tmp1, iRegINoSp tmp2,
4329 vReg_V4 v4, vReg_V5 v5, vReg_V6 v6, vReg_V7 v7)
4330 %{
4331 predicate(UseRVV && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
4332 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
4333 effect(TEMP_DEF result, USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
4334 TEMP tmp1, TEMP tmp2, TEMP v4, TEMP v5, TEMP v6, TEMP v7);
4335
4336 format %{ "StringLatin1 IndexOf char[] $str1, $cnt1, $ch -> $result" %}
4337
4338 ins_encode %{
4339 __ string_indexof_char_v($str1$$Register, $cnt1$$Register, $ch$$Register,
4340 $result$$Register, $tmp1$$Register, $tmp2$$Register,
4341 true /* isL */);
4342 %}
4343
4344 ins_pipe(pipe_class_memory);
4345 %}
4346
4347 // clearing of an array
4348 instruct vclearArray_reg_reg(iRegL_R29 cnt, iRegP_R28 base, Universe dummy,
4349 vReg_V4 v4, vReg_V5 v5, vReg_V6 v6, vReg_V7 v7)
4350 %{
4351 predicate(!UseBlockZeroing && UseRVV);
4352 match(Set dummy (ClearArray cnt base));
4353 effect(USE_KILL cnt, USE_KILL base, TEMP v4, TEMP v5, TEMP v6, TEMP v7);
4354
4355 format %{ "ClearArray $cnt, $base\t#@clearArray_reg_reg" %}
4356
4357 ins_encode %{
4358 __ clear_array_v($base$$Register, $cnt$$Register);
4359 %}
4360
4361 ins_pipe(pipe_class_memory);
4362 %}
4363
4364 // Vector Load Const
4365 instruct vloadcon(vReg dst, immI0 src) %{
4366 match(Set dst (VectorLoadConst src));
4367 format %{ "vloadcon $dst\t# generate iota indices" %}
4368 ins_encode %{
4369 BasicType bt = Matcher::vector_element_basic_type(this);
4370 __ vsetvli_helper(bt, Matcher::vector_length(this));
4371 __ vid_v(as_VectorRegister($dst$$reg));
4372 if (is_floating_point_type(bt)) {
4373 __ vfcvt_f_x_v(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg));
4374 }
4375 %}
4376 ins_pipe(pipe_slow);
4377 %}
4378
4379 instruct vmask_gen_I(vRegMask dst, iRegI src) %{
4380 match(Set dst (VectorMaskGen (ConvI2L src)));
4381 format %{ "vmask_gen_I $dst, $src" %}
4382 ins_encode %{
4383 BasicType bt = Matcher::vector_element_basic_type(this);
4384 __ vsetvli_helper(bt, Matcher::vector_length(this));
4385 __ vid_v(as_VectorRegister($dst$$reg));
4386 __ vmsltu_vx(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg), $src$$Register);
4387 %}
4388 ins_pipe(pipe_slow);
4389 %}
4390
4391 instruct vmask_gen_L(vRegMask dst, iRegL src) %{
4392 match(Set dst (VectorMaskGen src));
4393 format %{ "vmask_gen_L $dst, $src" %}
4394 ins_encode %{
4395 BasicType bt = Matcher::vector_element_basic_type(this);
4396 __ vsetvli_helper(bt, Matcher::vector_length(this));
4397 __ vid_v(as_VectorRegister($dst$$reg));
4398 __ vmsltu_vx(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg), $src$$Register);
4399 %}
4400 ins_pipe(pipe_slow);
4401 %}
4402
4403 instruct vmask_gen_imm(vRegMask dst, immL con) %{
4404 predicate(n->in(1)->get_long() <= 16 ||
4405 n->in(1)->get_long() == Matcher::vector_length(n));
4406 match(Set dst (VectorMaskGen con));
4407 format %{ "vmask_gen_imm $dst, $con" %}
4408 ins_encode %{
4409 BasicType bt = Matcher::vector_element_basic_type(this);
4410 __ vsetvli_helper(bt, Matcher::vector_length(this));
4411 if ((uint)($con$$constant) == 0) {
4412 __ vmclr_m(as_VectorRegister($dst$$reg));
4413 } else if ((uint)($con$$constant) == Matcher::vector_length(this)) {
4414 __ vmset_m(as_VectorRegister($dst$$reg));
4415 } else {
4416 assert((uint)($con$$constant) < Matcher::vector_length(this), "unsupported input lane_cnt");
4417 __ vid_v(as_VectorRegister($dst$$reg));
4418 __ vmsleu_vi(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg), (uint)($con$$constant) - 1);
4419 }
4420 %}
4421 ins_pipe(pipe_slow);
4422 %}
4423
4424 instruct vmaskAll_immI(vRegMask dst, immI src) %{
4425 match(Set dst (MaskAll src));
4426 format %{ "vmaskAll_immI $dst, $src" %}
4427 ins_encode %{
4428 BasicType bt = Matcher::vector_element_basic_type(this);
4429 __ vsetvli_helper(bt, Matcher::vector_length(this));
4430 int con = (int)$src$$constant;
4431 if (con == 0) {
4432 __ vmclr_m(as_VectorRegister($dst$$reg));
4433 } else {
4434 assert(con == -1, "invalid constant value for mask");
4435 __ vmset_m(as_VectorRegister($dst$$reg));
4436 }
4437 %}
4438 ins_pipe(pipe_slow);
4439 %}
4440
4441 instruct vmaskAllI(vRegMask dst, iRegIorL2I src) %{
4442 match(Set dst (MaskAll src));
4443 format %{ "vmaskAllI $dst, $src" %}
4444 ins_encode %{
4445 BasicType bt = Matcher::vector_element_basic_type(this);
4446 __ vsetvli_helper(bt, Matcher::vector_length(this));
4447 __ vmv_v_x(as_VectorRegister($dst$$reg), as_Register($src$$reg));
4448 __ vmsne_vx(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg), zr);
4449 %}
4450 ins_pipe(pipe_slow);
4451 %}
4452
4453 instruct vmaskAll_immL(vRegMask dst, immL src) %{
4454 match(Set dst (MaskAll src));
4455 format %{ "vmaskAll_immL $dst, $src" %}
4456 ins_encode %{
4457 BasicType bt = Matcher::vector_element_basic_type(this);
4458 __ vsetvli_helper(bt, Matcher::vector_length(this));
4459 long con = (long)$src$$constant;
4460 if (con == 0) {
4461 __ vmclr_m(as_VectorRegister($dst$$reg));
4462 } else {
4463 assert(con == -1, "invalid constant value for mask");
4464 __ vmset_m(as_VectorRegister($dst$$reg));
4465 }
4466 %}
4467 ins_pipe(pipe_slow);
4468 %}
4469
4470 instruct vmaskAllL(vRegMask dst, iRegL src) %{
4471 match(Set dst (MaskAll src));
4472 format %{ "vmaskAllL $dst, $src" %}
4473 ins_encode %{
4474 BasicType bt = Matcher::vector_element_basic_type(this);
4475 __ vsetvli_helper(bt, Matcher::vector_length(this));
4476 __ vmv_v_x(as_VectorRegister($dst$$reg), as_Register($src$$reg));
4477 __ vmsne_vx(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg), zr);
4478 %}
4479 ins_pipe(pipe_slow);
4480 %}
4481
4482 // ------------------------------ Vector mask basic OPs ------------------------
4483
4484 // vector mask logical ops: and/or/xor
4485
4486 instruct vmask_and(vRegMask dst, vRegMask src1, vRegMask src2) %{
4487 match(Set dst (AndVMask src1 src2));
4488 format %{ "vmask_and $dst, $src1, $src2" %}
4489 ins_encode %{
4490 BasicType bt = Matcher::vector_element_basic_type(this);
4491 __ vsetvli_helper(bt, Matcher::vector_length(this));
4492 __ vmand_mm(as_VectorRegister($dst$$reg),
4493 as_VectorRegister($src1$$reg),
4494 as_VectorRegister($src2$$reg));
4495 %}
4496 ins_pipe(pipe_slow);
4497 %}
4498
4499 instruct vmask_or(vRegMask dst, vRegMask src1, vRegMask src2) %{
4500 match(Set dst (OrVMask src1 src2));
4501 format %{ "vmask_or $dst, $src1, $src2" %}
4502 ins_encode %{
4503 BasicType bt = Matcher::vector_element_basic_type(this);
4504 __ vsetvli_helper(bt, Matcher::vector_length(this));
4505 __ vmor_mm(as_VectorRegister($dst$$reg),
4506 as_VectorRegister($src1$$reg),
4507 as_VectorRegister($src2$$reg));
4508 %}
4509 ins_pipe(pipe_slow);
4510 %}
4511
4512 instruct vmask_xor(vRegMask dst, vRegMask src1, vRegMask src2) %{
4513 match(Set dst (XorVMask src1 src2));
4514 format %{ "vmask_xor $dst, $src1, $src2" %}
4515 ins_encode %{
4516 BasicType bt = Matcher::vector_element_basic_type(this);
4517 __ vsetvli_helper(bt, Matcher::vector_length(this));
4518 __ vmxor_mm(as_VectorRegister($dst$$reg),
4519 as_VectorRegister($src1$$reg),
4520 as_VectorRegister($src2$$reg));
4521 %}
4522 ins_pipe(pipe_slow);
4523 %}
4524
4525 instruct vmaskcast(vRegMask dst_src) %{
4526 match(Set dst_src (VectorMaskCast dst_src));
4527 ins_cost(0);
4528 format %{ "vmaskcast $dst_src, $dst_src\t# do nothing" %}
4529 ins_encode(/* empty encoding */);
4530 ins_pipe(pipe_class_empty);
4531 %}
4532
4533 // vector load/store - predicated
4534
4535 instruct loadV_masked(vReg dst, vmemA mem, vRegMask_V0 v0) %{
4536 match(Set dst (LoadVectorMasked mem v0));
4537 format %{ "loadV_masked $dst, $mem, $v0" %}
4538 ins_encode %{
4539 VectorRegister dst_reg = as_VectorRegister($dst$$reg);
4540 loadStore(masm, false, dst_reg,
4541 Matcher::vector_element_basic_type(this), as_Register($mem$$base),
4542 Matcher::vector_length(this), Assembler::v0_t);
4543 %}
4544 ins_pipe(pipe_slow);
4545 %}
4546
4547 instruct storeV_masked(vReg src, vmemA mem, vRegMask_V0 v0) %{
4548 match(Set mem (StoreVectorMasked mem (Binary src v0)));
4549 format %{ "storeV_masked $mem, $src, $v0" %}
4550 ins_encode %{
4551 VectorRegister src_reg = as_VectorRegister($src$$reg);
4552 loadStore(masm, true, src_reg,
4553 Matcher::vector_element_basic_type(this, $src), as_Register($mem$$base),
4554 Matcher::vector_length(this, $src), Assembler::v0_t);
4555 %}
4556 ins_pipe(pipe_slow);
4557 %}
4558
4559 // ------------------------------ Vector blend ---------------------------------
4560
4561 instruct vblend(vReg dst, vReg src1, vReg src2, vRegMask_V0 v0) %{
4562 match(Set dst (VectorBlend (Binary src1 src2) v0));
4563 format %{ "vblend $dst, $src1, $src2, v0" %}
4564 ins_encode %{
4565 BasicType bt = Matcher::vector_element_basic_type(this);
4566 __ vsetvli_helper(bt, Matcher::vector_length(this));
4567 __ vmerge_vvm(as_VectorRegister($dst$$reg), as_VectorRegister($src1$$reg),
4568 as_VectorRegister($src2$$reg));
4569 %}
4570 ins_pipe(pipe_slow);
4571 %}
4572
4573 // ------------------------------ Vector cast ----------------------------------
4574
4575 // VectorCastB2X, VectorUCastB2X
4576
4577 instruct vcvtBtoX(vReg dst, vReg src) %{
4578 match(Set dst (VectorCastB2X src));
4579 effect(TEMP_DEF dst);
4580 format %{ "vcvtBtoX $dst, $src" %}
4581 ins_encode %{
4582 BasicType bt = Matcher::vector_element_basic_type(this);
4583 if (is_floating_point_type(bt)) {
4584 __ integer_extend_v(as_VectorRegister($dst$$reg), bt == T_FLOAT ? T_INT : T_LONG,
4585 Matcher::vector_length(this), as_VectorRegister($src$$reg), T_BYTE,
4586 true /* is_signed */);
4587 __ vfcvt_f_x_v(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg));
4588 } else {
4589 __ integer_extend_v(as_VectorRegister($dst$$reg), bt,
4590 Matcher::vector_length(this), as_VectorRegister($src$$reg), T_BYTE,
4591 true /* is_signed */);
4592 }
4593 %}
4594 ins_pipe(pipe_slow);
4595 %}
4596
4597 instruct vcvtUBtoX(vReg dst, vReg src) %{
4598 predicate(Matcher::vector_element_basic_type(n) == T_SHORT ||
4599 Matcher::vector_element_basic_type(n) == T_INT ||
4600 Matcher::vector_element_basic_type(n) == T_LONG);
4601 match(Set dst (VectorUCastB2X src));
4602 effect(TEMP_DEF dst);
4603 format %{ "vcvtUBtoX $dst, $src" %}
4604 ins_encode %{
4605 BasicType bt = Matcher::vector_element_basic_type(this);
4606 __ integer_extend_v(as_VectorRegister($dst$$reg), bt,
4607 Matcher::vector_length(this), as_VectorRegister($src$$reg), T_BYTE,
4608 false /* is_signed */);
4609 %}
4610 ins_pipe(pipe_slow);
4611 %}
4612
4613 // VectorCastS2X, VectorUCastS2X
4614
4615 instruct vcvtStoB(vReg dst, vReg src) %{
4616 predicate(Matcher::vector_element_basic_type(n) == T_BYTE);
4617 match(Set dst (VectorCastS2X src));
4618 format %{ "vcvtStoB $dst, $src" %}
4619 ins_encode %{
4620 __ integer_narrow_v(as_VectorRegister($dst$$reg), T_BYTE, Matcher::vector_length(this),
4621 as_VectorRegister($src$$reg), T_SHORT);
4622 %}
4623 ins_pipe(pipe_slow);
4624 %}
4625
4626 instruct vcvtStoX(vReg dst, vReg src) %{
4627 predicate(Matcher::vector_element_basic_type(n) == T_INT ||
4628 Matcher::vector_element_basic_type(n) == T_LONG);
4629 match(Set dst (VectorCastS2X src));
4630 effect(TEMP_DEF dst);
4631 format %{ "vcvtStoX $dst, $src" %}
4632 ins_encode %{
4633 __ integer_extend_v(as_VectorRegister($dst$$reg), Matcher::vector_element_basic_type(this),
4634 Matcher::vector_length(this), as_VectorRegister($src$$reg), T_SHORT,
4635 true /* is_signed */);
4636 %}
4637 ins_pipe(pipe_slow);
4638 %}
4639
4640 instruct vcvtStoX_fp(vReg dst, vReg src) %{
4641 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT ||
4642 Matcher::vector_element_basic_type(n) == T_DOUBLE);
4643 match(Set dst (VectorCastS2X src));
4644 effect(TEMP_DEF dst);
4645 format %{ "vcvtStoX_fp $dst, $src" %}
4646 ins_encode %{
4647 BasicType bt = Matcher::vector_element_basic_type(this);
4648 __ integer_extend_v(as_VectorRegister($dst$$reg), (bt == T_FLOAT ? T_INT : T_LONG),
4649 Matcher::vector_length(this), as_VectorRegister($src$$reg), T_SHORT,
4650 true /* is_signed */);
4651 __ vsetvli_helper(bt, Matcher::vector_length(this));
4652 __ vfcvt_f_x_v(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg));
4653 %}
4654 ins_pipe(pipe_slow);
4655 %}
4656
4657
4658 instruct vcvtUStoX(vReg dst, vReg src) %{
4659 predicate(Matcher::vector_element_basic_type(n) == T_INT ||
4660 Matcher::vector_element_basic_type(n) == T_LONG);
4661 match(Set dst (VectorUCastS2X src));
4662 effect(TEMP_DEF dst);
4663 format %{ "vcvtUStoX $dst, $src" %}
4664 ins_encode %{
4665 __ integer_extend_v(as_VectorRegister($dst$$reg), Matcher::vector_element_basic_type(this),
4666 Matcher::vector_length(this), as_VectorRegister($src$$reg), T_SHORT,
4667 false /* is_signed */);
4668 %}
4669 ins_pipe(pipe_slow);
4670 %}
4671
4672 // VectorCastI2X, VectorUCastI2X
4673
4674 instruct vcvtItoX_narrow(vReg dst, vReg src) %{
4675 predicate((Matcher::vector_element_basic_type(n) == T_BYTE ||
4676 Matcher::vector_element_basic_type(n) == T_SHORT));
4677 match(Set dst (VectorCastI2X src));
4678 format %{ "vcvtItoX_narrow $dst, $src" %}
4679 ins_encode %{
4680 BasicType bt = Matcher::vector_element_basic_type(this);
4681 __ integer_narrow_v(as_VectorRegister($dst$$reg), bt, Matcher::vector_length(this),
4682 as_VectorRegister($src$$reg), T_INT);
4683 %}
4684 ins_pipe(pipe_slow);
4685 %}
4686
4687 instruct vcvtItoL(vReg dst, vReg src) %{
4688 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
4689 match(Set dst (VectorCastI2X src));
4690 effect(TEMP_DEF dst);
4691 format %{ "vcvtItoL $dst, $src" %}
4692 ins_encode %{
4693 __ integer_extend_v(as_VectorRegister($dst$$reg), T_LONG,
4694 Matcher::vector_length(this), as_VectorRegister($src$$reg), T_INT,
4695 true /* is_signed */);
4696 %}
4697 ins_pipe(pipe_slow);
4698 %}
4699
4700 instruct vcvtUItoL(vReg dst, vReg src) %{
4701 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
4702 match(Set dst (VectorUCastI2X src));
4703 effect(TEMP_DEF dst);
4704 format %{ "vcvtUItoL $dst, $src" %}
4705 ins_encode %{
4706 __ integer_extend_v(as_VectorRegister($dst$$reg), T_LONG,
4707 Matcher::vector_length(this), as_VectorRegister($src$$reg), T_INT,
4708 false /* is_signed */);
4709 %}
4710 ins_pipe(pipe_slow);
4711 %}
4712
4713 instruct vcvtItoF(vReg dst, vReg src) %{
4714 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
4715 match(Set dst (VectorCastI2X src));
4716 format %{ "vcvtItoF $dst, $src" %}
4717 ins_encode %{
4718 __ vsetvli_helper(T_FLOAT, Matcher::vector_length(this));
4719 __ vfcvt_f_x_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg));
4720 %}
4721 ins_pipe(pipe_slow);
4722 %}
4723
4724 instruct vcvtItoD(vReg dst, vReg src) %{
4725 predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
4726 match(Set dst (VectorCastI2X src));
4727 effect(TEMP_DEF dst);
4728 format %{ "vcvtItoD $dst, $src" %}
4729 ins_encode %{
4730 __ vsetvli_helper(T_INT, Matcher::vector_length(this), Assembler::mf2);
4731 __ vfwcvt_f_x_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg));
4732 %}
4733 ins_pipe(pipe_slow);
4734 %}
4735
4736 // VectorCastL2X
4737
4738 instruct vcvtLtoI(vReg dst, vReg src) %{
4739 predicate(Matcher::vector_element_basic_type(n) == T_BYTE ||
4740 Matcher::vector_element_basic_type(n) == T_SHORT ||
4741 Matcher::vector_element_basic_type(n) == T_INT);
4742 match(Set dst (VectorCastL2X src));
4743 format %{ "vcvtLtoI $dst, $src" %}
4744 ins_encode %{
4745 BasicType bt = Matcher::vector_element_basic_type(this);
4746 __ integer_narrow_v(as_VectorRegister($dst$$reg), bt, Matcher::vector_length(this),
4747 as_VectorRegister($src$$reg), T_LONG);
4748 %}
4749 ins_pipe(pipe_slow);
4750 %}
4751
4752 instruct vcvtLtoF(vReg dst, vReg src) %{
4753 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
4754 match(Set dst (VectorCastL2X src));
4755 format %{ "vcvtLtoF $dst, $src" %}
4756 ins_encode %{
4757 __ vsetvli_helper(T_FLOAT, Matcher::vector_length(this), Assembler::mf2);
4758 __ vfncvt_f_x_w(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg));
4759 %}
4760 ins_pipe(pipe_slow);
4761 %}
4762
4763 instruct vcvtLtoD(vReg dst, vReg src) %{
4764 predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
4765 match(Set dst (VectorCastL2X src));
4766 format %{ "vcvtLtoD $dst, $src" %}
4767 ins_encode %{
4768 __ vsetvli_helper(T_DOUBLE, Matcher::vector_length(this));
4769 __ vfcvt_f_x_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg));
4770 %}
4771 ins_pipe(pipe_slow);
4772 %}
4773
4774 // VectorCastF2X
4775
4776 instruct vcvtFtoX_narrow(vReg dst, vReg src, vRegMask_V0 v0) %{
4777 predicate(Matcher::vector_element_basic_type(n) == T_BYTE ||
4778 Matcher::vector_element_basic_type(n) == T_SHORT);
4779 match(Set dst (VectorCastF2X src));
4780 effect(TEMP_DEF dst, TEMP v0);
4781 format %{ "vcvtFtoX_narrow $dst, $src" %}
4782 ins_encode %{
4783 __ vsetvli_helper(T_FLOAT, Matcher::vector_length(this));
4784 __ vfcvt_rtz_x_f_v_safe(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg));
4785 BasicType bt = Matcher::vector_element_basic_type(this);
4786 __ integer_narrow_v(as_VectorRegister($dst$$reg), bt, Matcher::vector_length(this),
4787 as_VectorRegister($dst$$reg), T_INT);
4788 %}
4789 ins_pipe(pipe_slow);
4790 %}
4791
4792 instruct vcvtFtoI(vReg dst, vReg src, vRegMask_V0 v0) %{
4793 predicate(Matcher::vector_element_basic_type(n) == T_INT);
4794 match(Set dst (VectorCastF2X src));
4795 effect(TEMP_DEF dst, TEMP v0);
4796 format %{ "vcvtFtoI $dst, $src" %}
4797 ins_encode %{
4798 __ vsetvli_helper(T_FLOAT, Matcher::vector_length(this));
4799 __ vfcvt_rtz_x_f_v_safe(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg));
4800 %}
4801 ins_pipe(pipe_slow);
4802 %}
4803
4804 instruct vcvtFtoL(vReg dst, vReg src, vRegMask_V0 v0) %{
4805 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
4806 match(Set dst (VectorCastF2X src));
4807 effect(TEMP_DEF dst, TEMP v0);
4808 format %{ "vcvtFtoL $dst, $src" %}
4809 ins_encode %{
4810 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
4811 __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg));
4812 __ vsetvli_helper(T_FLOAT, Matcher::vector_length(this), Assembler::mf2);
4813 __ vmfeq_vv(as_VectorRegister($v0$$reg), as_VectorRegister($src$$reg), as_VectorRegister($src$$reg));
4814 __ vfwcvt_rtz_x_f_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), Assembler::v0_t);
4815 %}
4816 ins_pipe(pipe_slow);
4817 %}
4818
4819 instruct vcvtFtoD(vReg dst, vReg src) %{
4820 predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
4821 match(Set dst (VectorCastF2X src));
4822 effect(TEMP_DEF dst);
4823 format %{ "vcvtFtoD $dst, $src" %}
4824 ins_encode %{
4825 __ vsetvli_helper(T_FLOAT, Matcher::vector_length(this), Assembler::mf2);
4826 __ vfwcvt_f_f_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg));
4827 %}
4828 ins_pipe(pipe_slow);
4829 %}
4830
4831 // VectorCastD2X
4832
4833 instruct vcvtDtoX_narrow(vReg dst, vReg src, vRegMask_V0 v0) %{
4834 predicate(Matcher::vector_element_basic_type(n) == T_BYTE ||
4835 Matcher::vector_element_basic_type(n) == T_SHORT ||
4836 Matcher::vector_element_basic_type(n) == T_INT);
4837 match(Set dst (VectorCastD2X src));
4838 effect(TEMP_DEF dst, TEMP v0);
4839 format %{ "vcvtDtoX_narrow $dst, $src" %}
4840 ins_encode %{
4841 __ vsetvli_helper(T_DOUBLE, Matcher::vector_length(this));
4842 __ vmfeq_vv(as_VectorRegister($v0$$reg), as_VectorRegister($src$$reg), as_VectorRegister($src$$reg));
4843 __ vsetvli_helper(T_INT, Matcher::vector_length(this), Assembler::mf2);
4844 __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg));
4845 __ vfncvt_rtz_x_f_w(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), Assembler::v0_t);
4846 BasicType bt = Matcher::vector_element_basic_type(this);
4847 if (bt == T_BYTE || bt == T_SHORT) {
4848 __ integer_narrow_v(as_VectorRegister($dst$$reg), bt, Matcher::vector_length(this),
4849 as_VectorRegister($dst$$reg), T_INT);
4850 }
4851 %}
4852 ins_pipe(pipe_slow);
4853 %}
4854
4855 instruct vcvtDtoL(vReg dst, vReg src, vRegMask_V0 v0) %{
4856 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
4857 match(Set dst (VectorCastD2X src));
4858 effect(TEMP_DEF dst, TEMP v0);
4859 format %{ "vcvtDtoL $dst, $src" %}
4860 ins_encode %{
4861 __ vsetvli_helper(T_LONG, Matcher::vector_length(this));
4862 __ vfcvt_rtz_x_f_v_safe(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg));
4863 %}
4864 ins_pipe(pipe_slow);
4865 %}
4866
4867 instruct vcvtDtoF(vReg dst, vReg src) %{
4868 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
4869 match(Set dst (VectorCastD2X src));
4870 format %{ "vcvtDtoF $dst, $src" %}
4871 ins_encode %{
4872 __ vsetvli_helper(T_FLOAT, Matcher::vector_length(this), Assembler::mf2);
4873 __ vfncvt_f_f_w(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg));
4874 %}
4875 ins_pipe(pipe_slow);
4876 %}
4877
4878 // ------------------------------ Vector reinterpret ---------------------------
4879
4880 instruct reinterpret(vReg dst_src) %{
4881 predicate(Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
4882 match(Set dst_src (VectorReinterpret dst_src));
4883 ins_cost(0);
4884 format %{ "# reinterpret $dst_src, $dst_src\t# do nothing" %}
4885 ins_encode %{
4886 // empty
4887 %}
4888 ins_pipe(pipe_class_empty);
4889 %}
4890
4891 instruct reinterpretResize(vReg dst, vReg src) %{
4892 predicate(Matcher::vector_length_in_bytes(n) != Matcher::vector_length_in_bytes(n->in(1)));
4893 match(Set dst (VectorReinterpret src));
4894 effect(TEMP_DEF dst);
4895 format %{ "reinterpretResize $dst, $src" %}
4896 ins_encode %{
4897 uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
4898 uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
4899 uint length_in_bytes_resize = length_in_bytes_src < length_in_bytes_dst ?
4900 length_in_bytes_src : length_in_bytes_dst;
4901 assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
4902 "invalid vector length");
4903 BasicType bt = Matcher::vector_element_basic_type(this);
4904 __ vsetvli_helper(bt, Matcher::vector_length(this));
4905 __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg));
4906 __ vsetvli_helper(T_BYTE, length_in_bytes_resize);
4907 __ vmv_v_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg));
4908 %}
4909 ins_pipe(pipe_slow);
4910 %}
4911
4912 // vector mask reinterpret
4913
4914 instruct vmask_reinterpret_same_esize(vRegMask dst_src) %{
4915 predicate(Matcher::vector_length(n) == Matcher::vector_length(n->in(1)) &&
4916 Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
4917 match(Set dst_src (VectorReinterpret dst_src));
4918 ins_cost(0);
4919 format %{ "vmask_reinterpret_same_esize $dst_src, $dst_src\t# do nothing" %}
4920 ins_encode(/* empty encoding */);
4921 ins_pipe(pipe_class_empty);
4922 %}
4923
4924 instruct vmask_reinterpret_diff_esize(vRegMask dst, vRegMask_V0 src, vReg tmp) %{
4925 predicate(Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
4926 Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
4927 match(Set dst (VectorReinterpret src));
4928 effect(TEMP tmp);
4929 format %{ "vmask_reinterpret_diff_esize $dst, $src\t# KILL $tmp" %}
4930 ins_encode %{
4931 BasicType from_bt = Matcher::vector_element_basic_type(this, $src);
4932 __ vsetvli_helper(from_bt, Matcher::vector_length(this, $src));
4933 __ vxor_vv(as_VectorRegister($tmp$$reg), as_VectorRegister($tmp$$reg), as_VectorRegister($tmp$$reg));
4934 __ vmerge_vim(as_VectorRegister($tmp$$reg), as_VectorRegister($tmp$$reg), -1);
4935 BasicType to_bt = Matcher::vector_element_basic_type(this);
4936 __ vsetvli_helper(to_bt, Matcher::vector_length(this));
4937 __ vmseq_vi(as_VectorRegister($dst$$reg), as_VectorRegister($tmp$$reg), -1);
4938 %}
4939 ins_pipe(pipe_slow);
4940 %}
4941
4942 // ------------------------------ Vector selectFrom -----------------------------
4943
4944 instruct select_from_two_vectors(vReg dst, vReg src1, vReg src2, vReg index, vRegMask_V0 v0, vReg tmp) %{
4945 match(Set dst (SelectFromTwoVector (Binary index src1) src2));
4946 effect(TEMP_DEF dst, TEMP v0, TEMP tmp);
4947 format %{ "select_from_two_vectors $dst, $src1, $src2, $index" %}
4948 ins_encode %{
4949 BasicType bt = Matcher::vector_element_basic_type(this);
4950 __ vsetvli_helper(bt, Matcher::vector_length(this));
4951 __ vrgather_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src1$$reg),
4952 as_VectorRegister($index$$reg));
4953 bool use_imm = __ is_simm5(Matcher::vector_length(this) - 1);
4954 if (use_imm) {
4955 __ vmsgtu_vi(v0, as_VectorRegister($index$$reg), Matcher::vector_length(this) - 1);
4956 __ vadd_vi(as_VectorRegister($tmp$$reg), as_VectorRegister($index$$reg),
4957 -Matcher::vector_length(this), Assembler::v0_t);
4958 } else {
4959 __ mv(t0, Matcher::vector_length(this) - 1);
4960 __ vmsgtu_vx(v0, as_VectorRegister($index$$reg), t0);
4961 __ mv(t0, -Matcher::vector_length(this));
4962 __ vadd_vx(as_VectorRegister($tmp$$reg), as_VectorRegister($index$$reg), t0, Assembler::v0_t);
4963 }
4964 __ vrgather_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src2$$reg),
4965 as_VectorRegister($tmp$$reg), Assembler::v0_t);
4966 %}
4967 ins_pipe(pipe_slow);
4968 %}
4969
4970 // ------------------------------ Vector rearrange -----------------------------
4971
4972 instruct rearrange(vReg dst, vReg src, vReg shuffle) %{
4973 match(Set dst (VectorRearrange src shuffle));
4974 effect(TEMP_DEF dst);
4975 format %{ "rearrange $dst, $src, $shuffle" %}
4976 ins_encode %{
4977 BasicType bt = Matcher::vector_element_basic_type(this);
4978 __ vsetvli_helper(bt, Matcher::vector_length(this));
4979 __ vrgather_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
4980 as_VectorRegister($shuffle$$reg));
4981 %}
4982 ins_pipe(pipe_slow);
4983 %}
4984
4985 instruct rearrange_masked(vReg dst, vReg src, vReg shuffle, vRegMask_V0 v0) %{
4986 match(Set dst (VectorRearrange (Binary src shuffle) v0));
4987 effect(TEMP_DEF dst);
4988 format %{ "rearrange_masked $dst, $src, $shuffle, $v0" %}
4989 ins_encode %{
4990 BasicType bt = Matcher::vector_element_basic_type(this);
4991 __ vsetvli_helper(bt, Matcher::vector_length(this));
4992 __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg),
4993 as_VectorRegister($dst$$reg));
4994 __ vrgather_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
4995 as_VectorRegister($shuffle$$reg), Assembler::v0_t);
4996 %}
4997 ins_pipe(pipe_slow);
4998 %}
4999
5000 // ------------------------------ Vector extract ---------------------------------
5001
5002 instruct extract(iRegINoSp dst, vReg src, immI idx, vReg tmp)
5003 %{
5004 match(Set dst (ExtractB src idx));
5005 match(Set dst (ExtractS src idx));
5006 match(Set dst (ExtractI src idx));
5007 effect(TEMP tmp);
5008 format %{ "extract $dst, $src, $idx\t# KILL $tmp" %}
5009 ins_encode %{
5010 __ extract_v($dst$$Register, as_VectorRegister($src$$reg),
5011 Matcher::vector_element_basic_type(this, $src), (int)($idx$$constant),
5012 as_VectorRegister($tmp$$reg));
5013 %}
5014 ins_pipe(pipe_slow);
5015 %}
5016
5017 instruct extractL(iRegLNoSp dst, vReg src, immI idx, vReg tmp)
5018 %{
5019 match(Set dst (ExtractL src idx));
5020 effect(TEMP tmp);
5021 format %{ "extractL $dst, $src, $idx\t# KILL $tmp" %}
5022 ins_encode %{
5023 __ extract_v($dst$$Register, as_VectorRegister($src$$reg), T_LONG,
5024 (int)($idx$$constant), as_VectorRegister($tmp$$reg));
5025 %}
5026 ins_pipe(pipe_slow);
5027 %}
5028
5029
5030 instruct extractF(fRegF dst, vReg src, immI idx, vReg tmp)
5031 %{
5032 match(Set dst (ExtractF src idx));
5033 effect(TEMP tmp);
5034 format %{ "extractF $dst, $src, $idx\t# KILL $tmp" %}
5035 ins_encode %{
5036 __ extract_fp_v($dst$$FloatRegister, as_VectorRegister($src$$reg), T_FLOAT,
5037 (int)($idx$$constant), as_VectorRegister($tmp$$reg));
5038 %}
5039 ins_pipe(pipe_slow);
5040 %}
5041
5042 instruct extractD(fRegD dst, vReg src, immI idx, vReg tmp)
5043 %{
5044 match(Set dst (ExtractD src idx));
5045 effect(TEMP tmp);
5046 format %{ "extractD $dst, $src, $idx\t# KILL $tmp" %}
5047 ins_encode %{
5048 __ extract_fp_v($dst$$FloatRegister, as_VectorRegister($src$$reg), T_DOUBLE,
5049 (int)($idx$$constant), as_VectorRegister($tmp$$reg));
5050 %}
5051 ins_pipe(pipe_slow);
5052 %}
5053
5054 // ------------------------------ Compress/Expand Operations -------------------
5055
5056 instruct mcompress(vRegMask dst, vRegMask src, vReg tmp) %{
5057 match(Set dst (CompressM src));
5058 effect(TEMP tmp);
5059 format %{ "mcompress $dst, $src\t# KILL $tmp" %}
5060 ins_encode %{
5061 BasicType bt = Matcher::vector_element_basic_type(this);
5062 __ vsetvli_helper(bt, Matcher::vector_length(this));
5063 __ vid_v(as_VectorRegister($tmp$$reg));
5064 __ vcpop_m(t0, as_VectorRegister($src$$reg));
5065 __ vmsltu_vx(as_VectorRegister($dst$$reg), as_VectorRegister($tmp$$reg), t0);
5066 %}
5067 ins_pipe(pipe_slow);
5068 %}
5069
5070 instruct vcompress(vReg dst, vReg src, vRegMask_V0 v0) %{
5071 match(Set dst (CompressV src v0));
5072 effect(TEMP_DEF dst);
5073 format %{ "vcompress $dst, $src, $v0" %}
5074 ins_encode %{
5075 BasicType bt = Matcher::vector_element_basic_type(this);
5076 __ vsetvli_helper(bt, Matcher::vector_length(this));
5077 __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg),
5078 as_VectorRegister($dst$$reg));
5079 __ vcompress_vm(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
5080 as_VectorRegister($v0$$reg));
5081 %}
5082 ins_pipe(pipe_slow);
5083 %}
5084
5085 instruct vexpand(vReg dst, vReg src, vRegMask_V0 v0, vReg tmp) %{
5086 match(Set dst (ExpandV src v0));
5087 effect(TEMP_DEF dst, TEMP tmp);
5088 format %{ "vexpand $dst, $src, $v0\t# KILL $tmp" %}
5089 ins_encode %{
5090 BasicType bt = Matcher::vector_element_basic_type(this);
5091 __ vsetvli_helper(bt, Matcher::vector_length(this));
5092 __ viota_m(as_VectorRegister($tmp$$reg), as_VectorRegister($v0$$reg));
5093 __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg),
5094 as_VectorRegister($dst$$reg));
5095 __ vrgather_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
5096 as_VectorRegister($tmp$$reg), Assembler::v0_t);
5097 %}
5098 ins_pipe(pipe_slow);
5099 %}
5100
5101 // ------------------------------ Vector signum --------------------------------
5102
5103 // Vector Math.signum
5104
5105 instruct vsignum_reg(vReg dst, vReg zero, vReg one, vRegMask_V0 v0) %{
5106 match(Set dst (SignumVF dst (Binary zero one)));
5107 match(Set dst (SignumVD dst (Binary zero one)));
5108 effect(TEMP_DEF dst, TEMP v0);
5109 format %{ "vsignum $dst, $dst\t" %}
5110 ins_encode %{
5111 BasicType bt = Matcher::vector_element_basic_type(this);
5112 __ signum_fp_v(as_VectorRegister($dst$$reg), as_VectorRegister($one$$reg),
5113 bt, Matcher::vector_length(this));
5114 %}
5115 ins_pipe(pipe_slow);
5116 %}
5117
5118 // ---------------- Round float/double Vector Operations ----------------
5119
5120 instruct vround_f(vReg dst, vReg src, fRegF tmp, vRegMask_V0 v0) %{
5121 match(Set dst (RoundVF src));
5122 effect(TEMP_DEF dst, TEMP tmp, TEMP v0);
5123 format %{ "java_round_float_v $dst, $src\t" %}
5124 ins_encode %{
5125 BasicType bt = Matcher::vector_element_basic_type(this);
5126 uint vector_length = Matcher::vector_length(this);
5127 __ java_round_float_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
5128 as_FloatRegister($tmp$$reg), bt, vector_length);
5129 %}
5130 ins_pipe(pipe_slow);
5131 %}
5132
5133 instruct vround_d(vReg dst, vReg src, fRegD tmp, vRegMask_V0 v0) %{
5134 match(Set dst (RoundVD src));
5135 effect(TEMP_DEF dst, TEMP tmp, TEMP v0);
5136 format %{ "java_round_double_v $dst, $src\t" %}
5137 ins_encode %{
5138 BasicType bt = Matcher::vector_element_basic_type(this);
5139 uint vector_length = Matcher::vector_length(this);
5140 __ java_round_double_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
5141 as_FloatRegister($tmp$$reg), bt, vector_length);
5142 %}
5143 ins_pipe(pipe_slow);
5144 %}
5145
5146 // -------------------------------- Reverse Bits Vector Operations ------------------------
5147
5148 instruct vreverse_masked(vReg dst_src, vRegMask_V0 v0) %{
5149 match(Set dst_src (ReverseV dst_src v0));
5150 format %{ "vreverse_masked $dst_src, $dst_src, v0" %}
5151 ins_encode %{
5152 BasicType bt = Matcher::vector_element_basic_type(this);
5153 uint vlen = Matcher::vector_length(this);
5154 __ vsetvli_helper(bt, vlen);
5155 __ vbrev_v(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg), Assembler::v0_t);
5156 %}
5157 ins_pipe(pipe_slow);
5158 %}
5159
5160 instruct vreverse(vReg dst, vReg src) %{
5161 match(Set dst (ReverseV src));
5162 format %{ "vreverse $dst, $src" %}
5163 ins_encode %{
5164 BasicType bt = Matcher::vector_element_basic_type(this);
5165 uint vlen = Matcher::vector_length(this);
5166 __ vsetvli_helper(bt, vlen);
5167 __ vbrev_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg));
5168 %}
5169 ins_pipe(pipe_slow);
5170 %}
5171
5172 // -------------------------------- Reverse Bytes Vector Operations ------------------------
5173
5174 instruct vreverse_bytes_masked(vReg dst_src, vRegMask_V0 v0) %{
5175 match(Set dst_src (ReverseBytesV dst_src v0));
5176 format %{ "vreverse_bytes_masked $dst_src, $dst_src, v0" %}
5177 ins_encode %{
5178 BasicType bt = Matcher::vector_element_basic_type(this);
5179 uint vlen = Matcher::vector_length(this);
5180 __ vsetvli_helper(bt, vlen);
5181 __ vrev8_v(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg), Assembler::v0_t);
5182 %}
5183 ins_pipe(pipe_slow);
5184 %}
5185
5186 instruct vreverse_bytes(vReg dst, vReg src) %{
5187 match(Set dst (ReverseBytesV src));
5188 format %{ "vreverse_bytes $dst, $src" %}
5189 ins_encode %{
5190 BasicType bt = Matcher::vector_element_basic_type(this);
5191 uint vlen = Matcher::vector_length(this);
5192 __ vsetvli_helper(bt, vlen);
5193 __ vrev8_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg));
5194 %}
5195 ins_pipe(pipe_slow);
5196 %}
5197
5198 // ---------------- Convert Half Floating to Floating Vector Operations ----------------
5199
5200 // half precision -> single
5201
5202 instruct vconvHF2F(vReg dst, vReg src, vRegMask_V0 v0) %{
5203 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
5204 match(Set dst (VectorCastHF2F src));
5205 effect(TEMP_DEF dst, TEMP v0);
5206 format %{ "vfwcvt.f.f.v $dst, $src\t# convert half to single precision" %}
5207 ins_encode %{
5208 __ float16_to_float_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
5209 Matcher::vector_length(this));
5210 %}
5211 ins_pipe(pipe_slow);
5212 %}
5213
5214 // single precision -> half
5215
5216 instruct vconvF2HF(vReg dst, vReg src, vReg vtmp, vRegMask_V0 v0, iRegINoSp tmp) %{
5217 predicate(Matcher::vector_element_basic_type(n) == T_SHORT);
5218 match(Set dst (VectorCastF2HF src));
5219 effect(TEMP_DEF dst, TEMP v0, TEMP vtmp, TEMP tmp);
5220 format %{ "vfncvt.f.f.w $dst, $src\t# convert single to half precision" %}
5221 ins_encode %{
5222 __ float_to_float16_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
5223 as_VectorRegister($vtmp$$reg), $tmp$$Register,
5224 Matcher::vector_length(this));
5225 %}
5226 ins_pipe(pipe_slow);
5227 %}
5228
5229
5230 // ------------------------------ Popcount vector ------------------------------
5231
5232 instruct vpopcount_masked(vReg dst_src, vRegMask_V0 v0) %{
5233 match(Set dst_src (PopCountVI dst_src v0));
5234 match(Set dst_src (PopCountVL dst_src v0));
5235 format %{ "vcpop_v $dst_src, $dst_src, $v0\t# vcpop_v with mask" %}
5236 ins_encode %{
5237 BasicType bt = Matcher::vector_element_basic_type(this);
5238 uint vlen = Matcher::vector_length(this);
5239 __ vsetvli_helper(bt, vlen);
5240 __ vcpop_v(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg), Assembler::v0_t);
5241 %}
5242 ins_pipe(pipe_slow);
5243 %}
5244
5245 instruct vpopcount(vReg dst, vReg src) %{
5246 match(Set dst (PopCountVI src));
5247 match(Set dst (PopCountVL src));
5248 format %{ "vcpop_v $dst, $src\t# vcpop_v without mask" %}
5249 ins_encode %{
5250 BasicType bt = Matcher::vector_element_basic_type(this);
5251 uint vlen = Matcher::vector_length(this);
5252 __ vsetvli_helper(bt, vlen);
5253 __ vcpop_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg));
5254 %}
5255 ins_pipe(pipe_slow);
5256 %}
5257
5258 // ------------------------------ CountLeadingZerosV --------------------------
5259
5260 instruct vcountLeadingZeros_masked(vReg dst_src, vRegMask_V0 v0) %{
5261 match(Set dst_src (CountLeadingZerosV dst_src v0));
5262 format %{ "vcount_leading_zeros_masked $dst_src, $dst_src, v0" %}
5263 ins_encode %{
5264 BasicType bt = Matcher::vector_element_basic_type(this);
5265 uint vlen = Matcher::vector_length(this);
5266 __ vsetvli_helper(bt, vlen);
5267 __ vclz_v(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg), Assembler::v0_t);
5268 %}
5269 ins_pipe(pipe_slow);
5270 %}
5271
5272 instruct vcountLeadingZeros(vReg dst, vReg src) %{
5273 match(Set dst (CountLeadingZerosV src));
5274 format %{ "vcount_leading_zeros $dst, $src" %}
5275 ins_encode %{
5276 BasicType bt = Matcher::vector_element_basic_type(this);
5277 uint vlen = Matcher::vector_length(this);
5278 __ vsetvli_helper(bt, vlen);
5279 __ vclz_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg));
5280 %}
5281 ins_pipe(pipe_slow);
5282 %}
5283
5284 // ------------------------------ CountTrailingZerosV --------------------------
5285
5286 instruct vcountTrailingZeros_masked(vReg dst_src, vRegMask_V0 v0) %{
5287 match(Set dst_src (CountTrailingZerosV dst_src v0));
5288 format %{ "vcount_trailing_zeros_masked $dst_src, $dst_src, v0" %}
5289 ins_encode %{
5290 BasicType bt = Matcher::vector_element_basic_type(this);
5291 uint vlen = Matcher::vector_length(this);
5292 __ vsetvli_helper(bt, vlen);
5293 __ vctz_v(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg), Assembler::v0_t);
5294 %}
5295 ins_pipe(pipe_slow);
5296 %}
5297
5298 instruct vcountTrailingZeros(vReg dst, vReg src) %{
5299 match(Set dst (CountTrailingZerosV src));
5300 format %{ "vcount_trailing_zeros $dst, $src" %}
5301 ins_encode %{
5302 BasicType bt = Matcher::vector_element_basic_type(this);
5303 uint vlen = Matcher::vector_length(this);
5304 __ vsetvli_helper(bt, vlen);
5305 __ vctz_v(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg));
5306 %}
5307 ins_pipe(pipe_slow);
5308 %}
5309
5310 // ------------------------------ Vector Load Gather ---------------------------
5311
5312 instruct gather_loadS(vReg dst, indirect mem, vReg idx) %{
5313 predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) == 4);
5314 match(Set dst (LoadVectorGather mem idx));
5315 effect(TEMP_DEF dst);
5316 format %{ "gather_loadS $dst, $mem, $idx" %}
5317 ins_encode %{
5318 BasicType bt = Matcher::vector_element_basic_type(this);
5319 Assembler::SEW sew = Assembler::elemtype_to_sew(bt);
5320 __ vsetvli_helper(bt, Matcher::vector_length(this));
5321 __ vsll_vi(as_VectorRegister($dst$$reg), as_VectorRegister($idx$$reg), (int)sew);
5322 __ vluxei32_v(as_VectorRegister($dst$$reg), as_Register($mem$$base),
5323 as_VectorRegister($dst$$reg));
5324 %}
5325 ins_pipe(pipe_slow);
5326 %}
5327
5328 instruct gather_loadD(vReg dst, indirect mem, vReg idx) %{
5329 predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) == 8);
5330 match(Set dst (LoadVectorGather mem idx));
5331 effect(TEMP_DEF dst);
5332 format %{ "gather_loadD $dst, $mem, $idx" %}
5333 ins_encode %{
5334 BasicType bt = Matcher::vector_element_basic_type(this);
5335 Assembler::SEW sew = Assembler::elemtype_to_sew(bt);
5336 __ vsetvli_helper(bt, Matcher::vector_length(this));
5337 __ vzext_vf2(as_VectorRegister($dst$$reg), as_VectorRegister($idx$$reg));
5338 __ vsll_vi(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg), (int)sew);
5339 __ vluxei64_v(as_VectorRegister($dst$$reg), as_Register($mem$$base),
5340 as_VectorRegister($dst$$reg));
5341 %}
5342 ins_pipe(pipe_slow);
5343 %}
5344
5345 instruct gather_loadS_masked(vReg dst, indirect mem, vReg idx, vRegMask_V0 v0, vReg tmp) %{
5346 predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) == 4);
5347 match(Set dst (LoadVectorGatherMasked mem (Binary idx v0)));
5348 effect(TEMP_DEF dst, TEMP tmp);
5349 format %{ "gather_loadS_masked $dst, $mem, $idx, $v0\t# KILL $tmp" %}
5350 ins_encode %{
5351 BasicType bt = Matcher::vector_element_basic_type(this);
5352 Assembler::SEW sew = Assembler::elemtype_to_sew(bt);
5353 __ vsetvli_helper(bt, Matcher::vector_length(this));
5354 __ vsll_vi(as_VectorRegister($tmp$$reg), as_VectorRegister($idx$$reg), (int)sew);
5355 __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg),
5356 as_VectorRegister($dst$$reg));
5357 __ vluxei32_v(as_VectorRegister($dst$$reg), as_Register($mem$$base),
5358 as_VectorRegister($tmp$$reg), Assembler::v0_t);
5359 %}
5360 ins_pipe(pipe_slow);
5361 %}
5362
5363 instruct gather_loadD_masked(vReg dst, indirect mem, vReg idx, vRegMask_V0 v0, vReg tmp) %{
5364 predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) == 8);
5365 match(Set dst (LoadVectorGatherMasked mem (Binary idx v0)));
5366 effect(TEMP_DEF dst, TEMP tmp);
5367 format %{ "gather_loadD_masked $dst, $mem, $idx, $v0\t# KILL $tmp" %}
5368 ins_encode %{
5369 BasicType bt = Matcher::vector_element_basic_type(this);
5370 Assembler::SEW sew = Assembler::elemtype_to_sew(bt);
5371 __ vsetvli_helper(bt, Matcher::vector_length(this));
5372 __ vzext_vf2(as_VectorRegister($tmp$$reg), as_VectorRegister($idx$$reg));
5373 __ vsll_vi(as_VectorRegister($tmp$$reg), as_VectorRegister($tmp$$reg), (int)sew);
5374 __ vxor_vv(as_VectorRegister($dst$$reg), as_VectorRegister($dst$$reg),
5375 as_VectorRegister($dst$$reg));
5376 __ vluxei64_v(as_VectorRegister($dst$$reg), as_Register($mem$$base),
5377 as_VectorRegister($tmp$$reg), Assembler::v0_t);
5378 %}
5379 ins_pipe(pipe_slow);
5380 %}
5381
5382 // ------------------------------ Vector Store Scatter -------------------------
5383
5384 instruct scatter_storeS(indirect mem, vReg src, vReg idx, vReg tmp) %{
5385 predicate(type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 4);
5386 match(Set mem (StoreVectorScatter mem (Binary src idx)));
5387 effect(TEMP tmp);
5388 format %{ "scatter_storeS $mem, $idx, $src\t# KILL $tmp" %}
5389 ins_encode %{
5390 BasicType bt = Matcher::vector_element_basic_type(this, $src);
5391 Assembler::SEW sew = Assembler::elemtype_to_sew(bt);
5392 __ vsetvli_helper(bt, Matcher::vector_length(this, $src));
5393 __ vsll_vi(as_VectorRegister($tmp$$reg), as_VectorRegister($idx$$reg), (int)sew);
5394 __ vsuxei32_v(as_VectorRegister($src$$reg), as_Register($mem$$base),
5395 as_VectorRegister($tmp$$reg));
5396 %}
5397 ins_pipe(pipe_slow);
5398 %}
5399
5400 instruct scatter_storeD(indirect mem, vReg src, vReg idx, vReg tmp) %{
5401 predicate(type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 8);
5402 match(Set mem (StoreVectorScatter mem (Binary src idx)));
5403 effect(TEMP tmp);
5404 format %{ "scatter_storeD $mem, $idx, $src\t# KILL $tmp" %}
5405 ins_encode %{
5406 BasicType bt = Matcher::vector_element_basic_type(this, $src);
5407 Assembler::SEW sew = Assembler::elemtype_to_sew(bt);
5408 __ vsetvli_helper(bt, Matcher::vector_length(this, $src));
5409 __ vzext_vf2(as_VectorRegister($tmp$$reg), as_VectorRegister($idx$$reg));
5410 __ vsll_vi(as_VectorRegister($tmp$$reg), as_VectorRegister($tmp$$reg), (int)sew);
5411 __ vsuxei64_v(as_VectorRegister($src$$reg), as_Register($mem$$base),
5412 as_VectorRegister($tmp$$reg));
5413 %}
5414 ins_pipe(pipe_slow);
5415 %}
5416
5417 instruct scatter_storeS_masked(indirect mem, vReg src, vReg idx, vRegMask_V0 v0, vReg tmp) %{
5418 predicate(type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 4);
5419 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx v0))));
5420 effect(TEMP tmp);
5421 format %{ "scatter_storeS_masked $mem, $idx, $src, $v0\t# KILL $tmp" %}
5422 ins_encode %{
5423 BasicType bt = Matcher::vector_element_basic_type(this, $src);
5424 Assembler::SEW sew = Assembler::elemtype_to_sew(bt);
5425 __ vsetvli_helper(bt, Matcher::vector_length(this, $src));
5426 __ vsll_vi(as_VectorRegister($tmp$$reg), as_VectorRegister($idx$$reg), (int)sew);
5427 __ vsuxei32_v(as_VectorRegister($src$$reg), as_Register($mem$$base),
5428 as_VectorRegister($tmp$$reg), Assembler::v0_t);
5429 %}
5430 ins_pipe(pipe_slow);
5431 %}
5432
5433 instruct scatter_storeD_masked(indirect mem, vReg src, vReg idx, vRegMask_V0 v0, vReg tmp) %{
5434 predicate(type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 8);
5435 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx v0))));
5436 effect(TEMP tmp);
5437 format %{ "scatter_storeD_masked $mem, $idx, $src, $v0\t# KILL $tmp" %}
5438 ins_encode %{
5439 BasicType bt = Matcher::vector_element_basic_type(this, $src);
5440 Assembler::SEW sew = Assembler::elemtype_to_sew(bt);
5441 __ vsetvli_helper(bt, Matcher::vector_length(this, $src));
5442 __ vzext_vf2(as_VectorRegister($tmp$$reg), as_VectorRegister($idx$$reg));
5443 __ vsll_vi(as_VectorRegister($tmp$$reg), as_VectorRegister($tmp$$reg), (int)sew);
5444 __ vsuxei64_v(as_VectorRegister($src$$reg), as_Register($mem$$base),
5445 as_VectorRegister($tmp$$reg), Assembler::v0_t);
5446 %}
5447 ins_pipe(pipe_slow);
5448 %}
5449
5450 // ------------------------------ Populate Index to a Vector -------------------
5451
5452 instruct populateindex(vReg dst, iRegIorL2I src1, iRegIorL2I src2, vReg tmp) %{
5453 match(Set dst (PopulateIndex src1 src2));
5454 effect(TEMP_DEF dst, TEMP tmp);
5455 format %{ "populateindex $dst, $src1, $src2\t# KILL $tmp" %}
5456 ins_encode %{
5457 BasicType bt = Matcher::vector_element_basic_type(this);
5458 Assembler::SEW sew = Assembler::elemtype_to_sew(bt);
5459 __ vsetvli_helper(bt, Matcher::vector_length(this));
5460 __ vmv_v_x(as_VectorRegister($dst$$reg), as_Register($src1$$reg));
5461 __ vid_v(as_VectorRegister($tmp$$reg));
5462 __ vmacc_vx(as_VectorRegister($dst$$reg), as_Register($src2$$reg), as_VectorRegister($tmp$$reg));
5463 %}
5464 ins_pipe(pipe_slow);
5465 %}
5466
5467 // ------------------------------ Vector insert --------------------------------
5468
5469 // BYTE, SHORT, INT
5470
5471 instruct insert_index_lt32(vReg dst, vReg src, iRegIorL2I val, immI idx, vRegMask_V0 v0) %{
5472 predicate(n->in(2)->get_int() < 32 &&
5473 (Matcher::vector_element_basic_type(n) == T_BYTE ||
5474 Matcher::vector_element_basic_type(n) == T_SHORT ||
5475 Matcher::vector_element_basic_type(n) == T_INT));
5476 match(Set dst (VectorInsert (Binary src val) idx));
5477 effect(TEMP v0);
5478 format %{ "insert_index_lt32 $dst, $src, $val, $idx" %}
5479 ins_encode %{
5480 BasicType bt = Matcher::vector_element_basic_type(this);
5481 __ vsetvli_helper(bt, Matcher::vector_length(this));
5482 __ vid_v(as_VectorRegister($v0$$reg));
5483 __ vadd_vi(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg), -16);
5484 __ vmseq_vi(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg), (int)($idx$$constant) - 16);
5485 __ vmerge_vxm(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), $val$$Register);
5486 %}
5487 ins_pipe(pipe_slow);
5488 %}
5489
5490 instruct insert_index(vReg dst, vReg src, iRegIorL2I val, iRegIorL2I idx, vReg tmp, vRegMask_V0 v0) %{
5491 predicate(n->in(2)->get_int() >= 32 &&
5492 (Matcher::vector_element_basic_type(n) == T_BYTE ||
5493 Matcher::vector_element_basic_type(n) == T_SHORT ||
5494 Matcher::vector_element_basic_type(n) == T_INT));
5495 match(Set dst (VectorInsert (Binary src val) idx));
5496 effect(TEMP tmp, TEMP v0);
5497 format %{ "insert_index $dst, $src, $val, $idx\t# KILL $tmp" %}
5498 ins_encode %{
5499 BasicType bt = Matcher::vector_element_basic_type(this);
5500 __ vsetvli_helper(bt, Matcher::vector_length(this));
5501 __ vid_v(as_VectorRegister($v0$$reg));
5502 __ vmv_v_x(as_VectorRegister($tmp$$reg), $idx$$Register);
5503 __ vmseq_vv(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg), as_VectorRegister($tmp$$reg));
5504 __ vmerge_vxm(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), $val$$Register);
5505 %}
5506 ins_pipe(pipe_slow);
5507 %}
5508
5509 // LONG
5510
5511 instruct insertL_index_lt32(vReg dst, vReg src, iRegL val, immI idx, vRegMask_V0 v0) %{
5512 predicate(n->in(2)->get_int() < 32 &&
5513 (Matcher::vector_element_basic_type(n) == T_LONG));
5514 match(Set dst (VectorInsert (Binary src val) idx));
5515 effect(TEMP v0);
5516 format %{ "insertL_index_lt32 $dst, $src, $val, $idx" %}
5517 ins_encode %{
5518 BasicType bt = Matcher::vector_element_basic_type(this);
5519 __ vsetvli_helper(bt, Matcher::vector_length(this));
5520 __ vid_v(as_VectorRegister($v0$$reg));
5521 __ vadd_vi(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg), -16);
5522 __ vmseq_vi(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg), (int)($idx$$constant) - 16);
5523 __ vmerge_vxm(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), $val$$Register);
5524 %}
5525 ins_pipe(pipe_slow);
5526 %}
5527
5528 instruct insertL_index(vReg dst, vReg src, iRegL val, iRegIorL2I idx, vReg tmp, vRegMask_V0 v0) %{
5529 predicate(n->in(2)->get_int() >= 32 &&
5530 (Matcher::vector_element_basic_type(n) == T_LONG));
5531 match(Set dst (VectorInsert (Binary src val) idx));
5532 effect(TEMP tmp, TEMP v0);
5533 format %{ "insertL_index $dst, $src, $val, $idx\t# KILL $tmp" %}
5534 ins_encode %{
5535 BasicType bt = Matcher::vector_element_basic_type(this);
5536 __ vsetvli_helper(bt, Matcher::vector_length(this));
5537 __ vid_v(as_VectorRegister($v0$$reg));
5538 __ vmv_v_x(as_VectorRegister($tmp$$reg), $idx$$Register);
5539 __ vmseq_vv(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg), as_VectorRegister($tmp$$reg));
5540 __ vmerge_vxm(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), $val$$Register);
5541 %}
5542 ins_pipe(pipe_slow);
5543 %}
5544
5545 // FLOAT
5546
5547 instruct insertF_index_lt32(vReg dst, vReg src, fRegF val, immI idx, vRegMask_V0 v0) %{
5548 predicate(n->in(2)->get_int() < 32 &&
5549 (Matcher::vector_element_basic_type(n) == T_FLOAT));
5550 match(Set dst (VectorInsert (Binary src val) idx));
5551 effect(TEMP v0);
5552 format %{ "insertF_index_lt32 $dst, $src, $val, $idx" %}
5553 ins_encode %{
5554 __ vsetvli_helper(T_FLOAT, Matcher::vector_length(this));
5555 __ vid_v(as_VectorRegister($v0$$reg));
5556 __ vadd_vi(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg), -16);
5557 __ vmseq_vi(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg), (int)($idx$$constant) - 16);
5558 __ vfmerge_vfm(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), $val$$FloatRegister);
5559 %}
5560 ins_pipe(pipe_slow);
5561 %}
5562
5563 instruct insertF_index(vReg dst, vReg src, fRegF val, iRegIorL2I idx, vReg tmp, vRegMask_V0 v0) %{
5564 predicate(n->in(2)->get_int() >= 32 &&
5565 (Matcher::vector_element_basic_type(n) == T_FLOAT));
5566 match(Set dst (VectorInsert (Binary src val) idx));
5567 effect(TEMP tmp, TEMP v0);
5568 format %{ "insertF_index $dst, $src, $val, $idx\t# KILL $tmp" %}
5569 ins_encode %{
5570 __ vsetvli_helper(T_FLOAT, Matcher::vector_length(this));
5571 __ vid_v(as_VectorRegister($v0$$reg));
5572 __ vmv_v_x(as_VectorRegister($tmp$$reg), $idx$$Register);
5573 __ vmseq_vv(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg), as_VectorRegister($tmp$$reg));
5574 __ vfmerge_vfm(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), $val$$FloatRegister);
5575 %}
5576 ins_pipe(pipe_slow);
5577 %}
5578
5579 // DOUBLE
5580
5581 instruct insertD_index_lt32(vReg dst, vReg src, fRegD val, immI idx, vRegMask_V0 v0) %{
5582 predicate(n->in(2)->get_int() < 32 &&
5583 (Matcher::vector_element_basic_type(n) == T_DOUBLE));
5584 match(Set dst (VectorInsert (Binary src val) idx));
5585 effect(TEMP v0);
5586 format %{ "insertD_index_lt32 $dst, $src, $val, $idx" %}
5587 ins_encode %{
5588 __ vsetvli_helper(T_DOUBLE, Matcher::vector_length(this));
5589 __ vid_v(as_VectorRegister($v0$$reg));
5590 __ vadd_vi(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg), -16);
5591 __ vmseq_vi(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg), (int)($idx$$constant) - 16);
5592 __ vfmerge_vfm(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), $val$$FloatRegister);
5593 %}
5594 ins_pipe(pipe_slow);
5595 %}
5596
5597 instruct insertD_index(vReg dst, vReg src, fRegD val, iRegIorL2I idx, vReg tmp, vRegMask_V0 v0) %{
5598 predicate(n->in(2)->get_int() >= 32 &&
5599 (Matcher::vector_element_basic_type(n) == T_DOUBLE));
5600 match(Set dst (VectorInsert (Binary src val) idx));
5601 effect(TEMP tmp, TEMP v0);
5602 format %{ "insertD_index $dst, $src, $val, $idx\t# KILL $tmp" %}
5603 ins_encode %{
5604 __ vsetvli_helper(T_DOUBLE, Matcher::vector_length(this));
5605 __ vid_v(as_VectorRegister($v0$$reg));
5606 __ vmv_v_x(as_VectorRegister($tmp$$reg), $idx$$Register);
5607 __ vmseq_vv(as_VectorRegister($v0$$reg), as_VectorRegister($v0$$reg), as_VectorRegister($tmp$$reg));
5608 __ vfmerge_vfm(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), $val$$FloatRegister);
5609 %}
5610 ins_pipe(pipe_slow);
5611 %}
5612
5613 // ------------------------------ Vector mask reductions -----------------------
5614
5615 // true count
5616
5617 instruct vmask_truecount(iRegINoSp dst, vRegMask src) %{
5618 match(Set dst (VectorMaskTrueCount src));
5619 format %{ "vmask_truecount $dst, $src" %}
5620 ins_encode %{
5621 BasicType bt = Matcher::vector_element_basic_type(this, $src);
5622 __ vsetvli_helper(bt, Matcher::vector_length(this, $src));
5623 __ vcpop_m($dst$$Register, as_VectorRegister($src$$reg));
5624 %}
5625 ins_pipe(pipe_slow);
5626 %}
5627
5628 // first true
5629
5630 // Return the index of the first mask lane that is set, or vector length if none of
5631 // them are set.
5632
5633 instruct vmask_firsttrue(iRegINoSp dst, vRegMask src, vRegMask tmp) %{
5634 match(Set dst (VectorMaskFirstTrue src));
5635 effect(TEMP tmp);
5636 format %{ "vmask_firsttrue $dst, $src\t# KILL $tmp" %}
5637 ins_encode %{
5638 BasicType bt = Matcher::vector_element_basic_type(this, $src);
5639 __ vsetvli_helper(bt, Matcher::vector_length(this, $src));
5640 __ vmsbf_m(as_VectorRegister($tmp$$reg), as_VectorRegister($src$$reg));
5641 __ vcpop_m($dst$$Register, as_VectorRegister($tmp$$reg));
5642 %}
5643 ins_pipe(pipe_slow);
5644 %}
5645
5646 // last true
5647
5648 // Return the index of the first last lane that is set, or -1 if none of
5649 // them are set.
5650
5651 instruct vmask_lasttrue(iRegINoSp dst, vRegMask src) %{
5652 match(Set dst (VectorMaskLastTrue src));
5653 format %{ "vmask_lasttrue $dst, $src" %}
5654 ins_encode %{
5655 uint vector_length = Matcher::vector_length(this, $src);
5656 assert(UseZbb && vector_length <= XLEN, "precondition");
5657 __ vsetvli_helper(T_LONG, 1);
5658 __ vmv_x_s($dst$$Register, as_VectorRegister($src$$reg));
5659 if (XLEN != vector_length) {
5660 __ slli($dst$$Register, $dst$$Register, XLEN - vector_length);
5661 __ srli($dst$$Register, $dst$$Register, XLEN - vector_length);
5662 }
5663 __ clz($dst$$Register, $dst$$Register);
5664 __ mv(t0, XLEN - 1);
5665 __ sub($dst$$Register, t0, $dst$$Register);
5666 %}
5667 ins_pipe(pipe_slow);
5668 %}
5669
5670 // tolong
5671
5672 instruct vmask_tolong(iRegLNoSp dst, vRegMask src) %{
5673 match(Set dst (VectorMaskToLong src));
5674 format %{ "vmask_tolong $dst, $src" %}
5675 ins_encode %{
5676 uint vector_length = Matcher::vector_length(this, $src);
5677 assert(vector_length <= XLEN, "precondition");
5678 __ vsetvli_helper(T_LONG, 1);
5679 __ vmv_x_s($dst$$Register, as_VectorRegister($src$$reg));
5680 if (XLEN != vector_length) {
5681 __ slli($dst$$Register, $dst$$Register, XLEN - vector_length);
5682 __ srli($dst$$Register, $dst$$Register, XLEN - vector_length);
5683 }
5684 %}
5685 ins_pipe(pipe_slow);
5686 %}
5687
5688 // fromlong
5689
5690 instruct vmask_fromlong(vRegMask dst, iRegL src) %{
5691 match(Set dst (VectorLongToMask src));
5692 format %{ "vmask_fromlong $dst, $src" %}
5693 ins_encode %{
5694 assert(Matcher::vector_length(this) <= XLEN, "precondition");
5695 __ vsetvli_helper(T_LONG, 1);
5696 __ vmv_s_x(as_VectorRegister($dst$$reg), $src$$Register);
5697 %}
5698 ins_pipe(pipe_slow);
5699 %}
5700
5701 // ------------------------------ VectorTest -----------------------------------
5702
5703 // anytrue
5704
5705 // Not matched. Condition is negated and value zero is moved to the right side in CMoveINode::Ideal.
5706
5707 // instruct cmovI_vtest_anytrue(iRegINoSp dst, cmpOp cop, vRegMask op1, vRegMask op2, immI0 zero, immI_1 one) %{
5708 // predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne &&
5709 // static_cast<const VectorTestNode*>(n->in(1)->in(2))->get_predicate() == BoolTest::ne);
5710 // match(Set dst (CMoveI (Binary cop (VectorTest op1 op2)) (Binary zero one)));
5711 // format %{ "CMove $dst, (vectortest $cop $op1 $op2), zero, one\t#@cmovI_vtest_anytrue" %}
5712 // ins_encode %{
5713 // BasicType bt = Matcher::vector_element_basic_type(this, $op1);
5714 // uint vector_length = Matcher::vector_length(this, $op1);
5715 // __ vsetvli_helper(bt, vector_length);
5716 // __ vcpop_m($dst$$Register, as_VectorRegister($op1$$reg));
5717 // __ snez($dst$$Register, $dst$$Register);
5718 // %}
5719 // ins_pipe(pipe_slow);
5720 // %}
5721
5722 instruct cmovI_vtest_anytrue_negate(iRegINoSp dst, cmpOp cop, vRegMask op1, vRegMask op2, immI0 zero, immI_1 one) %{
5723 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq &&
5724 static_cast<const VectorTestNode*>(n->in(1)->in(2))->get_predicate() == BoolTest::ne);
5725 match(Set dst (CMoveI (Binary cop (VectorTest op1 op2)) (Binary one zero)));
5726 format %{ "CMove $dst, (vectortest $cop $op1 $op2), zero, one\t#@cmovI_vtest_anytrue_negate" %}
5727 ins_encode %{
5728 BasicType bt = Matcher::vector_element_basic_type(this, $op1);
5729 uint vector_length = Matcher::vector_length(this, $op1);
5730 __ vsetvli_helper(bt, vector_length);
5731 __ vcpop_m($dst$$Register, as_VectorRegister($op1$$reg));
5732 __ snez($dst$$Register, $dst$$Register);
5733 %}
5734 ins_pipe(pipe_slow);
5735 %}
5736
5737 // alltrue
5738
5739 // Not matched. Condition is negated and value zero is moved to the right side in CMoveINode::Ideal.
5740
5741 // instruct cmovI_vtest_alltrue(iRegINoSp dst, cmpOp cop, vRegMask op1, vRegMask op2, immI0 zero, immI_1 one) %{
5742 // predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq &&
5743 // static_cast<const VectorTestNode*>(n->in(1)->in(2))->get_predicate() == BoolTest::overflow);
5744 // match(Set dst (CMoveI (Binary cop (VectorTest op1 op2)) (Binary zero one)));
5745 // format %{ "CMove $dst, (vectortest $cop $op1 $op2), zero, one\t#@cmovI_vtest_alltrue" %}
5746 // ins_encode %{
5747 // BasicType bt = Matcher::vector_element_basic_type(this, $op1);
5748 // uint vector_length = Matcher::vector_length(this, $op1);
5749 // __ vsetvli_helper(bt, vector_length);
5750 // __ vcpop_m($dst$$Register, as_VectorRegister($op1$$reg));
5751 // __ sub($dst$$Register, $dst$$Register, vector_length);
5752 // __ seqz($dst$$Register, $dst$$Register);
5753 // %}
5754 // ins_pipe(pipe_slow);
5755 // %}
5756
5757 instruct cmovI_vtest_alltrue_negate(iRegINoSp dst, cmpOp cop, vRegMask op1, vRegMask op2, immI0 zero, immI_1 one) %{
5758 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne &&
5759 static_cast<const VectorTestNode*>(n->in(1)->in(2))->get_predicate() == BoolTest::overflow);
5760 match(Set dst (CMoveI (Binary cop (VectorTest op1 op2)) (Binary one zero)));
5761 format %{ "CMove $dst, (vectortest $cop $op1 $op2), zero, one\t#@cmovI_vtest_alltrue_negate" %}
5762 ins_encode %{
5763 BasicType bt = Matcher::vector_element_basic_type(this, $op1);
5764 uint vector_length = Matcher::vector_length(this, $op1);
5765 __ vsetvli_helper(bt, vector_length);
5766 __ vcpop_m($dst$$Register, as_VectorRegister($op1$$reg));
5767 __ sub($dst$$Register, $dst$$Register, vector_length);
5768 __ seqz($dst$$Register, $dst$$Register);
5769 %}
5770 ins_pipe(pipe_slow);
5771 %}
5772
5773 // anytrue
5774
5775 instruct vtest_anytrue_branch(cmpOpEqNe cop, vRegMask op1, vRegMask op2, label lbl) %{
5776 predicate(static_cast<const VectorTestNode*>(n->in(2))->get_predicate() == BoolTest::ne);
5777 match(If cop (VectorTest op1 op2));
5778 effect(USE lbl);
5779 format %{ "b$cop (vectortest ne $op1, $op2) $lbl\t#@vtest_anytrue_branch" %}
5780 ins_encode %{
5781 uint vector_length = Matcher::vector_length(this, $op1);
5782 BasicType bt = Matcher::vector_element_basic_type(this, $op1);
5783 __ vsetvli_helper(bt, vector_length);
5784 __ vcpop_m(t0, as_VectorRegister($op1$$reg));
5785 __ enc_cmpEqNe_imm0_branch($cop$$cmpcode, t0, *($lbl$$label), /* is_far */ true);
5786 %}
5787 ins_pipe(pipe_slow);
5788 %}
5789
5790 // alltrue
5791
5792 instruct vtest_alltrue_branch(cmpOpEqNe cop, vRegMask op1, vRegMask op2, label lbl) %{
5793 predicate(static_cast<const VectorTestNode*>(n->in(2))->get_predicate() == BoolTest::overflow);
5794 match(If cop (VectorTest op1 op2));
5795 effect(USE lbl);
5796 format %{ "b$cop (vectortest overflow $op1, $op2) $lbl\t#@vtest_alltrue_branch" %}
5797 ins_encode %{
5798 uint vector_length = Matcher::vector_length(this, $op1);
5799 BasicType bt = Matcher::vector_element_basic_type(this, $op1);
5800 __ vsetvli_helper(bt, vector_length);
5801 __ vcpop_m(t0, as_VectorRegister($op1$$reg));
5802 __ sub(t0, t0, vector_length);
5803 __ enc_cmpEqNe_imm0_branch($cop$$cmpcode, t0, *($lbl$$label), /* is_far */ true);
5804 %}
5805 ins_pipe(pipe_slow);
5806 %}