1 /*
2 * Copyright (c) 2007, 2022, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "memory/allocation.inline.hpp"
26 #include "opto/connode.hpp"
27 #include "opto/mulnode.hpp"
28 #include "opto/subnode.hpp"
29 #include "opto/vectornode.hpp"
30 #include "opto/convertnode.hpp"
31 #include "utilities/powerOfTwo.hpp"
32 #include "utilities/globalDefinitions.hpp"
33
34 //------------------------------VectorNode--------------------------------------
35
36 // Return the vector operator for the specified scalar operation
37 // and vector length.
38 int VectorNode::opcode(int sopc, BasicType bt) {
39 switch (sopc) {
40 case Op_AddI:
41 switch (bt) {
42 case T_BOOLEAN:
43 case T_BYTE: return Op_AddVB;
44 case T_CHAR:
45 case T_SHORT: return Op_AddVS;
46 case T_INT: return Op_AddVI;
47 default: return 0;
48 }
49 case Op_AddL: return (bt == T_LONG ? Op_AddVL : 0);
50 case Op_AddF: return (bt == T_FLOAT ? Op_AddVF : 0);
51 case Op_AddD: return (bt == T_DOUBLE ? Op_AddVD : 0);
52
53 case Op_SubI:
54 switch (bt) {
55 case T_BOOLEAN:
56 case T_BYTE: return Op_SubVB;
57 case T_CHAR:
58 case T_SHORT: return Op_SubVS;
59 case T_INT: return Op_SubVI;
60 default: return 0;
61 }
62 case Op_SubL: return (bt == T_LONG ? Op_SubVL : 0);
63 case Op_SubF: return (bt == T_FLOAT ? Op_SubVF : 0);
64 case Op_SubD: return (bt == T_DOUBLE ? Op_SubVD : 0);
65
66 case Op_MulI:
67 switch (bt) {
68 case T_BOOLEAN:return 0;
69 case T_BYTE: return Op_MulVB;
70 case T_CHAR:
71 case T_SHORT: return Op_MulVS;
72 case T_INT: return Op_MulVI;
73 default: return 0;
74 }
75 case Op_MulL: return (bt == T_LONG ? Op_MulVL : 0);
76 case Op_MulF:
77 return (bt == T_FLOAT ? Op_MulVF : 0);
78 case Op_MulD:
79 return (bt == T_DOUBLE ? Op_MulVD : 0);
80 case Op_FmaD:
81 return (bt == T_DOUBLE ? Op_FmaVD : 0);
82 case Op_FmaF:
83 return (bt == T_FLOAT ? Op_FmaVF : 0);
84 case Op_CMoveF:
85 return (bt == T_FLOAT ? Op_CMoveVF : 0);
86 case Op_CMoveD:
87 return (bt == T_DOUBLE ? Op_CMoveVD : 0);
88 case Op_DivF:
89 return (bt == T_FLOAT ? Op_DivVF : 0);
90 case Op_DivD:
91 return (bt == T_DOUBLE ? Op_DivVD : 0);
92 case Op_AbsI:
93 switch (bt) {
94 case T_BOOLEAN:
95 case T_CHAR: return 0; // abs does not make sense for unsigned
96 case T_BYTE: return Op_AbsVB;
97 case T_SHORT: return Op_AbsVS;
98 case T_INT: return Op_AbsVI;
99 default: return 0;
100 }
101 case Op_AbsL:
102 return (bt == T_LONG ? Op_AbsVL : 0);
103 case Op_MinI:
104 switch (bt) {
105 case T_BOOLEAN:
106 case T_CHAR: return 0;
107 case T_BYTE:
108 case T_SHORT:
109 case T_INT: return Op_MinV;
110 default: return 0;
111 }
112 case Op_MinL:
113 return (bt == T_LONG ? Op_MinV : 0);
114 case Op_MinF:
115 return (bt == T_FLOAT ? Op_MinV : 0);
116 case Op_MinD:
117 return (bt == T_DOUBLE ? Op_MinV : 0);
118 case Op_MaxI:
119 switch (bt) {
120 case T_BOOLEAN:
121 case T_CHAR: return 0;
122 case T_BYTE:
123 case T_SHORT:
124 case T_INT: return Op_MaxV;
125 default: return 0;
126 }
127 case Op_MaxL:
128 return (bt == T_LONG ? Op_MaxV : 0);
129 case Op_MaxF:
130 return (bt == T_FLOAT ? Op_MaxV : 0);
131 case Op_MaxD:
132 return (bt == T_DOUBLE ? Op_MaxV : 0);
133 case Op_AbsF:
134 return (bt == T_FLOAT ? Op_AbsVF : 0);
135 case Op_AbsD:
136 return (bt == T_DOUBLE ? Op_AbsVD : 0);
137 case Op_NegI:
138 switch (bt) {
139 case T_BYTE:
140 case T_SHORT:
141 case T_INT: return Op_NegVI;
142 default: return 0;
143 }
144 case Op_NegL:
145 return (bt == T_LONG ? Op_NegVL : 0);
146 case Op_NegF:
147 return (bt == T_FLOAT ? Op_NegVF : 0);
148 case Op_NegD:
149 return (bt == T_DOUBLE ? Op_NegVD : 0);
150 case Op_RoundDoubleMode:
151 return (bt == T_DOUBLE ? Op_RoundDoubleModeV : 0);
152 case Op_RotateLeft:
153 return (is_integral_type(bt) ? Op_RotateLeftV : 0);
154 case Op_RotateRight:
155 return (is_integral_type(bt) ? Op_RotateRightV : 0);
156 case Op_SqrtF:
157 return (bt == T_FLOAT ? Op_SqrtVF : 0);
158 case Op_SqrtD:
159 return (bt == T_DOUBLE ? Op_SqrtVD : 0);
160 case Op_RoundF:
161 return (bt == T_INT ? Op_RoundVF : 0);
162 case Op_RoundD:
163 return (bt == T_LONG ? Op_RoundVD : 0);
164 case Op_PopCountI:
165 return Op_PopCountVI;
166 case Op_PopCountL:
167 return Op_PopCountVL;
168 case Op_ReverseI:
169 case Op_ReverseL:
170 return (is_integral_type(bt) ? Op_ReverseV : 0);
171 case Op_ReverseBytesS:
172 case Op_ReverseBytesI:
173 case Op_ReverseBytesL:
174 return (is_integral_type(bt) ? Op_ReverseBytesV : 0);
175 case Op_CompressBits:
176 // Not implemented. Returning 0 temporarily
177 return 0;
178 case Op_ExpandBits:
179 // Not implemented. Returning 0 temporarily
180 return 0;
181 case Op_LShiftI:
182 switch (bt) {
183 case T_BOOLEAN:
184 case T_BYTE: return Op_LShiftVB;
185 case T_CHAR:
186 case T_SHORT: return Op_LShiftVS;
187 case T_INT: return Op_LShiftVI;
188 default: return 0;
189 }
190 case Op_LShiftL:
191 return (bt == T_LONG ? Op_LShiftVL : 0);
192 case Op_RShiftI:
193 switch (bt) {
194 case T_BOOLEAN:return Op_URShiftVB; // boolean is unsigned value
195 case T_CHAR: return Op_URShiftVS; // char is unsigned value
196 case T_BYTE: return Op_RShiftVB;
197 case T_SHORT: return Op_RShiftVS;
198 case T_INT: return Op_RShiftVI;
199 default: return 0;
200 }
201 case Op_RShiftL:
202 return (bt == T_LONG ? Op_RShiftVL : 0);
203 case Op_URShiftB:
204 return (bt == T_BYTE ? Op_URShiftVB : 0);
205 case Op_URShiftS:
206 return (bt == T_SHORT ? Op_URShiftVS : 0);
207 case Op_URShiftI:
208 switch (bt) {
209 case T_BOOLEAN:return Op_URShiftVB;
210 case T_CHAR: return Op_URShiftVS;
211 case T_BYTE:
212 case T_SHORT: return 0; // Vector logical right shift for signed short
213 // values produces incorrect Java result for
214 // negative data because java code should convert
215 // a short value into int value with sign
216 // extension before a shift.
217 case T_INT: return Op_URShiftVI;
218 default: return 0;
219 }
220 case Op_URShiftL:
221 return (bt == T_LONG ? Op_URShiftVL : 0);
222 case Op_AndI:
223 case Op_AndL:
224 return Op_AndV;
225 case Op_OrI:
226 case Op_OrL:
227 return Op_OrV;
228 case Op_XorI:
229 case Op_XorL:
230 return Op_XorV;
231
232 case Op_LoadB:
233 case Op_LoadUB:
234 case Op_LoadUS:
235 case Op_LoadS:
236 case Op_LoadI:
237 case Op_LoadL:
238 case Op_LoadF:
239 case Op_LoadD:
240 return Op_LoadVector;
241
242 case Op_StoreB:
243 case Op_StoreC:
244 case Op_StoreI:
245 case Op_StoreL:
246 case Op_StoreF:
247 case Op_StoreD:
248 return Op_StoreVector;
249 case Op_MulAddS2I:
250 return Op_MulAddVS2VI;
251 case Op_ConvI2F:
252 return Op_VectorCastI2X;
253 case Op_ConvL2D:
254 return Op_VectorCastL2X;
255 case Op_ConvF2I:
256 return Op_VectorCastF2X;
257 case Op_ConvD2L:
258 return Op_VectorCastD2X;
259 case Op_CountLeadingZerosI:
260 case Op_CountLeadingZerosL:
261 return Op_CountLeadingZerosV;
262 case Op_CountTrailingZerosI:
263 case Op_CountTrailingZerosL:
264 return Op_CountTrailingZerosV;
265
266 default:
267 return 0; // Unimplemented
268 }
269 }
270
271 int VectorNode::replicate_opcode(BasicType bt) {
272 switch(bt) {
273 case T_BOOLEAN:
274 case T_BYTE:
275 return Op_ReplicateB;
276 case T_SHORT:
277 case T_CHAR:
278 return Op_ReplicateS;
279 case T_INT:
280 return Op_ReplicateI;
281 case T_LONG:
282 return Op_ReplicateL;
283 case T_FLOAT:
284 return Op_ReplicateF;
285 case T_DOUBLE:
286 return Op_ReplicateD;
287 default:
288 assert(false, "wrong type: %s", type2name(bt));
289 return 0;
290 }
291 }
292
293 // Also used to check if the code generator
294 // supports the vector operation.
295 bool VectorNode::implemented(int opc, uint vlen, BasicType bt) {
296 if (is_java_primitive(bt) &&
297 (vlen > 1) && is_power_of_2(vlen) &&
298 Matcher::vector_size_supported(bt, vlen)) {
299 int vopc = VectorNode::opcode(opc, bt);
300 // For rotate operation we will do a lazy de-generation into
301 // OrV/LShiftV/URShiftV pattern if the target does not support
302 // vector rotation instruction.
303 if (VectorNode::is_vector_rotate(vopc)) {
304 return is_vector_rotate_supported(vopc, vlen, bt);
305 }
306 if (VectorNode::is_vector_integral_negate(vopc)) {
307 return is_vector_integral_negate_supported(vopc, vlen, bt, false);
308 }
309 return vopc > 0 && Matcher::match_rule_supported_vector(vopc, vlen, bt);
310 }
311 return false;
312 }
313
314 bool VectorNode::is_type_transition_short_to_int(Node* n) {
315 switch (n->Opcode()) {
316 case Op_MulAddS2I:
317 return true;
318 }
319 return false;
320 }
321
322 bool VectorNode::is_type_transition_to_int(Node* n) {
323 return is_type_transition_short_to_int(n);
324 }
325
326 bool VectorNode::is_muladds2i(Node* n) {
327 if (n->Opcode() == Op_MulAddS2I) {
328 return true;
329 }
330 return false;
331 }
332
333 bool VectorNode::is_type_transition_long_to_int(Node* n) {
334 switch(n->Opcode()) {
335 case Op_PopCountL:
336 case Op_CountLeadingZerosL:
337 case Op_CountTrailingZerosL:
338 return true;
339 default:
340 return false;
341 }
342 }
343
344 bool VectorNode::is_roundopD(Node* n) {
345 if (n->Opcode() == Op_RoundDoubleMode) {
346 return true;
347 }
348 return false;
349 }
350
351 bool VectorNode::is_vector_rotate_supported(int vopc, uint vlen, BasicType bt) {
352 assert(VectorNode::is_vector_rotate(vopc), "wrong opcode");
353
354 // If target defines vector rotation patterns then no
355 // need for degeneration.
356 if (Matcher::match_rule_supported_vector(vopc, vlen, bt)) {
357 return true;
358 }
359
360 // If target does not support variable shift operations then no point
361 // in creating a rotate vector node since it will not be disintegratable.
362 // Adding a pessimistic check to avoid complex pattern matching which
363 // may not be full proof.
364 if (!Matcher::supports_vector_variable_shifts()) {
365 return false;
366 }
367
368 // Validate existence of nodes created in case of rotate degeneration.
369 switch (bt) {
370 case T_INT:
371 return Matcher::match_rule_supported_vector(Op_OrV, vlen, bt) &&
372 Matcher::match_rule_supported_vector(Op_LShiftVI, vlen, bt) &&
373 Matcher::match_rule_supported_vector(Op_URShiftVI, vlen, bt);
374 case T_LONG:
375 return Matcher::match_rule_supported_vector(Op_OrV, vlen, bt) &&
376 Matcher::match_rule_supported_vector(Op_LShiftVL, vlen, bt) &&
377 Matcher::match_rule_supported_vector(Op_URShiftVL, vlen, bt);
378 default:
379 assert(false, "not supported: %s", type2name(bt));
380 return false;
381 }
382 }
383
384 // Check whether the architecture supports the vector negate instructions. If not, then check
385 // whether the alternative vector nodes used to implement vector negation are supported.
386 // Return false if neither of them is supported.
387 bool VectorNode::is_vector_integral_negate_supported(int opc, uint vlen, BasicType bt, bool use_predicate) {
388 if (!use_predicate) {
389 // Check whether the NegVI/L is supported by the architecture.
390 if (Matcher::match_rule_supported_vector(opc, vlen, bt)) {
391 return true;
392 }
393 // Negate is implemented with "(SubVI/L (ReplicateI/L 0) src)", if NegVI/L is not supported.
394 int sub_opc = (bt == T_LONG) ? Op_SubL : Op_SubI;
395 if (Matcher::match_rule_supported_vector(VectorNode::opcode(sub_opc, bt), vlen, bt) &&
396 Matcher::match_rule_supported_vector(VectorNode::replicate_opcode(bt), vlen, bt)) {
397 return true;
398 }
399 } else {
400 // Check whether the predicated NegVI/L is supported by the architecture.
401 if (Matcher::match_rule_supported_vector_masked(opc, vlen, bt)) {
402 return true;
403 }
404 // Predicated negate is implemented with "(AddVI/L (XorV src (ReplicateI/L -1)) (ReplicateI/L 1))",
405 // if predicated NegVI/L is not supported.
406 int add_opc = (bt == T_LONG) ? Op_AddL : Op_AddI;
407 if (Matcher::match_rule_supported_vector_masked(Op_XorV, vlen, bt) &&
408 Matcher::match_rule_supported_vector_masked(VectorNode::opcode(add_opc, bt), vlen, bt) &&
409 Matcher::match_rule_supported_vector(VectorNode::replicate_opcode(bt), vlen, bt)) {
410 return true;
411 }
412 }
413 return false;
414 }
415
416 bool VectorNode::is_shift_opcode(int opc) {
417 switch (opc) {
418 case Op_LShiftI:
419 case Op_LShiftL:
420 case Op_RShiftI:
421 case Op_RShiftL:
422 case Op_URShiftB:
423 case Op_URShiftS:
424 case Op_URShiftI:
425 case Op_URShiftL:
426 return true;
427 default:
428 return false;
429 }
430 }
431
432 bool VectorNode::is_shift(Node* n) {
433 return is_shift_opcode(n->Opcode());
434 }
435
436 bool VectorNode::is_rotate_opcode(int opc) {
437 switch (opc) {
438 case Op_RotateRight:
439 case Op_RotateLeft:
440 return true;
441 default:
442 return false;
443 }
444 }
445
446 bool VectorNode::is_scalar_rotate(Node* n) {
447 if (is_rotate_opcode(n->Opcode())) {
448 return true;
449 }
450 return false;
451 }
452
453 bool VectorNode::is_vshift_cnt_opcode(int opc) {
454 switch (opc) {
455 case Op_LShiftCntV:
456 case Op_RShiftCntV:
457 return true;
458 default:
459 return false;
460 }
461 }
462
463 bool VectorNode::is_vshift_cnt(Node* n) {
464 return is_vshift_cnt_opcode(n->Opcode());
465 }
466
467 // Check if input is loop invariant vector.
468 bool VectorNode::is_invariant_vector(Node* n) {
469 // Only Replicate vector nodes are loop invariant for now.
470 switch (n->Opcode()) {
471 case Op_ReplicateB:
472 case Op_ReplicateS:
473 case Op_ReplicateI:
474 case Op_ReplicateL:
475 case Op_ReplicateF:
476 case Op_ReplicateD:
477 return true;
478 default:
479 return false;
480 }
481 }
482
483 // [Start, end) half-open range defining which operands are vectors
484 void VectorNode::vector_operands(Node* n, uint* start, uint* end) {
485 switch (n->Opcode()) {
486 case Op_LoadB: case Op_LoadUB:
487 case Op_LoadS: case Op_LoadUS:
488 case Op_LoadI: case Op_LoadL:
489 case Op_LoadF: case Op_LoadD:
490 case Op_LoadP: case Op_LoadN:
491 *start = 0;
492 *end = 0; // no vector operands
493 break;
494 case Op_StoreB: case Op_StoreC:
495 case Op_StoreI: case Op_StoreL:
496 case Op_StoreF: case Op_StoreD:
497 case Op_StoreP: case Op_StoreN:
498 *start = MemNode::ValueIn;
499 *end = MemNode::ValueIn + 1; // 1 vector operand
500 break;
501 case Op_LShiftI: case Op_LShiftL:
502 case Op_RShiftI: case Op_RShiftL:
503 case Op_URShiftI: case Op_URShiftL:
504 *start = 1;
505 *end = 2; // 1 vector operand
506 break;
507 case Op_AddI: case Op_AddL: case Op_AddF: case Op_AddD:
508 case Op_SubI: case Op_SubL: case Op_SubF: case Op_SubD:
509 case Op_MulI: case Op_MulL: case Op_MulF: case Op_MulD:
510 case Op_DivF: case Op_DivD:
511 case Op_AndI: case Op_AndL:
512 case Op_OrI: case Op_OrL:
513 case Op_XorI: case Op_XorL:
514 case Op_MulAddS2I:
515 *start = 1;
516 *end = 3; // 2 vector operands
517 break;
518 case Op_CMoveI: case Op_CMoveL: case Op_CMoveF: case Op_CMoveD:
519 *start = 2;
520 *end = n->req();
521 break;
522 case Op_FmaD:
523 case Op_FmaF:
524 *start = 1;
525 *end = 4; // 3 vector operands
526 break;
527 default:
528 *start = 1;
529 *end = n->req(); // default is all operands
530 }
531 }
532
533 VectorNode* VectorNode::make_mask_node(int vopc, Node* n1, Node* n2, uint vlen, BasicType bt) {
534 guarantee(vopc > 0, "vopc must be > 0");
535 const TypeVect* vmask_type = TypeVect::makemask(bt, vlen);
536 switch (vopc) {
537 case Op_AndV:
538 if (Matcher::match_rule_supported_vector_masked(Op_AndVMask, vlen, bt)) {
539 return new AndVMaskNode(n1, n2, vmask_type);
540 }
541 return new AndVNode(n1, n2, vmask_type);
542 case Op_OrV:
543 if (Matcher::match_rule_supported_vector_masked(Op_OrVMask, vlen, bt)) {
544 return new OrVMaskNode(n1, n2, vmask_type);
545 }
546 return new OrVNode(n1, n2, vmask_type);
547 case Op_XorV:
548 if (Matcher::match_rule_supported_vector_masked(Op_XorVMask, vlen, bt)) {
549 return new XorVMaskNode(n1, n2, vmask_type);
550 }
551 return new XorVNode(n1, n2, vmask_type);
552 default:
553 fatal("Unsupported mask vector creation for '%s'", NodeClassNames[vopc]);
554 return NULL;
555 }
556 }
557
558 // Make a vector node for binary operation
559 VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, const TypeVect* vt, bool is_mask, bool is_var_shift) {
560 // This method should not be called for unimplemented vectors.
561 guarantee(vopc > 0, "vopc must be > 0");
562
563 if (is_mask) {
564 return make_mask_node(vopc, n1, n2, vt->length(), vt->element_basic_type());
565 }
566
567 switch (vopc) {
568 case Op_AddVB: return new AddVBNode(n1, n2, vt);
569 case Op_AddVS: return new AddVSNode(n1, n2, vt);
570 case Op_AddVI: return new AddVINode(n1, n2, vt);
571 case Op_AddVL: return new AddVLNode(n1, n2, vt);
572 case Op_AddVF: return new AddVFNode(n1, n2, vt);
573 case Op_AddVD: return new AddVDNode(n1, n2, vt);
574
575 case Op_SubVB: return new SubVBNode(n1, n2, vt);
576 case Op_SubVS: return new SubVSNode(n1, n2, vt);
577 case Op_SubVI: return new SubVINode(n1, n2, vt);
578 case Op_SubVL: return new SubVLNode(n1, n2, vt);
579 case Op_SubVF: return new SubVFNode(n1, n2, vt);
580 case Op_SubVD: return new SubVDNode(n1, n2, vt);
581
582 case Op_MulVB: return new MulVBNode(n1, n2, vt);
583 case Op_MulVS: return new MulVSNode(n1, n2, vt);
584 case Op_MulVI: return new MulVINode(n1, n2, vt);
585 case Op_MulVL: return new MulVLNode(n1, n2, vt);
586 case Op_MulVF: return new MulVFNode(n1, n2, vt);
587 case Op_MulVD: return new MulVDNode(n1, n2, vt);
588
589 case Op_DivVF: return new DivVFNode(n1, n2, vt);
590 case Op_DivVD: return new DivVDNode(n1, n2, vt);
591
592 case Op_MinV: return new MinVNode(n1, n2, vt);
593 case Op_MaxV: return new MaxVNode(n1, n2, vt);
594
595 case Op_AbsVF: return new AbsVFNode(n1, vt);
596 case Op_AbsVD: return new AbsVDNode(n1, vt);
597 case Op_AbsVB: return new AbsVBNode(n1, vt);
598 case Op_AbsVS: return new AbsVSNode(n1, vt);
599 case Op_AbsVI: return new AbsVINode(n1, vt);
600 case Op_AbsVL: return new AbsVLNode(n1, vt);
601
602 case Op_NegVI: return new NegVINode(n1, vt);
603 case Op_NegVL: return new NegVLNode(n1, vt);
604 case Op_NegVF: return new NegVFNode(n1, vt);
605 case Op_NegVD: return new NegVDNode(n1, vt);
606
607 case Op_ReverseV: return new ReverseVNode(n1, vt);
608 case Op_ReverseBytesV: return new ReverseBytesVNode(n1, vt);
609
610 case Op_SqrtVF: return new SqrtVFNode(n1, vt);
611 case Op_SqrtVD: return new SqrtVDNode(n1, vt);
612
613 case Op_RoundVF: return new RoundVFNode(n1, vt);
614 case Op_RoundVD: return new RoundVDNode(n1, vt);
615
616 case Op_PopCountVI: return new PopCountVINode(n1, vt);
617 case Op_PopCountVL: return new PopCountVLNode(n1, vt);
618 case Op_RotateLeftV: return new RotateLeftVNode(n1, n2, vt);
619 case Op_RotateRightV: return new RotateRightVNode(n1, n2, vt);
620
621 case Op_LShiftVB: return new LShiftVBNode(n1, n2, vt, is_var_shift);
622 case Op_LShiftVS: return new LShiftVSNode(n1, n2, vt, is_var_shift);
623 case Op_LShiftVI: return new LShiftVINode(n1, n2, vt, is_var_shift);
624 case Op_LShiftVL: return new LShiftVLNode(n1, n2, vt, is_var_shift);
625
626 case Op_RShiftVB: return new RShiftVBNode(n1, n2, vt, is_var_shift);
627 case Op_RShiftVS: return new RShiftVSNode(n1, n2, vt, is_var_shift);
628 case Op_RShiftVI: return new RShiftVINode(n1, n2, vt, is_var_shift);
629 case Op_RShiftVL: return new RShiftVLNode(n1, n2, vt, is_var_shift);
630
631 case Op_URShiftVB: return new URShiftVBNode(n1, n2, vt, is_var_shift);
632 case Op_URShiftVS: return new URShiftVSNode(n1, n2, vt, is_var_shift);
633 case Op_URShiftVI: return new URShiftVINode(n1, n2, vt, is_var_shift);
634 case Op_URShiftVL: return new URShiftVLNode(n1, n2, vt, is_var_shift);
635
636 case Op_AndV: return new AndVNode(n1, n2, vt);
637 case Op_OrV: return new OrVNode (n1, n2, vt);
638 case Op_XorV: return new XorVNode(n1, n2, vt);
639
640 case Op_RoundDoubleModeV: return new RoundDoubleModeVNode(n1, n2, vt);
641
642 case Op_MulAddVS2VI: return new MulAddVS2VINode(n1, n2, vt);
643
644 case Op_ExpandV: return new ExpandVNode(n1, n2, vt);
645 case Op_CompressV: return new CompressVNode(n1, n2, vt);
646 case Op_CompressM: assert(n1 == NULL, ""); return new CompressMNode(n2, vt);
647 case Op_CountLeadingZerosV: return new CountLeadingZerosVNode(n1, vt);
648 case Op_CountTrailingZerosV: return new CountTrailingZerosVNode(n1, vt);
649 default:
650 fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
651 return NULL;
652 }
653 }
654
655 // Return the vector version of a scalar binary operation node.
656 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, uint vlen, BasicType bt, bool is_var_shift) {
657 const TypeVect* vt = TypeVect::make(bt, vlen);
658 int vopc = VectorNode::opcode(opc, bt);
659 // This method should not be called for unimplemented vectors.
660 guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]);
661 return make(vopc, n1, n2, vt, false, is_var_shift);
662 }
663
664 // Make a vector node for ternary operation
665 VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, Node* n3, const TypeVect* vt) {
666 // This method should not be called for unimplemented vectors.
667 guarantee(vopc > 0, "vopc must be > 0");
668 switch (vopc) {
669 case Op_FmaVD: return new FmaVDNode(n1, n2, n3, vt);
670 case Op_FmaVF: return new FmaVFNode(n1, n2, n3, vt);
671 default:
672 fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
673 return NULL;
674 }
675 }
676
677 // Return the vector version of a scalar ternary operation node.
678 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, Node* n3, uint vlen, BasicType bt) {
679 const TypeVect* vt = TypeVect::make(bt, vlen);
680 int vopc = VectorNode::opcode(opc, bt);
681 // This method should not be called for unimplemented vectors.
682 guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]);
683 return make(vopc, n1, n2, n3, vt);
684 }
685
686 // Scalar promotion
687 VectorNode* VectorNode::scalar2vector(Node* s, uint vlen, const Type* opd_t, bool is_mask) {
688 BasicType bt = opd_t->array_element_basic_type();
689 if (is_mask && Matcher::match_rule_supported_vector(Op_MaskAll, vlen, bt)) {
690 const TypeVect* vt = TypeVect::make(opd_t, vlen, true);
691 return new MaskAllNode(s, vt);
692 }
693
694 const TypeVect* vt = opd_t->singleton() ? TypeVect::make(opd_t, vlen)
695 : TypeVect::make(bt, vlen);
696 switch (bt) {
697 case T_BOOLEAN:
698 case T_BYTE:
699 return new ReplicateBNode(s, vt);
700 case T_CHAR:
701 case T_SHORT:
702 return new ReplicateSNode(s, vt);
703 case T_INT:
704 return new ReplicateINode(s, vt);
705 case T_LONG:
706 return new ReplicateLNode(s, vt);
707 case T_FLOAT:
708 return new ReplicateFNode(s, vt);
709 case T_DOUBLE:
710 return new ReplicateDNode(s, vt);
711 default:
712 fatal("Type '%s' is not supported for vectors", type2name(bt));
713 return NULL;
714 }
715 }
716
717 VectorNode* VectorNode::shift_count(int opc, Node* cnt, uint vlen, BasicType bt) {
718 // Match shift count type with shift vector type.
719 const TypeVect* vt = TypeVect::make(bt, vlen);
720 switch (opc) {
721 case Op_LShiftI:
722 case Op_LShiftL:
723 return new LShiftCntVNode(cnt, vt);
724 case Op_RShiftI:
725 case Op_RShiftL:
726 case Op_URShiftB:
727 case Op_URShiftS:
728 case Op_URShiftI:
729 case Op_URShiftL:
730 return new RShiftCntVNode(cnt, vt);
731 default:
732 fatal("Missed vector creation for '%s'", NodeClassNames[opc]);
733 return NULL;
734 }
735 }
736
737 bool VectorNode::is_vector_rotate(int opc) {
738 switch (opc) {
739 case Op_RotateLeftV:
740 case Op_RotateRightV:
741 return true;
742 default:
743 return false;
744 }
745 }
746
747 bool VectorNode::is_vector_integral_negate(int opc) {
748 return opc == Op_NegVI || opc == Op_NegVL;
749 }
750
751 bool VectorNode::is_vector_shift(int opc) {
752 assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode");
753 switch (opc) {
754 case Op_LShiftVB:
755 case Op_LShiftVS:
756 case Op_LShiftVI:
757 case Op_LShiftVL:
758 case Op_RShiftVB:
759 case Op_RShiftVS:
760 case Op_RShiftVI:
761 case Op_RShiftVL:
762 case Op_URShiftVB:
763 case Op_URShiftVS:
764 case Op_URShiftVI:
765 case Op_URShiftVL:
766 return true;
767 default:
768 return false;
769 }
770 }
771
772 bool VectorNode::is_vector_shift_count(int opc) {
773 assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode");
774 switch (opc) {
775 case Op_RShiftCntV:
776 case Op_LShiftCntV:
777 return true;
778 default:
779 return false;
780 }
781 }
782
783 static bool is_con_M1(Node* n) {
784 if (n->is_Con()) {
785 const Type* t = n->bottom_type();
786 if (t->isa_int() && t->is_int()->get_con() == -1) {
787 return true;
788 }
789 if (t->isa_long() && t->is_long()->get_con() == -1) {
790 return true;
791 }
792 }
793 return false;
794 }
795
796 bool VectorNode::is_all_ones_vector(Node* n) {
797 switch (n->Opcode()) {
798 case Op_ReplicateB:
799 case Op_ReplicateS:
800 case Op_ReplicateI:
801 case Op_ReplicateL:
802 return is_con_M1(n->in(1));
803 default:
804 return false;
805 }
806 }
807
808 bool VectorNode::is_vector_bitwise_not_pattern(Node* n) {
809 if (n->Opcode() == Op_XorV) {
810 return is_all_ones_vector(n->in(1)) ||
811 is_all_ones_vector(n->in(2));
812 }
813 return false;
814 }
815
816 // Return initial Pack node. Additional operands added with add_opd() calls.
817 PackNode* PackNode::make(Node* s, uint vlen, BasicType bt) {
818 const TypeVect* vt = TypeVect::make(bt, vlen);
819 switch (bt) {
820 case T_BOOLEAN:
821 case T_BYTE:
822 return new PackBNode(s, vt);
823 case T_CHAR:
824 case T_SHORT:
825 return new PackSNode(s, vt);
826 case T_INT:
827 return new PackINode(s, vt);
828 case T_LONG:
829 return new PackLNode(s, vt);
830 case T_FLOAT:
831 return new PackFNode(s, vt);
832 case T_DOUBLE:
833 return new PackDNode(s, vt);
834 default:
835 fatal("Type '%s' is not supported for vectors", type2name(bt));
836 return NULL;
837 }
838 }
839
840 // Create a binary tree form for Packs. [lo, hi) (half-open) range
841 PackNode* PackNode::binary_tree_pack(int lo, int hi) {
842 int ct = hi - lo;
843 assert(is_power_of_2(ct), "power of 2");
844 if (ct == 2) {
845 PackNode* pk = PackNode::make(in(lo), 2, vect_type()->element_basic_type());
846 pk->add_opd(in(lo+1));
847 return pk;
848 } else {
849 int mid = lo + ct/2;
850 PackNode* n1 = binary_tree_pack(lo, mid);
851 PackNode* n2 = binary_tree_pack(mid, hi );
852
853 BasicType bt = n1->vect_type()->element_basic_type();
854 assert(bt == n2->vect_type()->element_basic_type(), "should be the same");
855 switch (bt) {
856 case T_BOOLEAN:
857 case T_BYTE:
858 return new PackSNode(n1, n2, TypeVect::make(T_SHORT, 2));
859 case T_CHAR:
860 case T_SHORT:
861 return new PackINode(n1, n2, TypeVect::make(T_INT, 2));
862 case T_INT:
863 return new PackLNode(n1, n2, TypeVect::make(T_LONG, 2));
864 case T_LONG:
865 return new Pack2LNode(n1, n2, TypeVect::make(T_LONG, 2));
866 case T_FLOAT:
867 return new PackDNode(n1, n2, TypeVect::make(T_DOUBLE, 2));
868 case T_DOUBLE:
869 return new Pack2DNode(n1, n2, TypeVect::make(T_DOUBLE, 2));
870 default:
871 fatal("Type '%s' is not supported for vectors", type2name(bt));
872 return NULL;
873 }
874 }
875 }
876
877 // Return the vector version of a scalar load node.
878 LoadVectorNode* LoadVectorNode::make(int opc, Node* ctl, Node* mem,
879 Node* adr, const TypePtr* atyp,
880 uint vlen, BasicType bt,
881 ControlDependency control_dependency) {
882 const TypeVect* vt = TypeVect::make(bt, vlen);
883 return new LoadVectorNode(ctl, mem, adr, atyp, vt, control_dependency);
884 }
885
886 // Return the vector version of a scalar store node.
887 StoreVectorNode* StoreVectorNode::make(int opc, Node* ctl, Node* mem,
888 Node* adr, const TypePtr* atyp, Node* val,
889 uint vlen) {
890 return new StoreVectorNode(ctl, mem, adr, atyp, val);
891 }
892
893 Node* LoadVectorMaskedNode::Ideal(PhaseGVN* phase, bool can_reshape) {
894 if (!in(3)->is_top() && in(3)->Opcode() == Op_VectorMaskGen) {
895 Node* mask_len = in(3)->in(1);
896 const TypeLong* ty = phase->type(mask_len)->isa_long();
897 if (ty && ty->is_con()) {
898 BasicType mask_bt = Matcher::vector_element_basic_type(in(3));
899 int load_sz = type2aelembytes(mask_bt) * ty->get_con();
900 assert(load_sz <= MaxVectorSize, "Unexpected load size");
901 if (load_sz == MaxVectorSize) {
902 Node* ctr = in(MemNode::Control);
903 Node* mem = in(MemNode::Memory);
904 Node* adr = in(MemNode::Address);
905 return phase->transform(new LoadVectorNode(ctr, mem, adr, adr_type(), vect_type()));
906 }
907 }
908 }
909 return NULL;
910 }
911
912 Node* StoreVectorMaskedNode::Ideal(PhaseGVN* phase, bool can_reshape) {
913 if (!in(4)->is_top() && in(4)->Opcode() == Op_VectorMaskGen) {
914 Node* mask_len = in(4)->in(1);
915 const TypeLong* ty = phase->type(mask_len)->isa_long();
916 if (ty && ty->is_con()) {
917 BasicType mask_bt = Matcher::vector_element_basic_type(in(4));
918 int load_sz = type2aelembytes(mask_bt) * ty->get_con();
919 assert(load_sz <= MaxVectorSize, "Unexpected store size");
920 if (load_sz == MaxVectorSize) {
921 Node* ctr = in(MemNode::Control);
922 Node* mem = in(MemNode::Memory);
923 Node* adr = in(MemNode::Address);
924 Node* val = in(MemNode::ValueIn);
925 return phase->transform(new StoreVectorNode(ctr, mem, adr, adr_type(), val));
926 }
927 }
928 }
929 return NULL;
930 }
931
932 int ExtractNode::opcode(BasicType bt) {
933 switch (bt) {
934 case T_BOOLEAN: return Op_ExtractUB;
935 case T_BYTE: return Op_ExtractB;
936 case T_CHAR: return Op_ExtractC;
937 case T_SHORT: return Op_ExtractS;
938 case T_INT: return Op_ExtractI;
939 case T_LONG: return Op_ExtractL;
940 case T_FLOAT: return Op_ExtractF;
941 case T_DOUBLE: return Op_ExtractD;
942 default:
943 assert(false, "wrong type: %s", type2name(bt));
944 return 0;
945 }
946 }
947
948 // Extract a scalar element of vector.
949 Node* ExtractNode::make(Node* v, uint position, BasicType bt) {
950 assert((int)position < Matcher::max_vector_size(bt), "pos in range");
951 ConINode* pos = ConINode::make((int)position);
952 switch (bt) {
953 case T_BOOLEAN: return new ExtractUBNode(v, pos);
954 case T_BYTE: return new ExtractBNode(v, pos);
955 case T_CHAR: return new ExtractCNode(v, pos);
956 case T_SHORT: return new ExtractSNode(v, pos);
957 case T_INT: return new ExtractINode(v, pos);
958 case T_LONG: return new ExtractLNode(v, pos);
959 case T_FLOAT: return new ExtractFNode(v, pos);
960 case T_DOUBLE: return new ExtractDNode(v, pos);
961 default:
962 assert(false, "wrong type: %s", type2name(bt));
963 return NULL;
964 }
965 }
966
967 int ReductionNode::opcode(int opc, BasicType bt) {
968 int vopc = opc;
969 switch (opc) {
970 case Op_AddI:
971 switch (bt) {
972 case T_BOOLEAN:
973 case T_CHAR: return 0;
974 case T_BYTE:
975 case T_SHORT:
976 case T_INT:
977 vopc = Op_AddReductionVI;
978 break;
979 default: ShouldNotReachHere(); return 0;
980 }
981 break;
982 case Op_AddL:
983 assert(bt == T_LONG, "must be");
984 vopc = Op_AddReductionVL;
985 break;
986 case Op_AddF:
987 assert(bt == T_FLOAT, "must be");
988 vopc = Op_AddReductionVF;
989 break;
990 case Op_AddD:
991 assert(bt == T_DOUBLE, "must be");
992 vopc = Op_AddReductionVD;
993 break;
994 case Op_MulI:
995 switch (bt) {
996 case T_BOOLEAN:
997 case T_CHAR: return 0;
998 case T_BYTE:
999 case T_SHORT:
1000 case T_INT:
1001 vopc = Op_MulReductionVI;
1002 break;
1003 default: ShouldNotReachHere(); return 0;
1004 }
1005 break;
1006 case Op_MulL:
1007 assert(bt == T_LONG, "must be");
1008 vopc = Op_MulReductionVL;
1009 break;
1010 case Op_MulF:
1011 assert(bt == T_FLOAT, "must be");
1012 vopc = Op_MulReductionVF;
1013 break;
1014 case Op_MulD:
1015 assert(bt == T_DOUBLE, "must be");
1016 vopc = Op_MulReductionVD;
1017 break;
1018 case Op_MinI:
1019 switch (bt) {
1020 case T_BOOLEAN:
1021 case T_CHAR: return 0;
1022 case T_BYTE:
1023 case T_SHORT:
1024 case T_INT:
1025 vopc = Op_MinReductionV;
1026 break;
1027 default: ShouldNotReachHere(); return 0;
1028 }
1029 break;
1030 case Op_MinL:
1031 assert(bt == T_LONG, "must be");
1032 vopc = Op_MinReductionV;
1033 break;
1034 case Op_MinF:
1035 assert(bt == T_FLOAT, "must be");
1036 vopc = Op_MinReductionV;
1037 break;
1038 case Op_MinD:
1039 assert(bt == T_DOUBLE, "must be");
1040 vopc = Op_MinReductionV;
1041 break;
1042 case Op_MaxI:
1043 switch (bt) {
1044 case T_BOOLEAN:
1045 case T_CHAR: return 0;
1046 case T_BYTE:
1047 case T_SHORT:
1048 case T_INT:
1049 vopc = Op_MaxReductionV;
1050 break;
1051 default: ShouldNotReachHere(); return 0;
1052 }
1053 break;
1054 case Op_MaxL:
1055 assert(bt == T_LONG, "must be");
1056 vopc = Op_MaxReductionV;
1057 break;
1058 case Op_MaxF:
1059 assert(bt == T_FLOAT, "must be");
1060 vopc = Op_MaxReductionV;
1061 break;
1062 case Op_MaxD:
1063 assert(bt == T_DOUBLE, "must be");
1064 vopc = Op_MaxReductionV;
1065 break;
1066 case Op_AndI:
1067 switch (bt) {
1068 case T_BOOLEAN:
1069 case T_CHAR: return 0;
1070 case T_BYTE:
1071 case T_SHORT:
1072 case T_INT:
1073 vopc = Op_AndReductionV;
1074 break;
1075 default: ShouldNotReachHere(); return 0;
1076 }
1077 break;
1078 case Op_AndL:
1079 assert(bt == T_LONG, "must be");
1080 vopc = Op_AndReductionV;
1081 break;
1082 case Op_OrI:
1083 switch(bt) {
1084 case T_BOOLEAN:
1085 case T_CHAR: return 0;
1086 case T_BYTE:
1087 case T_SHORT:
1088 case T_INT:
1089 vopc = Op_OrReductionV;
1090 break;
1091 default: ShouldNotReachHere(); return 0;
1092 }
1093 break;
1094 case Op_OrL:
1095 assert(bt == T_LONG, "must be");
1096 vopc = Op_OrReductionV;
1097 break;
1098 case Op_XorI:
1099 switch(bt) {
1100 case T_BOOLEAN:
1101 case T_CHAR: return 0;
1102 case T_BYTE:
1103 case T_SHORT:
1104 case T_INT:
1105 vopc = Op_XorReductionV;
1106 break;
1107 default: ShouldNotReachHere(); return 0;
1108 }
1109 break;
1110 case Op_XorL:
1111 assert(bt == T_LONG, "must be");
1112 vopc = Op_XorReductionV;
1113 break;
1114 default:
1115 break;
1116 }
1117 return vopc;
1118 }
1119
1120 // Return the appropriate reduction node.
1121 ReductionNode* ReductionNode::make(int opc, Node *ctrl, Node* n1, Node* n2, BasicType bt) {
1122
1123 int vopc = opcode(opc, bt);
1124
1125 // This method should not be called for unimplemented vectors.
1126 guarantee(vopc != opc, "Vector for '%s' is not implemented", NodeClassNames[opc]);
1127
1128 switch (vopc) {
1129 case Op_AddReductionVI: return new AddReductionVINode(ctrl, n1, n2);
1130 case Op_AddReductionVL: return new AddReductionVLNode(ctrl, n1, n2);
1131 case Op_AddReductionVF: return new AddReductionVFNode(ctrl, n1, n2);
1132 case Op_AddReductionVD: return new AddReductionVDNode(ctrl, n1, n2);
1133 case Op_MulReductionVI: return new MulReductionVINode(ctrl, n1, n2);
1134 case Op_MulReductionVL: return new MulReductionVLNode(ctrl, n1, n2);
1135 case Op_MulReductionVF: return new MulReductionVFNode(ctrl, n1, n2);
1136 case Op_MulReductionVD: return new MulReductionVDNode(ctrl, n1, n2);
1137 case Op_MinReductionV: return new MinReductionVNode(ctrl, n1, n2);
1138 case Op_MaxReductionV: return new MaxReductionVNode(ctrl, n1, n2);
1139 case Op_AndReductionV: return new AndReductionVNode(ctrl, n1, n2);
1140 case Op_OrReductionV: return new OrReductionVNode(ctrl, n1, n2);
1141 case Op_XorReductionV: return new XorReductionVNode(ctrl, n1, n2);
1142 default:
1143 assert(false, "unknown node: %s", NodeClassNames[vopc]);
1144 return NULL;
1145 }
1146 }
1147
1148 Node* VectorLoadMaskNode::Identity(PhaseGVN* phase) {
1149 BasicType out_bt = type()->is_vect()->element_basic_type();
1150 if (!Matcher::has_predicated_vectors() && out_bt == T_BOOLEAN) {
1151 return in(1); // redundant conversion
1152 }
1153
1154 return this;
1155 }
1156
1157 Node* VectorStoreMaskNode::Identity(PhaseGVN* phase) {
1158 // Identity transformation on boolean vectors.
1159 // VectorStoreMask (VectorLoadMask bv) elem_size ==> bv
1160 // vector[n]{bool} => vector[n]{t} => vector[n]{bool}
1161 if (in(1)->Opcode() == Op_VectorLoadMask) {
1162 return in(1)->in(1);
1163 }
1164 return this;
1165 }
1166
1167 VectorStoreMaskNode* VectorStoreMaskNode::make(PhaseGVN& gvn, Node* in, BasicType in_type, uint num_elem) {
1168 assert(in->bottom_type()->isa_vect(), "sanity");
1169 const TypeVect* vt = TypeVect::make(T_BOOLEAN, num_elem);
1170 int elem_size = type2aelembytes(in_type);
1171 return new VectorStoreMaskNode(in, gvn.intcon(elem_size), vt);
1172 }
1173
1174 VectorCastNode* VectorCastNode::make(int vopc, Node* n1, BasicType bt, uint vlen) {
1175 const TypeVect* vt = TypeVect::make(bt, vlen);
1176 switch (vopc) {
1177 case Op_VectorCastB2X: return new VectorCastB2XNode(n1, vt);
1178 case Op_VectorCastS2X: return new VectorCastS2XNode(n1, vt);
1179 case Op_VectorCastI2X: return new VectorCastI2XNode(n1, vt);
1180 case Op_VectorCastL2X: return new VectorCastL2XNode(n1, vt);
1181 case Op_VectorCastF2X: return new VectorCastF2XNode(n1, vt);
1182 case Op_VectorCastD2X: return new VectorCastD2XNode(n1, vt);
1183 case Op_VectorUCastB2X: return new VectorUCastB2XNode(n1, vt);
1184 case Op_VectorUCastS2X: return new VectorUCastS2XNode(n1, vt);
1185 case Op_VectorUCastI2X: return new VectorUCastI2XNode(n1, vt);
1186 default:
1187 assert(false, "unknown node: %s", NodeClassNames[vopc]);
1188 return NULL;
1189 }
1190 }
1191
1192 int VectorCastNode::opcode(BasicType bt, bool is_signed) {
1193 assert((is_integral_type(bt) && bt != T_LONG) || is_signed, "");
1194 switch (bt) {
1195 case T_BYTE: return is_signed ? Op_VectorCastB2X : Op_VectorUCastB2X;
1196 case T_SHORT: return is_signed ? Op_VectorCastS2X : Op_VectorUCastS2X;
1197 case T_INT: return is_signed ? Op_VectorCastI2X : Op_VectorUCastI2X;
1198 case T_LONG: return Op_VectorCastL2X;
1199 case T_FLOAT: return Op_VectorCastF2X;
1200 case T_DOUBLE: return Op_VectorCastD2X;
1201 default:
1202 assert(false, "unknown type: %s", type2name(bt));
1203 return 0;
1204 }
1205 }
1206
1207 Node* VectorCastNode::Identity(PhaseGVN* phase) {
1208 if (!in(1)->is_top()) {
1209 BasicType in_bt = in(1)->bottom_type()->is_vect()->element_basic_type();
1210 BasicType out_bt = vect_type()->element_basic_type();
1211 if (in_bt == out_bt) {
1212 return in(1); // redundant cast
1213 }
1214 }
1215 return this;
1216 }
1217
1218 Node* ReductionNode::make_reduction_input(PhaseGVN& gvn, int opc, BasicType bt) {
1219 int vopc = opcode(opc, bt);
1220 guarantee(vopc != opc, "Vector reduction for '%s' is not implemented", NodeClassNames[opc]);
1221
1222 switch (vopc) {
1223 case Op_AndReductionV:
1224 switch (bt) {
1225 case T_BYTE:
1226 case T_SHORT:
1227 case T_INT:
1228 return gvn.makecon(TypeInt::MINUS_1);
1229 case T_LONG:
1230 return gvn.makecon(TypeLong::MINUS_1);
1231 default:
1232 fatal("Missed vector creation for '%s' as the basic type is not correct.", NodeClassNames[vopc]);
1233 return NULL;
1234 }
1235 break;
1236 case Op_AddReductionVI: // fallthrough
1237 case Op_AddReductionVL: // fallthrough
1238 case Op_AddReductionVF: // fallthrough
1239 case Op_AddReductionVD:
1240 case Op_OrReductionV:
1241 case Op_XorReductionV:
1242 return gvn.zerocon(bt);
1243 case Op_MulReductionVI:
1244 return gvn.makecon(TypeInt::ONE);
1245 case Op_MulReductionVL:
1246 return gvn.makecon(TypeLong::ONE);
1247 case Op_MulReductionVF:
1248 return gvn.makecon(TypeF::ONE);
1249 case Op_MulReductionVD:
1250 return gvn.makecon(TypeD::ONE);
1251 case Op_MinReductionV:
1252 switch (bt) {
1253 case T_BYTE:
1254 return gvn.makecon(TypeInt::make(max_jbyte));
1255 case T_SHORT:
1256 return gvn.makecon(TypeInt::make(max_jshort));
1257 case T_INT:
1258 return gvn.makecon(TypeInt::MAX);
1259 case T_LONG:
1260 return gvn.makecon(TypeLong::MAX);
1261 case T_FLOAT:
1262 return gvn.makecon(TypeF::POS_INF);
1263 case T_DOUBLE:
1264 return gvn.makecon(TypeD::POS_INF);
1265 default: Unimplemented(); return NULL;
1266 }
1267 break;
1268 case Op_MaxReductionV:
1269 switch (bt) {
1270 case T_BYTE:
1271 return gvn.makecon(TypeInt::make(min_jbyte));
1272 case T_SHORT:
1273 return gvn.makecon(TypeInt::make(min_jshort));
1274 case T_INT:
1275 return gvn.makecon(TypeInt::MIN);
1276 case T_LONG:
1277 return gvn.makecon(TypeLong::MIN);
1278 case T_FLOAT:
1279 return gvn.makecon(TypeF::NEG_INF);
1280 case T_DOUBLE:
1281 return gvn.makecon(TypeD::NEG_INF);
1282 default: Unimplemented(); return NULL;
1283 }
1284 break;
1285 default:
1286 fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
1287 return NULL;
1288 }
1289 }
1290
1291 bool ReductionNode::implemented(int opc, uint vlen, BasicType bt) {
1292 if (is_java_primitive(bt) &&
1293 (vlen > 1) && is_power_of_2(vlen) &&
1294 Matcher::vector_size_supported(bt, vlen)) {
1295 int vopc = ReductionNode::opcode(opc, bt);
1296 return vopc != opc && Matcher::match_rule_supported_vector(vopc, vlen, bt);
1297 }
1298 return false;
1299 }
1300
1301 MacroLogicVNode* MacroLogicVNode::make(PhaseGVN& gvn, Node* in1, Node* in2, Node* in3,
1302 Node* mask, uint truth_table, const TypeVect* vt) {
1303 assert(truth_table <= 0xFF, "invalid");
1304 assert(in1->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
1305 assert(in2->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
1306 assert(in3->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
1307 assert(!mask || mask->bottom_type()->isa_vectmask(), "predicated register type expected");
1308 Node* fn = gvn.intcon(truth_table);
1309 return new MacroLogicVNode(in1, in2, in3, fn, mask, vt);
1310 }
1311
1312 Node* VectorNode::degenerate_vector_rotate(Node* src, Node* cnt, bool is_rotate_left,
1313 int vlen, BasicType bt, PhaseGVN* phase) {
1314 assert(is_integral_type(bt), "sanity");
1315 const TypeVect* vt = TypeVect::make(bt, vlen);
1316
1317 int shift_mask = (type2aelembytes(bt) * 8) - 1;
1318 int shiftLOpc = (bt == T_LONG) ? Op_LShiftL : Op_LShiftI;
1319 auto urshiftopc = [=]() {
1320 switch(bt) {
1321 case T_INT: return Op_URShiftI;
1322 case T_LONG: return Op_URShiftL;
1323 case T_BYTE: return Op_URShiftB;
1324 case T_SHORT: return Op_URShiftS;
1325 default: return (Opcodes)0;
1326 }
1327 };
1328 int shiftROpc = urshiftopc();
1329
1330 // Compute shift values for right rotation and
1331 // later swap them in case of left rotation.
1332 Node* shiftRCnt = NULL;
1333 Node* shiftLCnt = NULL;
1334 const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
1335 bool is_binary_vector_op = false;
1336 if (cnt_type && cnt_type->is_con()) {
1337 // Constant shift.
1338 int shift = cnt_type->get_con() & shift_mask;
1339 shiftRCnt = phase->intcon(shift);
1340 shiftLCnt = phase->intcon(shift_mask + 1 - shift);
1341 } else if (VectorNode::is_invariant_vector(cnt)) {
1342 // Scalar variable shift, handle replicates generated by auto vectorizer.
1343 cnt = cnt->in(1);
1344 if (bt == T_LONG) {
1345 // Shift count vector for Rotate vector has long elements too.
1346 if (cnt->Opcode() == Op_ConvI2L) {
1347 cnt = cnt->in(1);
1348 } else {
1349 assert(cnt->bottom_type()->isa_long() &&
1350 cnt->bottom_type()->is_long()->is_con(), "Long constant expected");
1351 cnt = phase->transform(new ConvL2INode(cnt));
1352 }
1353 }
1354 shiftRCnt = phase->transform(new AndINode(cnt, phase->intcon(shift_mask)));
1355 shiftLCnt = phase->transform(new SubINode(phase->intcon(shift_mask + 1), shiftRCnt));
1356 } else {
1357 // Variable vector rotate count.
1358 assert(Matcher::supports_vector_variable_shifts(), "");
1359
1360 int subVopc = 0;
1361 int addVopc = 0;
1362 Node* shift_mask_node = NULL;
1363 Node* const_one_node = NULL;
1364
1365 assert(cnt->bottom_type()->isa_vect(), "Unexpected shift");
1366 const Type* elem_ty = Type::get_const_basic_type(bt);
1367
1368 if (bt == T_LONG) {
1369 shift_mask_node = phase->longcon(shift_mask);
1370 const_one_node = phase->longcon(1L);
1371 subVopc = VectorNode::opcode(Op_SubL, bt);
1372 addVopc = VectorNode::opcode(Op_AddL, bt);
1373 } else {
1374 shift_mask_node = phase->intcon(shift_mask);
1375 const_one_node = phase->intcon(1);
1376 subVopc = VectorNode::opcode(Op_SubI, bt);
1377 addVopc = VectorNode::opcode(Op_AddI, bt);
1378 }
1379 Node* vector_mask = phase->transform(VectorNode::scalar2vector(shift_mask_node, vlen, elem_ty));
1380 Node* vector_one = phase->transform(VectorNode::scalar2vector(const_one_node, vlen, elem_ty));
1381
1382 shiftRCnt = cnt;
1383 shiftRCnt = phase->transform(VectorNode::make(Op_AndV, shiftRCnt, vector_mask, vt));
1384 vector_mask = phase->transform(VectorNode::make(addVopc, vector_one, vector_mask, vt));
1385 shiftLCnt = phase->transform(VectorNode::make(subVopc, vector_mask, shiftRCnt, vt));
1386 is_binary_vector_op = true;
1387 }
1388
1389 // Swap the computed left and right shift counts.
1390 if (is_rotate_left) {
1391 swap(shiftRCnt,shiftLCnt);
1392 }
1393
1394 if (!is_binary_vector_op) {
1395 shiftLCnt = phase->transform(new LShiftCntVNode(shiftLCnt, vt));
1396 shiftRCnt = phase->transform(new RShiftCntVNode(shiftRCnt, vt));
1397 }
1398
1399 return new OrVNode(phase->transform(VectorNode::make(shiftLOpc, src, shiftLCnt, vlen, bt, is_binary_vector_op)),
1400 phase->transform(VectorNode::make(shiftROpc, src, shiftRCnt, vlen, bt, is_binary_vector_op)),
1401 vt);
1402 }
1403
1404 Node* RotateLeftVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1405 int vlen = length();
1406 BasicType bt = vect_type()->element_basic_type();
1407 if ((!in(2)->is_Con() && !Matcher::supports_vector_variable_rotates()) ||
1408 !Matcher::match_rule_supported_vector(Op_RotateLeftV, vlen, bt)) {
1409 return VectorNode::degenerate_vector_rotate(in(1), in(2), true, vlen, bt, phase);
1410 }
1411 return NULL;
1412 }
1413
1414 Node* RotateRightVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1415 int vlen = length();
1416 BasicType bt = vect_type()->element_basic_type();
1417 if ((!in(2)->is_Con() && !Matcher::supports_vector_variable_rotates()) ||
1418 !Matcher::match_rule_supported_vector(Op_RotateRightV, vlen, bt)) {
1419 return VectorNode::degenerate_vector_rotate(in(1), in(2), false, vlen, bt, phase);
1420 }
1421 return NULL;
1422 }
1423
1424 #ifndef PRODUCT
1425 void VectorMaskCmpNode::dump_spec(outputStream *st) const {
1426 st->print(" %d #", _predicate); _type->dump_on(st);
1427 }
1428 #endif // PRODUCT
1429
1430 Node* VectorReinterpretNode::Identity(PhaseGVN *phase) {
1431 Node* n = in(1);
1432 if (n->Opcode() == Op_VectorReinterpret) {
1433 // "VectorReinterpret (VectorReinterpret node) ==> node" if:
1434 // 1) Types of 'node' and 'this' are identical
1435 // 2) Truncations are not introduced by the first VectorReinterpret
1436 if (Type::cmp(bottom_type(), n->in(1)->bottom_type()) == 0 &&
1437 length_in_bytes() <= n->bottom_type()->is_vect()->length_in_bytes()) {
1438 return n->in(1);
1439 }
1440 }
1441 return this;
1442 }
1443
1444 Node* VectorInsertNode::make(Node* vec, Node* new_val, int position) {
1445 assert(position < (int)vec->bottom_type()->is_vect()->length(), "pos in range");
1446 ConINode* pos = ConINode::make(position);
1447 return new VectorInsertNode(vec, new_val, pos, vec->bottom_type()->is_vect());
1448 }
1449
1450 Node* VectorUnboxNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1451 Node* n = obj()->uncast();
1452 if (EnableVectorReboxing && n->Opcode() == Op_VectorBox) {
1453 if (Type::cmp(bottom_type(), n->in(VectorBoxNode::Value)->bottom_type()) == 0) {
1454 // Handled by VectorUnboxNode::Identity()
1455 } else {
1456 VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
1457 ciKlass* vbox_klass = vbox->box_type()->klass();
1458 const TypeVect* in_vt = vbox->vec_type();
1459 const TypeVect* out_vt = type()->is_vect();
1460
1461 if (in_vt->length() == out_vt->length()) {
1462 Node* value = vbox->in(VectorBoxNode::Value);
1463
1464 bool is_vector_mask = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
1465 bool is_vector_shuffle = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
1466 if (is_vector_mask) {
1467 const TypeVect* vmask_type = TypeVect::makemask(out_vt->element_basic_type(), out_vt->length());
1468 if (in_vt->length_in_bytes() == out_vt->length_in_bytes() &&
1469 Matcher::match_rule_supported_vector(Op_VectorMaskCast, out_vt->length(), out_vt->element_basic_type())) {
1470 // Apply "VectorUnbox (VectorBox vmask) ==> VectorMaskCast (vmask)"
1471 // directly. This could avoid the transformation ordering issue from
1472 // "VectorStoreMask (VectorLoadMask vmask) => vmask".
1473 return new VectorMaskCastNode(value, vmask_type);
1474 }
1475 // VectorUnbox (VectorBox vmask) ==> VectorLoadMask (VectorStoreMask vmask)
1476 value = phase->transform(VectorStoreMaskNode::make(*phase, value, in_vt->element_basic_type(), in_vt->length()));
1477 return new VectorLoadMaskNode(value, vmask_type);
1478 } else if (is_vector_shuffle) {
1479 if (!is_shuffle_to_vector()) {
1480 // VectorUnbox (VectorBox vshuffle) ==> VectorLoadShuffle vshuffle
1481 return new VectorLoadShuffleNode(value, out_vt);
1482 }
1483 } else {
1484 // Vector type mismatch is only supported for masks and shuffles, but sometimes it happens in pathological cases.
1485 }
1486 } else {
1487 // Vector length mismatch.
1488 // Sometimes happen in pathological cases (e.g., when unboxing happens in effectively dead code).
1489 }
1490 }
1491 }
1492 return NULL;
1493 }
1494
1495 Node* VectorUnboxNode::Identity(PhaseGVN* phase) {
1496 Node* n = obj()->uncast();
1497 if (EnableVectorReboxing && n->Opcode() == Op_VectorBox) {
1498 if (Type::cmp(bottom_type(), n->in(VectorBoxNode::Value)->bottom_type()) == 0) {
1499 return n->in(VectorBoxNode::Value); // VectorUnbox (VectorBox v) ==> v
1500 } else {
1501 // Handled by VectorUnboxNode::Ideal().
1502 }
1503 }
1504 return this;
1505 }
1506
1507 const TypeFunc* VectorBoxNode::vec_box_type(const TypeInstPtr* box_type) {
1508 const Type** fields = TypeTuple::fields(0);
1509 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms, fields);
1510
1511 fields = TypeTuple::fields(1);
1512 fields[TypeFunc::Parms+0] = box_type;
1513 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
1514
1515 return TypeFunc::make(domain, range);
1516 }
1517
1518 Node* ShiftVNode::Identity(PhaseGVN* phase) {
1519 Node* in2 = in(2);
1520 // Shift by ZERO does nothing
1521 if (is_vshift_cnt(in2) && phase->find_int_type(in2->in(1)) == TypeInt::ZERO) {
1522 return in(1);
1523 }
1524 return this;
1525 }
1526
1527 Node* VectorMaskGenNode::make(Node* length, BasicType mask_bt) {
1528 int max_vector = Matcher::max_vector_size(mask_bt);
1529 const TypeVectMask* t_vmask = TypeVectMask::make(mask_bt, max_vector);
1530 return new VectorMaskGenNode(length, t_vmask);
1531 }
1532
1533 Node* VectorMaskOpNode::make(Node* mask, const Type* ty, int mopc) {
1534 switch(mopc) {
1535 case Op_VectorMaskTrueCount:
1536 return new VectorMaskTrueCountNode(mask, ty);
1537 case Op_VectorMaskLastTrue:
1538 return new VectorMaskLastTrueNode(mask, ty);
1539 case Op_VectorMaskFirstTrue:
1540 return new VectorMaskFirstTrueNode(mask, ty);
1541 case Op_VectorMaskToLong:
1542 return new VectorMaskToLongNode(mask, ty);
1543 default:
1544 assert(false, "Unhandled operation");
1545 }
1546 return NULL;
1547 }
1548
1549 Node* VectorMaskToLongNode::Identity(PhaseGVN* phase) {
1550 if (in(1)->Opcode() == Op_VectorLongToMask) {
1551 return in(1)->in(1);
1552 }
1553 return this;
1554 }
1555
1556
1557 Node* VectorMaskCastNode::makeCastNode(PhaseGVN* phase, Node* src, const TypeVect* dst_type) {
1558 const TypeVect* src_type = src->bottom_type()->is_vect();
1559 assert(src_type->length() == dst_type->length(), "");
1560
1561 int num_elem = src_type->length();
1562 BasicType elem_bt_from = src_type->element_basic_type();
1563 BasicType elem_bt_to = dst_type->element_basic_type();
1564
1565 if (dst_type->isa_vectmask() == NULL && src_type->isa_vectmask() == NULL &&
1566 type2aelembytes(elem_bt_from) != type2aelembytes(elem_bt_to)) {
1567
1568 Node* op = src;
1569 BasicType new_elem_bt_from = elem_bt_from;
1570 BasicType new_elem_bt_to = elem_bt_to;
1571 if (is_floating_point_type(elem_bt_from)) {
1572 new_elem_bt_from = elem_bt_from == T_FLOAT ? T_INT : T_LONG;
1573 }
1574 if (is_floating_point_type(elem_bt_to)) {
1575 new_elem_bt_to = elem_bt_to == T_FLOAT ? T_INT : T_LONG;
1576 }
1577
1578 // Special handling for casting operation involving floating point types.
1579 // Case A) F -> X := F -> VectorMaskCast (F->I/L [NOP]) -> VectorCast[I/L]2X
1580 // Case B) X -> F := X -> VectorCastX2[I/L] -> VectorMaskCast ([I/L]->F [NOP])
1581 // Case C) F -> F := VectorMaskCast (F->I/L [NOP]) -> VectorCast[I/L]2[L/I] -> VectotMaskCast (L/I->F [NOP])
1582
1583 if (new_elem_bt_from != elem_bt_from) {
1584 const TypeVect* new_src_type = TypeVect::makemask(new_elem_bt_from, num_elem);
1585 op = phase->transform(new VectorMaskCastNode(op, new_src_type));
1586 }
1587
1588 op = phase->transform(VectorCastNode::make(VectorCastNode::opcode(new_elem_bt_from), op, new_elem_bt_to, num_elem));
1589
1590 if (new_elem_bt_to != elem_bt_to) {
1591 op = phase->transform(new VectorMaskCastNode(op, dst_type));
1592 }
1593 return op;
1594 } else {
1595 return new VectorMaskCastNode(src, dst_type);
1596 }
1597 }
1598
1599 Node* VectorLongToMaskNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1600 const TypeVect* dst_type = bottom_type()->is_vect();
1601 if (in(1)->Opcode() == Op_AndL &&
1602 in(1)->in(1)->Opcode() == Op_VectorMaskToLong &&
1603 in(1)->in(2)->bottom_type()->isa_long() &&
1604 in(1)->in(2)->bottom_type()->is_long()->is_con() &&
1605 in(1)->in(2)->bottom_type()->is_long()->get_con() == ((1L << dst_type->length()) - 1)) {
1606 // Different src/dst mask length represents a re-interpretation operation,
1607 // we can however generate a mask casting operation if length matches.
1608 Node* src = in(1)->in(1)->in(1);
1609 if (dst_type->isa_vectmask() == NULL) {
1610 if (src->Opcode() != Op_VectorStoreMask) {
1611 return NULL;
1612 }
1613 src = src->in(1);
1614 }
1615 const TypeVect* src_type = src->bottom_type()->is_vect();
1616 if (src_type->length() == dst_type->length() &&
1617 ((src_type->isa_vectmask() == NULL && dst_type->isa_vectmask() == NULL) ||
1618 (src_type->isa_vectmask() && dst_type->isa_vectmask()))) {
1619 return VectorMaskCastNode::makeCastNode(phase, src, dst_type);
1620 }
1621 }
1622 return NULL;
1623 }
1624
1625 // Generate other vector nodes to implement the masked/non-masked vector negation.
1626 Node* NegVNode::degenerate_integral_negate(PhaseGVN* phase, bool is_predicated) {
1627 const TypeVect* vt = vect_type();
1628 BasicType bt = vt->element_basic_type();
1629 uint vlen = length();
1630
1631 // Transformation for predicated NegVI/L
1632 if (is_predicated) {
1633 // (NegVI/L src m) ==> (AddVI/L (XorV src (ReplicateI/L -1) m) (ReplicateI/L 1) m)
1634 Node* const_minus_one = NULL;
1635 Node* const_one = NULL;
1636 int add_opc;
1637 if (bt == T_LONG) {
1638 const_minus_one = phase->longcon(-1L);
1639 const_one = phase->longcon(1L);
1640 add_opc = Op_AddL;
1641 } else {
1642 const_minus_one = phase->intcon(-1);
1643 const_one = phase->intcon(1);
1644 add_opc = Op_AddI;
1645 }
1646 const_minus_one = phase->transform(VectorNode::scalar2vector(const_minus_one, vlen, Type::get_const_basic_type(bt)));
1647 Node* xorv = VectorNode::make(Op_XorV, in(1), const_minus_one, vt);
1648 xorv->add_req(in(2));
1649 xorv->add_flag(Node::Flag_is_predicated_vector);
1650 phase->transform(xorv);
1651 const_one = phase->transform(VectorNode::scalar2vector(const_one, vlen, Type::get_const_basic_type(bt)));
1652 Node* addv = VectorNode::make(VectorNode::opcode(add_opc, bt), xorv, const_one, vt);
1653 addv->add_req(in(2));
1654 addv->add_flag(Node::Flag_is_predicated_vector);
1655 return addv;
1656 }
1657
1658 // NegVI/L ==> (SubVI/L (ReplicateI/L 0) src)
1659 Node* const_zero = NULL;
1660 int sub_opc;
1661 if (bt == T_LONG) {
1662 const_zero = phase->longcon(0L);
1663 sub_opc = Op_SubL;
1664 } else {
1665 const_zero = phase->intcon(0);
1666 sub_opc = Op_SubI;
1667 }
1668 const_zero = phase->transform(VectorNode::scalar2vector(const_zero, vlen, Type::get_const_basic_type(bt)));
1669 return VectorNode::make(VectorNode::opcode(sub_opc, bt), const_zero, in(1), vt);
1670 }
1671
1672 Node* NegVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1673 BasicType bt = vect_type()->element_basic_type();
1674 uint vlen = length();
1675 int opc = Opcode();
1676 if (is_vector_integral_negate(opc)) {
1677 if (is_predicated_vector()) {
1678 if (!Matcher::match_rule_supported_vector_masked(opc, vlen, bt)) {
1679 return degenerate_integral_negate(phase, true);
1680 }
1681 } else if (!Matcher::match_rule_supported_vector(opc, vlen, bt)) {
1682 return degenerate_integral_negate(phase, false);
1683 }
1684 }
1685 return NULL;
1686 }
1687
1688 #ifndef PRODUCT
1689 void VectorBoxAllocateNode::dump_spec(outputStream *st) const {
1690 CallStaticJavaNode::dump_spec(st);
1691 }
1692
1693 #endif // !PRODUCT