1 /*
2 * Copyright (c) 2007, 2021, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "memory/allocation.inline.hpp"
26 #include "opto/connode.hpp"
27 #include "opto/mulnode.hpp"
28 #include "opto/subnode.hpp"
29 #include "opto/vectornode.hpp"
30 #include "opto/convertnode.hpp"
31 #include "utilities/powerOfTwo.hpp"
32 #include "utilities/globalDefinitions.hpp"
33
34 //------------------------------VectorNode--------------------------------------
35
36 // Return the vector operator for the specified scalar operation
37 // and vector length.
38 int VectorNode::opcode(int sopc, BasicType bt) {
39 switch (sopc) {
40 case Op_AddI:
41 switch (bt) {
42 case T_BOOLEAN:
43 case T_BYTE: return Op_AddVB;
44 case T_CHAR:
45 case T_SHORT: return Op_AddVS;
46 case T_INT: return Op_AddVI;
47 default: return 0;
48 }
49 case Op_AddL: return (bt == T_LONG ? Op_AddVL : 0);
50 case Op_AddF: return (bt == T_FLOAT ? Op_AddVF : 0);
51 case Op_AddD: return (bt == T_DOUBLE ? Op_AddVD : 0);
52
53 case Op_SubI:
54 switch (bt) {
55 case T_BOOLEAN:
56 case T_BYTE: return Op_SubVB;
57 case T_CHAR:
58 case T_SHORT: return Op_SubVS;
59 case T_INT: return Op_SubVI;
60 default: return 0;
61 }
62 case Op_SubL: return (bt == T_LONG ? Op_SubVL : 0);
63 case Op_SubF: return (bt == T_FLOAT ? Op_SubVF : 0);
64 case Op_SubD: return (bt == T_DOUBLE ? Op_SubVD : 0);
65
66 case Op_MulI:
67 switch (bt) {
68 case T_BOOLEAN:return 0;
69 case T_BYTE: return Op_MulVB;
70 case T_CHAR:
71 case T_SHORT: return Op_MulVS;
72 case T_INT: return Op_MulVI;
73 default: return 0;
74 }
75 case Op_MulL: return (bt == T_LONG ? Op_MulVL : 0);
76 case Op_MulF:
77 return (bt == T_FLOAT ? Op_MulVF : 0);
78 case Op_MulD:
79 return (bt == T_DOUBLE ? Op_MulVD : 0);
80 case Op_FmaD:
81 return (bt == T_DOUBLE ? Op_FmaVD : 0);
82 case Op_FmaF:
83 return (bt == T_FLOAT ? Op_FmaVF : 0);
84 case Op_CMoveF:
85 return (bt == T_FLOAT ? Op_CMoveVF : 0);
86 case Op_CMoveD:
87 return (bt == T_DOUBLE ? Op_CMoveVD : 0);
88 case Op_DivF:
89 return (bt == T_FLOAT ? Op_DivVF : 0);
90 case Op_DivD:
91 return (bt == T_DOUBLE ? Op_DivVD : 0);
92 case Op_AbsI:
93 switch (bt) {
94 case T_BOOLEAN:
95 case T_CHAR: return 0; // abs does not make sense for unsigned
96 case T_BYTE: return Op_AbsVB;
97 case T_SHORT: return Op_AbsVS;
98 case T_INT: return Op_AbsVI;
99 default: return 0;
100 }
101 case Op_AbsL:
102 return (bt == T_LONG ? Op_AbsVL : 0);
103 case Op_MinI:
104 switch (bt) {
105 case T_BOOLEAN:
106 case T_CHAR: return 0;
107 case T_BYTE:
108 case T_SHORT:
109 case T_INT: return Op_MinV;
110 default: return 0;
111 }
112 case Op_MinL:
113 return (bt == T_LONG ? Op_MinV : 0);
114 case Op_MinF:
115 return (bt == T_FLOAT ? Op_MinV : 0);
116 case Op_MinD:
117 return (bt == T_DOUBLE ? Op_MinV : 0);
118 case Op_MaxI:
119 switch (bt) {
120 case T_BOOLEAN:
121 case T_CHAR: return 0;
122 case T_BYTE:
123 case T_SHORT:
124 case T_INT: return Op_MaxV;
125 default: return 0;
126 }
127 case Op_MaxL:
128 return (bt == T_LONG ? Op_MaxV : 0);
129 case Op_MaxF:
130 return (bt == T_FLOAT ? Op_MaxV : 0);
131 case Op_MaxD:
132 return (bt == T_DOUBLE ? Op_MaxV : 0);
133 case Op_AbsF:
134 return (bt == T_FLOAT ? Op_AbsVF : 0);
135 case Op_AbsD:
136 return (bt == T_DOUBLE ? Op_AbsVD : 0);
137 case Op_NegI:
138 return (bt == T_INT ? Op_NegVI : 0);
139 case Op_NegF:
140 return (bt == T_FLOAT ? Op_NegVF : 0);
141 case Op_NegD:
142 return (bt == T_DOUBLE ? Op_NegVD : 0);
143 case Op_RoundDoubleMode:
144 return (bt == T_DOUBLE ? Op_RoundDoubleModeV : 0);
145 case Op_RotateLeft:
146 return (is_integral_type(bt) ? Op_RotateLeftV : 0);
147 case Op_RotateRight:
148 return (is_integral_type(bt) ? Op_RotateRightV : 0);
149 case Op_SqrtF:
150 return (bt == T_FLOAT ? Op_SqrtVF : 0);
151 case Op_SqrtD:
152 return (bt == T_DOUBLE ? Op_SqrtVD : 0);
153 case Op_PopCountI:
154 // Unimplemented for subword types since bit count changes
155 // depending on size of lane (and sign bit).
156 return (bt == T_INT ? Op_PopCountVI : 0);
157 case Op_LShiftI:
158 switch (bt) {
159 case T_BOOLEAN:
160 case T_BYTE: return Op_LShiftVB;
161 case T_CHAR:
162 case T_SHORT: return Op_LShiftVS;
163 case T_INT: return Op_LShiftVI;
164 default: return 0;
165 }
166 case Op_LShiftL:
167 return (bt == T_LONG ? Op_LShiftVL : 0);
168 case Op_RShiftI:
169 switch (bt) {
170 case T_BOOLEAN:return Op_URShiftVB; // boolean is unsigned value
171 case T_CHAR: return Op_URShiftVS; // char is unsigned value
172 case T_BYTE: return Op_RShiftVB;
173 case T_SHORT: return Op_RShiftVS;
174 case T_INT: return Op_RShiftVI;
175 default: return 0;
176 }
177 case Op_RShiftL:
178 return (bt == T_LONG ? Op_RShiftVL : 0);
179 case Op_URShiftB:
180 return (bt == T_BYTE ? Op_URShiftVB : 0);
181 case Op_URShiftS:
182 return (bt == T_SHORT ? Op_URShiftVS : 0);
183 case Op_URShiftI:
184 switch (bt) {
185 case T_BOOLEAN:return Op_URShiftVB;
186 case T_CHAR: return Op_URShiftVS;
187 case T_BYTE:
188 case T_SHORT: return 0; // Vector logical right shift for signed short
189 // values produces incorrect Java result for
190 // negative data because java code should convert
191 // a short value into int value with sign
192 // extension before a shift.
193 case T_INT: return Op_URShiftVI;
194 default: return 0;
195 }
196 case Op_URShiftL:
197 return (bt == T_LONG ? Op_URShiftVL : 0);
198 case Op_AndI:
199 case Op_AndL:
200 return Op_AndV;
201 case Op_OrI:
202 case Op_OrL:
203 return Op_OrV;
204 case Op_XorI:
205 case Op_XorL:
206 return Op_XorV;
207
208 case Op_LoadB:
209 case Op_LoadUB:
210 case Op_LoadUS:
211 case Op_LoadS:
212 case Op_LoadI:
213 case Op_LoadL:
214 case Op_LoadF:
215 case Op_LoadD:
216 return Op_LoadVector;
217
218 case Op_StoreB:
219 case Op_StoreC:
220 case Op_StoreI:
221 case Op_StoreL:
222 case Op_StoreF:
223 case Op_StoreD:
224 return Op_StoreVector;
225 case Op_MulAddS2I:
226 return Op_MulAddVS2VI;
227
228 default:
229 return 0; // Unimplemented
230 }
231 }
232
233 int VectorNode::replicate_opcode(BasicType bt) {
234 switch(bt) {
235 case T_BOOLEAN:
236 case T_BYTE:
237 return Op_ReplicateB;
238 case T_SHORT:
239 case T_CHAR:
240 return Op_ReplicateS;
241 case T_INT:
242 return Op_ReplicateI;
243 case T_LONG:
244 return Op_ReplicateL;
245 case T_FLOAT:
246 return Op_ReplicateF;
247 case T_DOUBLE:
248 return Op_ReplicateD;
249 default:
250 assert(false, "wrong type: %s", type2name(bt));
251 return 0;
252 }
253 }
254
255 // Also used to check if the code generator
256 // supports the vector operation.
257 bool VectorNode::implemented(int opc, uint vlen, BasicType bt) {
258 if (is_java_primitive(bt) &&
259 (vlen > 1) && is_power_of_2(vlen) &&
260 Matcher::vector_size_supported(bt, vlen)) {
261 int vopc = VectorNode::opcode(opc, bt);
262 // For rotate operation we will do a lazy de-generation into
263 // OrV/LShiftV/URShiftV pattern if the target does not support
264 // vector rotation instruction.
265 if (VectorNode::is_vector_rotate(vopc)) {
266 return is_vector_rotate_supported(vopc, vlen, bt);
267 }
268 return vopc > 0 && Matcher::match_rule_supported_vector(vopc, vlen, bt);
269 }
270 return false;
271 }
272
273 bool VectorNode::is_type_transition_short_to_int(Node* n) {
274 switch (n->Opcode()) {
275 case Op_MulAddS2I:
276 return true;
277 }
278 return false;
279 }
280
281 bool VectorNode::is_type_transition_to_int(Node* n) {
282 return is_type_transition_short_to_int(n);
283 }
284
285 bool VectorNode::is_muladds2i(Node* n) {
286 if (n->Opcode() == Op_MulAddS2I) {
287 return true;
288 }
289 return false;
290 }
291
292 bool VectorNode::is_roundopD(Node* n) {
293 if (n->Opcode() == Op_RoundDoubleMode) {
294 return true;
295 }
296 return false;
297 }
298
299 bool VectorNode::is_vector_rotate_supported(int vopc, uint vlen, BasicType bt) {
300 assert(VectorNode::is_vector_rotate(vopc), "wrong opcode");
301
302 // If target defines vector rotation patterns then no
303 // need for degeneration.
304 if (Matcher::match_rule_supported_vector(vopc, vlen, bt)) {
305 return true;
306 }
307
308 // If target does not support variable shift operations then no point
309 // in creating a rotate vector node since it will not be disintegratable.
310 // Adding a pessimistic check to avoid complex pattern mathing which
311 // may not be full proof.
312 if (!Matcher::supports_vector_variable_shifts()) {
313 return false;
314 }
315
316 // Validate existence of nodes created in case of rotate degeneration.
317 switch (bt) {
318 case T_INT:
319 return Matcher::match_rule_supported_vector(Op_OrV, vlen, bt) &&
320 Matcher::match_rule_supported_vector(Op_LShiftVI, vlen, bt) &&
321 Matcher::match_rule_supported_vector(Op_URShiftVI, vlen, bt);
322 case T_LONG:
323 return Matcher::match_rule_supported_vector(Op_OrV, vlen, bt) &&
324 Matcher::match_rule_supported_vector(Op_LShiftVL, vlen, bt) &&
325 Matcher::match_rule_supported_vector(Op_URShiftVL, vlen, bt);
326 default:
327 assert(false, "not supported: %s", type2name(bt));
328 return false;
329 }
330 }
331
332 bool VectorNode::is_shift_opcode(int opc) {
333 switch (opc) {
334 case Op_LShiftI:
335 case Op_LShiftL:
336 case Op_RShiftI:
337 case Op_RShiftL:
338 case Op_URShiftB:
339 case Op_URShiftS:
340 case Op_URShiftI:
341 case Op_URShiftL:
342 return true;
343 default:
344 return false;
345 }
346 }
347
348 bool VectorNode::is_shift(Node* n) {
349 return is_shift_opcode(n->Opcode());
350 }
351
352 bool VectorNode::is_rotate_opcode(int opc) {
353 switch (opc) {
354 case Op_RotateRight:
355 case Op_RotateLeft:
356 return true;
357 default:
358 return false;
359 }
360 }
361
362 bool VectorNode::is_scalar_rotate(Node* n) {
363 if (is_rotate_opcode(n->Opcode())) {
364 return true;
365 }
366 return false;
367 }
368
369 bool VectorNode::is_vshift_cnt_opcode(int opc) {
370 switch (opc) {
371 case Op_LShiftCntV:
372 case Op_RShiftCntV:
373 return true;
374 default:
375 return false;
376 }
377 }
378
379 bool VectorNode::is_vshift_cnt(Node* n) {
380 return is_vshift_cnt_opcode(n->Opcode());
381 }
382
383 // Check if input is loop invariant vector.
384 bool VectorNode::is_invariant_vector(Node* n) {
385 // Only Replicate vector nodes are loop invariant for now.
386 switch (n->Opcode()) {
387 case Op_ReplicateB:
388 case Op_ReplicateS:
389 case Op_ReplicateI:
390 case Op_ReplicateL:
391 case Op_ReplicateF:
392 case Op_ReplicateD:
393 return true;
394 default:
395 return false;
396 }
397 }
398
399 // [Start, end) half-open range defining which operands are vectors
400 void VectorNode::vector_operands(Node* n, uint* start, uint* end) {
401 switch (n->Opcode()) {
402 case Op_LoadB: case Op_LoadUB:
403 case Op_LoadS: case Op_LoadUS:
404 case Op_LoadI: case Op_LoadL:
405 case Op_LoadF: case Op_LoadD:
406 case Op_LoadP: case Op_LoadN:
407 *start = 0;
408 *end = 0; // no vector operands
409 break;
410 case Op_StoreB: case Op_StoreC:
411 case Op_StoreI: case Op_StoreL:
412 case Op_StoreF: case Op_StoreD:
413 case Op_StoreP: case Op_StoreN:
414 *start = MemNode::ValueIn;
415 *end = MemNode::ValueIn + 1; // 1 vector operand
416 break;
417 case Op_LShiftI: case Op_LShiftL:
418 case Op_RShiftI: case Op_RShiftL:
419 case Op_URShiftI: case Op_URShiftL:
420 *start = 1;
421 *end = 2; // 1 vector operand
422 break;
423 case Op_AddI: case Op_AddL: case Op_AddF: case Op_AddD:
424 case Op_SubI: case Op_SubL: case Op_SubF: case Op_SubD:
425 case Op_MulI: case Op_MulL: case Op_MulF: case Op_MulD:
426 case Op_DivF: case Op_DivD:
427 case Op_AndI: case Op_AndL:
428 case Op_OrI: case Op_OrL:
429 case Op_XorI: case Op_XorL:
430 case Op_MulAddS2I:
431 *start = 1;
432 *end = 3; // 2 vector operands
433 break;
434 case Op_CMoveI: case Op_CMoveL: case Op_CMoveF: case Op_CMoveD:
435 *start = 2;
436 *end = n->req();
437 break;
438 case Op_FmaD:
439 case Op_FmaF:
440 *start = 1;
441 *end = 4; // 3 vector operands
442 break;
443 default:
444 *start = 1;
445 *end = n->req(); // default is all operands
446 }
447 }
448
449 VectorNode* VectorNode::make_mask_node(int vopc, Node* n1, Node* n2, uint vlen, BasicType bt) {
450 guarantee(vopc > 0, "vopc must be > 0");
451 const TypeVect* vmask_type = TypeVect::makemask(bt, vlen);
452 switch (vopc) {
453 case Op_AndV:
454 if (Matcher::match_rule_supported_vector_masked(Op_AndVMask, vlen, bt)) {
455 return new AndVMaskNode(n1, n2, vmask_type);
456 }
457 return new AndVNode(n1, n2, vmask_type);
458 case Op_OrV:
459 if (Matcher::match_rule_supported_vector_masked(Op_OrVMask, vlen, bt)) {
460 return new OrVMaskNode(n1, n2, vmask_type);
461 }
462 return new OrVNode(n1, n2, vmask_type);
463 case Op_XorV:
464 if (Matcher::match_rule_supported_vector_masked(Op_XorVMask, vlen, bt)) {
465 return new XorVMaskNode(n1, n2, vmask_type);
466 }
467 return new XorVNode(n1, n2, vmask_type);
468 default:
469 fatal("Unsupported mask vector creation for '%s'", NodeClassNames[vopc]);
470 return NULL;
471 }
472 }
473
474 // Make a vector node for binary operation
475 VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, const TypeVect* vt, bool is_mask) {
476 // This method should not be called for unimplemented vectors.
477 guarantee(vopc > 0, "vopc must be > 0");
478
479 if (is_mask) {
480 return make_mask_node(vopc, n1, n2, vt->length(), vt->element_basic_type());
481 }
482
483 switch (vopc) {
484 case Op_AddVB: return new AddVBNode(n1, n2, vt);
485 case Op_AddVS: return new AddVSNode(n1, n2, vt);
486 case Op_AddVI: return new AddVINode(n1, n2, vt);
487 case Op_AddVL: return new AddVLNode(n1, n2, vt);
488 case Op_AddVF: return new AddVFNode(n1, n2, vt);
489 case Op_AddVD: return new AddVDNode(n1, n2, vt);
490
491 case Op_SubVB: return new SubVBNode(n1, n2, vt);
492 case Op_SubVS: return new SubVSNode(n1, n2, vt);
493 case Op_SubVI: return new SubVINode(n1, n2, vt);
494 case Op_SubVL: return new SubVLNode(n1, n2, vt);
495 case Op_SubVF: return new SubVFNode(n1, n2, vt);
496 case Op_SubVD: return new SubVDNode(n1, n2, vt);
497
498 case Op_MulVB: return new MulVBNode(n1, n2, vt);
499 case Op_MulVS: return new MulVSNode(n1, n2, vt);
500 case Op_MulVI: return new MulVINode(n1, n2, vt);
501 case Op_MulVL: return new MulVLNode(n1, n2, vt);
502 case Op_MulVF: return new MulVFNode(n1, n2, vt);
503 case Op_MulVD: return new MulVDNode(n1, n2, vt);
504
505 case Op_DivVF: return new DivVFNode(n1, n2, vt);
506 case Op_DivVD: return new DivVDNode(n1, n2, vt);
507
508 case Op_MinV: return new MinVNode(n1, n2, vt);
509 case Op_MaxV: return new MaxVNode(n1, n2, vt);
510
511 case Op_AbsVF: return new AbsVFNode(n1, vt);
512 case Op_AbsVD: return new AbsVDNode(n1, vt);
513 case Op_AbsVB: return new AbsVBNode(n1, vt);
514 case Op_AbsVS: return new AbsVSNode(n1, vt);
515 case Op_AbsVI: return new AbsVINode(n1, vt);
516 case Op_AbsVL: return new AbsVLNode(n1, vt);
517
518 case Op_NegVI: return new NegVINode(n1, vt);
519 case Op_NegVF: return new NegVFNode(n1, vt);
520 case Op_NegVD: return new NegVDNode(n1, vt);
521
522 case Op_SqrtVF: return new SqrtVFNode(n1, vt);
523 case Op_SqrtVD: return new SqrtVDNode(n1, vt);
524
525 case Op_PopCountVI: return new PopCountVINode(n1, vt);
526 case Op_RotateLeftV: return new RotateLeftVNode(n1, n2, vt);
527 case Op_RotateRightV: return new RotateRightVNode(n1, n2, vt);
528
529 case Op_LShiftVB: return new LShiftVBNode(n1, n2, vt);
530 case Op_LShiftVS: return new LShiftVSNode(n1, n2, vt);
531 case Op_LShiftVI: return new LShiftVINode(n1, n2, vt);
532 case Op_LShiftVL: return new LShiftVLNode(n1, n2, vt);
533
534 case Op_RShiftVB: return new RShiftVBNode(n1, n2, vt);
535 case Op_RShiftVS: return new RShiftVSNode(n1, n2, vt);
536 case Op_RShiftVI: return new RShiftVINode(n1, n2, vt);
537 case Op_RShiftVL: return new RShiftVLNode(n1, n2, vt);
538
539 case Op_URShiftVB: return new URShiftVBNode(n1, n2, vt);
540 case Op_URShiftVS: return new URShiftVSNode(n1, n2, vt);
541 case Op_URShiftVI: return new URShiftVINode(n1, n2, vt);
542 case Op_URShiftVL: return new URShiftVLNode(n1, n2, vt);
543
544 case Op_AndV: return new AndVNode(n1, n2, vt);
545 case Op_OrV: return new OrVNode (n1, n2, vt);
546 case Op_XorV: return new XorVNode(n1, n2, vt);
547
548 case Op_RoundDoubleModeV: return new RoundDoubleModeVNode(n1, n2, vt);
549
550 case Op_MulAddVS2VI: return new MulAddVS2VINode(n1, n2, vt);
551 default:
552 fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
553 return NULL;
554 }
555 }
556
557 // Return the vector version of a scalar binary operation node.
558 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, uint vlen, BasicType bt) {
559 const TypeVect* vt = TypeVect::make(bt, vlen);
560 int vopc = VectorNode::opcode(opc, bt);
561 // This method should not be called for unimplemented vectors.
562 guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]);
563 return make(vopc, n1, n2, vt);
564 }
565
566 // Make a vector node for ternary operation
567 VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, Node* n3, const TypeVect* vt) {
568 // This method should not be called for unimplemented vectors.
569 guarantee(vopc > 0, "vopc must be > 0");
570 switch (vopc) {
571 case Op_FmaVD: return new FmaVDNode(n1, n2, n3, vt);
572 case Op_FmaVF: return new FmaVFNode(n1, n2, n3, vt);
573 default:
574 fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
575 return NULL;
576 }
577 }
578
579 // Return the vector version of a scalar ternary operation node.
580 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, Node* n3, uint vlen, BasicType bt) {
581 const TypeVect* vt = TypeVect::make(bt, vlen);
582 int vopc = VectorNode::opcode(opc, bt);
583 // This method should not be called for unimplemented vectors.
584 guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]);
585 return make(vopc, n1, n2, n3, vt);
586 }
587
588 // Scalar promotion
589 VectorNode* VectorNode::scalar2vector(Node* s, uint vlen, const Type* opd_t, bool is_mask) {
590 BasicType bt = opd_t->array_element_basic_type();
591 const TypeVect* vt = opd_t->singleton() ? TypeVect::make(opd_t, vlen, is_mask)
592 : TypeVect::make(bt, vlen, is_mask);
593
594 if (is_mask && Matcher::match_rule_supported_vector(Op_MaskAll, vlen, bt)) {
595 return new MaskAllNode(s, vt);
596 }
597
598 switch (bt) {
599 case T_BOOLEAN:
600 case T_BYTE:
601 return new ReplicateBNode(s, vt);
602 case T_CHAR:
603 case T_SHORT:
604 return new ReplicateSNode(s, vt);
605 case T_INT:
606 return new ReplicateINode(s, vt);
607 case T_LONG:
608 return new ReplicateLNode(s, vt);
609 case T_FLOAT:
610 return new ReplicateFNode(s, vt);
611 case T_DOUBLE:
612 return new ReplicateDNode(s, vt);
613 default:
614 fatal("Type '%s' is not supported for vectors", type2name(bt));
615 return NULL;
616 }
617 }
618
619 VectorNode* VectorNode::shift_count(int opc, Node* cnt, uint vlen, BasicType bt) {
620 // Match shift count type with shift vector type.
621 const TypeVect* vt = TypeVect::make(bt, vlen);
622 switch (opc) {
623 case Op_LShiftI:
624 case Op_LShiftL:
625 return new LShiftCntVNode(cnt, vt);
626 case Op_RShiftI:
627 case Op_RShiftL:
628 case Op_URShiftB:
629 case Op_URShiftS:
630 case Op_URShiftI:
631 case Op_URShiftL:
632 return new RShiftCntVNode(cnt, vt);
633 default:
634 fatal("Missed vector creation for '%s'", NodeClassNames[opc]);
635 return NULL;
636 }
637 }
638
639 bool VectorNode::is_vector_rotate(int opc) {
640 switch (opc) {
641 case Op_RotateLeftV:
642 case Op_RotateRightV:
643 return true;
644 default:
645 return false;
646 }
647 }
648
649 bool VectorNode::is_vector_shift(int opc) {
650 assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode");
651 switch (opc) {
652 case Op_LShiftVB:
653 case Op_LShiftVS:
654 case Op_LShiftVI:
655 case Op_LShiftVL:
656 case Op_RShiftVB:
657 case Op_RShiftVS:
658 case Op_RShiftVI:
659 case Op_RShiftVL:
660 case Op_URShiftVB:
661 case Op_URShiftVS:
662 case Op_URShiftVI:
663 case Op_URShiftVL:
664 return true;
665 default:
666 return false;
667 }
668 }
669
670 bool VectorNode::is_vector_shift_count(int opc) {
671 assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode");
672 switch (opc) {
673 case Op_RShiftCntV:
674 case Op_LShiftCntV:
675 return true;
676 default:
677 return false;
678 }
679 }
680
681 static bool is_con_M1(Node* n) {
682 if (n->is_Con()) {
683 const Type* t = n->bottom_type();
684 if (t->isa_int() && t->is_int()->get_con() == -1) {
685 return true;
686 }
687 if (t->isa_long() && t->is_long()->get_con() == -1) {
688 return true;
689 }
690 }
691 return false;
692 }
693
694 bool VectorNode::is_all_ones_vector(Node* n) {
695 switch (n->Opcode()) {
696 case Op_ReplicateB:
697 case Op_ReplicateS:
698 case Op_ReplicateI:
699 case Op_ReplicateL:
700 return is_con_M1(n->in(1));
701 default:
702 return false;
703 }
704 }
705
706 bool VectorNode::is_vector_bitwise_not_pattern(Node* n) {
707 if (n->Opcode() == Op_XorV) {
708 return is_all_ones_vector(n->in(1)) ||
709 is_all_ones_vector(n->in(2));
710 }
711 return false;
712 }
713
714 // Return initial Pack node. Additional operands added with add_opd() calls.
715 PackNode* PackNode::make(Node* s, uint vlen, BasicType bt) {
716 const TypeVect* vt = TypeVect::make(bt, vlen);
717 switch (bt) {
718 case T_BOOLEAN:
719 case T_BYTE:
720 return new PackBNode(s, vt);
721 case T_CHAR:
722 case T_SHORT:
723 return new PackSNode(s, vt);
724 case T_INT:
725 return new PackINode(s, vt);
726 case T_LONG:
727 return new PackLNode(s, vt);
728 case T_FLOAT:
729 return new PackFNode(s, vt);
730 case T_DOUBLE:
731 return new PackDNode(s, vt);
732 default:
733 fatal("Type '%s' is not supported for vectors", type2name(bt));
734 return NULL;
735 }
736 }
737
738 // Create a binary tree form for Packs. [lo, hi) (half-open) range
739 PackNode* PackNode::binary_tree_pack(int lo, int hi) {
740 int ct = hi - lo;
741 assert(is_power_of_2(ct), "power of 2");
742 if (ct == 2) {
743 PackNode* pk = PackNode::make(in(lo), 2, vect_type()->element_basic_type());
744 pk->add_opd(in(lo+1));
745 return pk;
746 } else {
747 int mid = lo + ct/2;
748 PackNode* n1 = binary_tree_pack(lo, mid);
749 PackNode* n2 = binary_tree_pack(mid, hi );
750
751 BasicType bt = n1->vect_type()->element_basic_type();
752 assert(bt == n2->vect_type()->element_basic_type(), "should be the same");
753 switch (bt) {
754 case T_BOOLEAN:
755 case T_BYTE:
756 return new PackSNode(n1, n2, TypeVect::make(T_SHORT, 2));
757 case T_CHAR:
758 case T_SHORT:
759 return new PackINode(n1, n2, TypeVect::make(T_INT, 2));
760 case T_INT:
761 return new PackLNode(n1, n2, TypeVect::make(T_LONG, 2));
762 case T_LONG:
763 return new Pack2LNode(n1, n2, TypeVect::make(T_LONG, 2));
764 case T_FLOAT:
765 return new PackDNode(n1, n2, TypeVect::make(T_DOUBLE, 2));
766 case T_DOUBLE:
767 return new Pack2DNode(n1, n2, TypeVect::make(T_DOUBLE, 2));
768 default:
769 fatal("Type '%s' is not supported for vectors", type2name(bt));
770 return NULL;
771 }
772 }
773 }
774
775 // Return the vector version of a scalar load node.
776 LoadVectorNode* LoadVectorNode::make(int opc, Node* ctl, Node* mem,
777 Node* adr, const TypePtr* atyp,
778 uint vlen, BasicType bt,
779 ControlDependency control_dependency) {
780 const TypeVect* vt = TypeVect::make(bt, vlen);
781 return new LoadVectorNode(ctl, mem, adr, atyp, vt, control_dependency);
782 }
783
784 // Return the vector version of a scalar store node.
785 StoreVectorNode* StoreVectorNode::make(int opc, Node* ctl, Node* mem,
786 Node* adr, const TypePtr* atyp, Node* val,
787 uint vlen) {
788 return new StoreVectorNode(ctl, mem, adr, atyp, val);
789 }
790
791 Node* LoadVectorMaskedNode::Ideal(PhaseGVN* phase, bool can_reshape) {
792 if (!in(3)->is_top() && in(3)->Opcode() == Op_VectorMaskGen) {
793 Node* mask_len = in(3)->in(1);
794 const TypeLong* ty = phase->type(mask_len)->isa_long();
795 if (ty && ty->is_con()) {
796 BasicType mask_bt = ((VectorMaskGenNode*)in(3))->get_elem_type();
797 uint load_sz = type2aelembytes(mask_bt) * ty->get_con();
798 if ( load_sz == 32 || load_sz == 64) {
799 assert(load_sz == 32 || MaxVectorSize > 32, "Unexpected load size");
800 Node* ctr = in(MemNode::Control);
801 Node* mem = in(MemNode::Memory);
802 Node* adr = in(MemNode::Address);
803 return phase->transform(new LoadVectorNode(ctr, mem, adr, adr_type(), vect_type()));
804 }
805 }
806 }
807 return NULL;
808 }
809
810 Node* StoreVectorMaskedNode::Ideal(PhaseGVN* phase, bool can_reshape) {
811 if (!in(4)->is_top() && in(4)->Opcode() == Op_VectorMaskGen) {
812 Node* mask_len = in(4)->in(1);
813 const TypeLong* ty = phase->type(mask_len)->isa_long();
814 if (ty && ty->is_con()) {
815 BasicType mask_bt = ((VectorMaskGenNode*)in(4))->get_elem_type();
816 uint load_sz = type2aelembytes(mask_bt) * ty->get_con();
817 if ( load_sz == 32 || load_sz == 64) {
818 assert(load_sz == 32 || MaxVectorSize > 32, "Unexpected store size");
819 Node* ctr = in(MemNode::Control);
820 Node* mem = in(MemNode::Memory);
821 Node* adr = in(MemNode::Address);
822 Node* val = in(MemNode::ValueIn);
823 return phase->transform(new StoreVectorNode(ctr, mem, adr, adr_type(), val));
824 }
825 }
826 }
827 return NULL;
828 }
829
830 int ExtractNode::opcode(BasicType bt) {
831 switch (bt) {
832 case T_BOOLEAN: return Op_ExtractUB;
833 case T_BYTE: return Op_ExtractB;
834 case T_CHAR: return Op_ExtractC;
835 case T_SHORT: return Op_ExtractS;
836 case T_INT: return Op_ExtractI;
837 case T_LONG: return Op_ExtractL;
838 case T_FLOAT: return Op_ExtractF;
839 case T_DOUBLE: return Op_ExtractD;
840 default:
841 assert(false, "wrong type: %s", type2name(bt));
842 return 0;
843 }
844 }
845
846 // Extract a scalar element of vector.
847 Node* ExtractNode::make(Node* v, uint position, BasicType bt) {
848 assert((int)position < Matcher::max_vector_size(bt), "pos in range");
849 ConINode* pos = ConINode::make((int)position);
850 switch (bt) {
851 case T_BOOLEAN: return new ExtractUBNode(v, pos);
852 case T_BYTE: return new ExtractBNode(v, pos);
853 case T_CHAR: return new ExtractCNode(v, pos);
854 case T_SHORT: return new ExtractSNode(v, pos);
855 case T_INT: return new ExtractINode(v, pos);
856 case T_LONG: return new ExtractLNode(v, pos);
857 case T_FLOAT: return new ExtractFNode(v, pos);
858 case T_DOUBLE: return new ExtractDNode(v, pos);
859 default:
860 assert(false, "wrong type: %s", type2name(bt));
861 return NULL;
862 }
863 }
864
865 int ReductionNode::opcode(int opc, BasicType bt) {
866 int vopc = opc;
867 switch (opc) {
868 case Op_AddI:
869 switch (bt) {
870 case T_BOOLEAN:
871 case T_CHAR: return 0;
872 case T_BYTE:
873 case T_SHORT:
874 case T_INT:
875 vopc = Op_AddReductionVI;
876 break;
877 default: ShouldNotReachHere(); return 0;
878 }
879 break;
880 case Op_AddL:
881 assert(bt == T_LONG, "must be");
882 vopc = Op_AddReductionVL;
883 break;
884 case Op_AddF:
885 assert(bt == T_FLOAT, "must be");
886 vopc = Op_AddReductionVF;
887 break;
888 case Op_AddD:
889 assert(bt == T_DOUBLE, "must be");
890 vopc = Op_AddReductionVD;
891 break;
892 case Op_MulI:
893 switch (bt) {
894 case T_BOOLEAN:
895 case T_CHAR: return 0;
896 case T_BYTE:
897 case T_SHORT:
898 case T_INT:
899 vopc = Op_MulReductionVI;
900 break;
901 default: ShouldNotReachHere(); return 0;
902 }
903 break;
904 case Op_MulL:
905 assert(bt == T_LONG, "must be");
906 vopc = Op_MulReductionVL;
907 break;
908 case Op_MulF:
909 assert(bt == T_FLOAT, "must be");
910 vopc = Op_MulReductionVF;
911 break;
912 case Op_MulD:
913 assert(bt == T_DOUBLE, "must be");
914 vopc = Op_MulReductionVD;
915 break;
916 case Op_MinI:
917 switch (bt) {
918 case T_BOOLEAN:
919 case T_CHAR: return 0;
920 case T_BYTE:
921 case T_SHORT:
922 case T_INT:
923 vopc = Op_MinReductionV;
924 break;
925 default: ShouldNotReachHere(); return 0;
926 }
927 break;
928 case Op_MinL:
929 assert(bt == T_LONG, "must be");
930 vopc = Op_MinReductionV;
931 break;
932 case Op_MinF:
933 assert(bt == T_FLOAT, "must be");
934 vopc = Op_MinReductionV;
935 break;
936 case Op_MinD:
937 assert(bt == T_DOUBLE, "must be");
938 vopc = Op_MinReductionV;
939 break;
940 case Op_MaxI:
941 switch (bt) {
942 case T_BOOLEAN:
943 case T_CHAR: return 0;
944 case T_BYTE:
945 case T_SHORT:
946 case T_INT:
947 vopc = Op_MaxReductionV;
948 break;
949 default: ShouldNotReachHere(); return 0;
950 }
951 break;
952 case Op_MaxL:
953 assert(bt == T_LONG, "must be");
954 vopc = Op_MaxReductionV;
955 break;
956 case Op_MaxF:
957 assert(bt == T_FLOAT, "must be");
958 vopc = Op_MaxReductionV;
959 break;
960 case Op_MaxD:
961 assert(bt == T_DOUBLE, "must be");
962 vopc = Op_MaxReductionV;
963 break;
964 case Op_AndI:
965 switch (bt) {
966 case T_BOOLEAN:
967 case T_CHAR: return 0;
968 case T_BYTE:
969 case T_SHORT:
970 case T_INT:
971 vopc = Op_AndReductionV;
972 break;
973 default: ShouldNotReachHere(); return 0;
974 }
975 break;
976 case Op_AndL:
977 assert(bt == T_LONG, "must be");
978 vopc = Op_AndReductionV;
979 break;
980 case Op_OrI:
981 switch(bt) {
982 case T_BOOLEAN:
983 case T_CHAR: return 0;
984 case T_BYTE:
985 case T_SHORT:
986 case T_INT:
987 vopc = Op_OrReductionV;
988 break;
989 default: ShouldNotReachHere(); return 0;
990 }
991 break;
992 case Op_OrL:
993 assert(bt == T_LONG, "must be");
994 vopc = Op_OrReductionV;
995 break;
996 case Op_XorI:
997 switch(bt) {
998 case T_BOOLEAN:
999 case T_CHAR: return 0;
1000 case T_BYTE:
1001 case T_SHORT:
1002 case T_INT:
1003 vopc = Op_XorReductionV;
1004 break;
1005 default: ShouldNotReachHere(); return 0;
1006 }
1007 break;
1008 case Op_XorL:
1009 assert(bt == T_LONG, "must be");
1010 vopc = Op_XorReductionV;
1011 break;
1012 default:
1013 break;
1014 }
1015 return vopc;
1016 }
1017
1018 // Return the appropriate reduction node.
1019 ReductionNode* ReductionNode::make(int opc, Node *ctrl, Node* n1, Node* n2, BasicType bt) {
1020
1021 int vopc = opcode(opc, bt);
1022
1023 // This method should not be called for unimplemented vectors.
1024 guarantee(vopc != opc, "Vector for '%s' is not implemented", NodeClassNames[opc]);
1025
1026 switch (vopc) {
1027 case Op_AddReductionVI: return new AddReductionVINode(ctrl, n1, n2);
1028 case Op_AddReductionVL: return new AddReductionVLNode(ctrl, n1, n2);
1029 case Op_AddReductionVF: return new AddReductionVFNode(ctrl, n1, n2);
1030 case Op_AddReductionVD: return new AddReductionVDNode(ctrl, n1, n2);
1031 case Op_MulReductionVI: return new MulReductionVINode(ctrl, n1, n2);
1032 case Op_MulReductionVL: return new MulReductionVLNode(ctrl, n1, n2);
1033 case Op_MulReductionVF: return new MulReductionVFNode(ctrl, n1, n2);
1034 case Op_MulReductionVD: return new MulReductionVDNode(ctrl, n1, n2);
1035 case Op_MinReductionV: return new MinReductionVNode(ctrl, n1, n2);
1036 case Op_MaxReductionV: return new MaxReductionVNode(ctrl, n1, n2);
1037 case Op_AndReductionV: return new AndReductionVNode(ctrl, n1, n2);
1038 case Op_OrReductionV: return new OrReductionVNode(ctrl, n1, n2);
1039 case Op_XorReductionV: return new XorReductionVNode(ctrl, n1, n2);
1040 default:
1041 assert(false, "unknown node: %s", NodeClassNames[vopc]);
1042 return NULL;
1043 }
1044 }
1045
1046 Node* VectorLoadMaskNode::Identity(PhaseGVN* phase) {
1047 BasicType out_bt = type()->is_vect()->element_basic_type();
1048 if (!Matcher::has_predicated_vectors() && out_bt == T_BOOLEAN) {
1049 return in(1); // redundant conversion
1050 }
1051
1052 return this;
1053 }
1054
1055 Node* VectorStoreMaskNode::Identity(PhaseGVN* phase) {
1056 // Identity transformation on boolean vectors.
1057 // VectorStoreMask (VectorLoadMask bv) elem_size ==> bv
1058 // vector[n]{bool} => vector[n]{t} => vector[n]{bool}
1059 if (in(1)->Opcode() == Op_VectorLoadMask) {
1060 return in(1)->in(1);
1061 }
1062 return this;
1063 }
1064
1065 VectorStoreMaskNode* VectorStoreMaskNode::make(PhaseGVN& gvn, Node* in, BasicType in_type, uint num_elem) {
1066 assert(in->bottom_type()->isa_vect(), "sanity");
1067 const TypeVect* vt = TypeVect::make(T_BOOLEAN, num_elem);
1068 int elem_size = type2aelembytes(in_type);
1069 return new VectorStoreMaskNode(in, gvn.intcon(elem_size), vt);
1070 }
1071
1072 VectorCastNode* VectorCastNode::make(int vopc, Node* n1, BasicType bt, uint vlen) {
1073 const TypeVect* vt = TypeVect::make(bt, vlen);
1074 switch (vopc) {
1075 case Op_VectorCastB2X: return new VectorCastB2XNode(n1, vt);
1076 case Op_VectorCastS2X: return new VectorCastS2XNode(n1, vt);
1077 case Op_VectorCastI2X: return new VectorCastI2XNode(n1, vt);
1078 case Op_VectorCastL2X: return new VectorCastL2XNode(n1, vt);
1079 case Op_VectorCastF2X: return new VectorCastF2XNode(n1, vt);
1080 case Op_VectorCastD2X: return new VectorCastD2XNode(n1, vt);
1081 default:
1082 assert(false, "unknown node: %s", NodeClassNames[vopc]);
1083 return NULL;
1084 }
1085 }
1086
1087 int VectorCastNode::opcode(BasicType bt) {
1088 switch (bt) {
1089 case T_BYTE: return Op_VectorCastB2X;
1090 case T_SHORT: return Op_VectorCastS2X;
1091 case T_INT: return Op_VectorCastI2X;
1092 case T_LONG: return Op_VectorCastL2X;
1093 case T_FLOAT: return Op_VectorCastF2X;
1094 case T_DOUBLE: return Op_VectorCastD2X;
1095 default:
1096 assert(false, "unknown type: %s", type2name(bt));
1097 return 0;
1098 }
1099 }
1100
1101 Node* VectorCastNode::Identity(PhaseGVN* phase) {
1102 if (!in(1)->is_top()) {
1103 BasicType in_bt = in(1)->bottom_type()->is_vect()->element_basic_type();
1104 BasicType out_bt = vect_type()->element_basic_type();
1105 if (in_bt == out_bt) {
1106 return in(1); // redundant cast
1107 }
1108 }
1109 return this;
1110 }
1111
1112 Node* ReductionNode::make_reduction_input(PhaseGVN& gvn, int opc, BasicType bt) {
1113 int vopc = opcode(opc, bt);
1114 guarantee(vopc != opc, "Vector reduction for '%s' is not implemented", NodeClassNames[opc]);
1115
1116 switch (vopc) {
1117 case Op_AndReductionV:
1118 switch (bt) {
1119 case T_BYTE:
1120 case T_SHORT:
1121 case T_INT:
1122 return gvn.makecon(TypeInt::MINUS_1);
1123 case T_LONG:
1124 return gvn.makecon(TypeLong::MINUS_1);
1125 default:
1126 fatal("Missed vector creation for '%s' as the basic type is not correct.", NodeClassNames[vopc]);
1127 return NULL;
1128 }
1129 break;
1130 case Op_AddReductionVI: // fallthrough
1131 case Op_AddReductionVL: // fallthrough
1132 case Op_AddReductionVF: // fallthrough
1133 case Op_AddReductionVD:
1134 case Op_OrReductionV:
1135 case Op_XorReductionV:
1136 return gvn.zerocon(bt);
1137 case Op_MulReductionVI:
1138 return gvn.makecon(TypeInt::ONE);
1139 case Op_MulReductionVL:
1140 return gvn.makecon(TypeLong::ONE);
1141 case Op_MulReductionVF:
1142 return gvn.makecon(TypeF::ONE);
1143 case Op_MulReductionVD:
1144 return gvn.makecon(TypeD::ONE);
1145 case Op_MinReductionV:
1146 switch (bt) {
1147 case T_BYTE:
1148 return gvn.makecon(TypeInt::make(max_jbyte));
1149 case T_SHORT:
1150 return gvn.makecon(TypeInt::make(max_jshort));
1151 case T_INT:
1152 return gvn.makecon(TypeInt::MAX);
1153 case T_LONG:
1154 return gvn.makecon(TypeLong::MAX);
1155 case T_FLOAT:
1156 return gvn.makecon(TypeF::POS_INF);
1157 case T_DOUBLE:
1158 return gvn.makecon(TypeD::POS_INF);
1159 default: Unimplemented(); return NULL;
1160 }
1161 break;
1162 case Op_MaxReductionV:
1163 switch (bt) {
1164 case T_BYTE:
1165 return gvn.makecon(TypeInt::make(min_jbyte));
1166 case T_SHORT:
1167 return gvn.makecon(TypeInt::make(min_jshort));
1168 case T_INT:
1169 return gvn.makecon(TypeInt::MIN);
1170 case T_LONG:
1171 return gvn.makecon(TypeLong::MIN);
1172 case T_FLOAT:
1173 return gvn.makecon(TypeF::NEG_INF);
1174 case T_DOUBLE:
1175 return gvn.makecon(TypeD::NEG_INF);
1176 default: Unimplemented(); return NULL;
1177 }
1178 break;
1179 default:
1180 fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
1181 return NULL;
1182 }
1183 }
1184
1185 bool ReductionNode::implemented(int opc, uint vlen, BasicType bt) {
1186 if (is_java_primitive(bt) &&
1187 (vlen > 1) && is_power_of_2(vlen) &&
1188 Matcher::vector_size_supported(bt, vlen)) {
1189 int vopc = ReductionNode::opcode(opc, bt);
1190 return vopc != opc && Matcher::match_rule_supported_vector(vopc, vlen, bt);
1191 }
1192 return false;
1193 }
1194
1195 MacroLogicVNode* MacroLogicVNode::make(PhaseGVN& gvn, Node* in1, Node* in2, Node* in3,
1196 uint truth_table, const TypeVect* vt) {
1197 assert(truth_table <= 0xFF, "invalid");
1198 assert(in1->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
1199 assert(in2->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
1200 assert(in3->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
1201 Node* fn = gvn.intcon(truth_table);
1202 return new MacroLogicVNode(in1, in2, in3, fn, vt);
1203 }
1204
1205 Node* VectorNode::degenerate_vector_rotate(Node* src, Node* cnt, bool is_rotate_left,
1206 int vlen, BasicType bt, PhaseGVN* phase) {
1207 assert(is_integral_type(bt), "sanity");
1208 const TypeVect* vt = TypeVect::make(bt, vlen);
1209
1210 int shift_mask = (type2aelembytes(bt) * 8) - 1;
1211 int shiftLOpc = (bt == T_LONG) ? Op_LShiftL : Op_LShiftI;
1212 auto urshiftopc = [=]() {
1213 switch(bt) {
1214 case T_INT: return Op_URShiftI;
1215 case T_LONG: return Op_URShiftL;
1216 case T_BYTE: return Op_URShiftB;
1217 case T_SHORT: return Op_URShiftS;
1218 default: return (Opcodes)0;
1219 }
1220 };
1221 int shiftROpc = urshiftopc();
1222
1223 // Compute shift values for right rotation and
1224 // later swap them in case of left rotation.
1225 Node* shiftRCnt = NULL;
1226 Node* shiftLCnt = NULL;
1227 const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
1228 bool is_binary_vector_op = false;
1229 if (cnt_type && cnt_type->is_con()) {
1230 // Constant shift.
1231 int shift = cnt_type->get_con() & shift_mask;
1232 shiftRCnt = phase->intcon(shift);
1233 shiftLCnt = phase->intcon(shift_mask + 1 - shift);
1234 } else if (VectorNode::is_invariant_vector(cnt)) {
1235 // Scalar variable shift, handle replicates generated by auto vectorizer.
1236 cnt = cnt->in(1);
1237 if (bt == T_LONG) {
1238 // Shift count vector for Rotate vector has long elements too.
1239 if (cnt->Opcode() == Op_ConvI2L) {
1240 cnt = cnt->in(1);
1241 } else {
1242 assert(cnt->bottom_type()->isa_long() &&
1243 cnt->bottom_type()->is_long()->is_con(), "Long constant expected");
1244 cnt = phase->transform(new ConvL2INode(cnt));
1245 }
1246 }
1247 shiftRCnt = phase->transform(new AndINode(cnt, phase->intcon(shift_mask)));
1248 shiftLCnt = phase->transform(new SubINode(phase->intcon(shift_mask + 1), shiftRCnt));
1249 } else {
1250 // Variable vector rotate count.
1251 assert(Matcher::supports_vector_variable_shifts(), "");
1252
1253 int subVopc = 0;
1254 int addVopc = 0;
1255 Node* shift_mask_node = NULL;
1256 Node* const_one_node = NULL;
1257
1258 assert(cnt->bottom_type()->isa_vect(), "Unexpected shift");
1259 const Type* elem_ty = Type::get_const_basic_type(bt);
1260
1261 if (bt == T_LONG) {
1262 shift_mask_node = phase->longcon(shift_mask);
1263 const_one_node = phase->longcon(1L);
1264 subVopc = VectorNode::opcode(Op_SubL, bt);
1265 addVopc = VectorNode::opcode(Op_AddL, bt);
1266 } else {
1267 shift_mask_node = phase->intcon(shift_mask);
1268 const_one_node = phase->intcon(1);
1269 subVopc = VectorNode::opcode(Op_SubI, bt);
1270 addVopc = VectorNode::opcode(Op_AddI, bt);
1271 }
1272 Node* vector_mask = phase->transform(VectorNode::scalar2vector(shift_mask_node, vlen, elem_ty));
1273 Node* vector_one = phase->transform(VectorNode::scalar2vector(const_one_node, vlen, elem_ty));
1274
1275 shiftRCnt = cnt;
1276 shiftRCnt = phase->transform(VectorNode::make(Op_AndV, shiftRCnt, vector_mask, vt));
1277 vector_mask = phase->transform(VectorNode::make(addVopc, vector_one, vector_mask, vt));
1278 shiftLCnt = phase->transform(VectorNode::make(subVopc, vector_mask, shiftRCnt, vt));
1279 is_binary_vector_op = true;
1280 }
1281
1282 // Swap the computed left and right shift counts.
1283 if (is_rotate_left) {
1284 swap(shiftRCnt,shiftLCnt);
1285 }
1286
1287 if (!is_binary_vector_op) {
1288 shiftLCnt = phase->transform(new LShiftCntVNode(shiftLCnt, vt));
1289 shiftRCnt = phase->transform(new RShiftCntVNode(shiftRCnt, vt));
1290 }
1291
1292 return new OrVNode(phase->transform(VectorNode::make(shiftLOpc, src, shiftLCnt, vlen, bt)),
1293 phase->transform(VectorNode::make(shiftROpc, src, shiftRCnt, vlen, bt)),
1294 vt);
1295 }
1296
1297 Node* RotateLeftVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1298 int vlen = length();
1299 BasicType bt = vect_type()->element_basic_type();
1300 if ((!in(2)->is_Con() && !Matcher::supports_vector_variable_rotates()) ||
1301 !Matcher::match_rule_supported_vector(Op_RotateLeftV, vlen, bt)) {
1302 return VectorNode::degenerate_vector_rotate(in(1), in(2), true, vlen, bt, phase);
1303 }
1304 return NULL;
1305 }
1306
1307 Node* RotateRightVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1308 int vlen = length();
1309 BasicType bt = vect_type()->element_basic_type();
1310 if ((!in(2)->is_Con() && !Matcher::supports_vector_variable_rotates()) ||
1311 !Matcher::match_rule_supported_vector(Op_RotateRightV, vlen, bt)) {
1312 return VectorNode::degenerate_vector_rotate(in(1), in(2), false, vlen, bt, phase);
1313 }
1314 return NULL;
1315 }
1316
1317 #ifndef PRODUCT
1318 void VectorMaskCmpNode::dump_spec(outputStream *st) const {
1319 st->print(" %d #", _predicate); _type->dump_on(st);
1320 }
1321 #endif // PRODUCT
1322
1323 Node* VectorReinterpretNode::Identity(PhaseGVN *phase) {
1324 Node* n = in(1);
1325 if (n->Opcode() == Op_VectorReinterpret) {
1326 // "VectorReinterpret (VectorReinterpret node) ==> node" if:
1327 // 1) Types of 'node' and 'this' are identical
1328 // 2) Truncations are not introduced by the first VectorReinterpret
1329 if (Type::cmp(bottom_type(), n->in(1)->bottom_type()) == 0 &&
1330 length_in_bytes() <= n->bottom_type()->is_vect()->length_in_bytes()) {
1331 return n->in(1);
1332 }
1333 }
1334 return this;
1335 }
1336
1337 Node* VectorInsertNode::make(Node* vec, Node* new_val, int position) {
1338 assert(position < (int)vec->bottom_type()->is_vect()->length(), "pos in range");
1339 ConINode* pos = ConINode::make(position);
1340 return new VectorInsertNode(vec, new_val, pos, vec->bottom_type()->is_vect());
1341 }
1342
1343 Node* VectorUnboxNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1344 Node* n = obj()->uncast();
1345 if (EnableVectorReboxing && n->Opcode() == Op_VectorBox) {
1346 if (Type::cmp(bottom_type(), n->in(VectorBoxNode::Value)->bottom_type()) == 0) {
1347 // Handled by VectorUnboxNode::Identity()
1348 } else {
1349 VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
1350 ciKlass* vbox_klass = vbox->box_type()->klass();
1351 const TypeVect* in_vt = vbox->vec_type();
1352 const TypeVect* out_vt = type()->is_vect();
1353
1354 if (in_vt->length() == out_vt->length()) {
1355 Node* value = vbox->in(VectorBoxNode::Value);
1356
1357 bool is_vector_mask = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
1358 bool is_vector_shuffle = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
1359 if (is_vector_mask) {
1360 const TypeVect* vmask_type = TypeVect::makemask(out_vt->element_basic_type(), out_vt->length());
1361 if (in_vt->length_in_bytes() == out_vt->length_in_bytes() &&
1362 Matcher::match_rule_supported_vector(Op_VectorMaskCast, out_vt->length(), out_vt->element_basic_type())) {
1363 // Apply "VectorUnbox (VectorBox vmask) ==> VectorMaskCast (vmask)"
1364 // directly. This could avoid the transformation ordering issue from
1365 // "VectorStoreMask (VectorLoadMask vmask) => vmask".
1366 return new VectorMaskCastNode(value, vmask_type);
1367 }
1368 // VectorUnbox (VectorBox vmask) ==> VectorLoadMask (VectorStoreMask vmask)
1369 value = phase->transform(VectorStoreMaskNode::make(*phase, value, in_vt->element_basic_type(), in_vt->length()));
1370 return new VectorLoadMaskNode(value, vmask_type);
1371 } else if (is_vector_shuffle) {
1372 if (!is_shuffle_to_vector()) {
1373 // VectorUnbox (VectorBox vshuffle) ==> VectorLoadShuffle vshuffle
1374 return new VectorLoadShuffleNode(value, out_vt);
1375 }
1376 } else {
1377 // Vector type mismatch is only supported for masks and shuffles, but sometimes it happens in pathological cases.
1378 }
1379 } else {
1380 // Vector length mismatch.
1381 // Sometimes happen in pathological cases (e.g., when unboxing happens in effectively dead code).
1382 }
1383 }
1384 }
1385 return NULL;
1386 }
1387
1388 Node* VectorUnboxNode::Identity(PhaseGVN* phase) {
1389 Node* n = obj()->uncast();
1390 if (EnableVectorReboxing && n->Opcode() == Op_VectorBox) {
1391 if (Type::cmp(bottom_type(), n->in(VectorBoxNode::Value)->bottom_type()) == 0) {
1392 return n->in(VectorBoxNode::Value); // VectorUnbox (VectorBox v) ==> v
1393 } else {
1394 // Handled by VectorUnboxNode::Ideal().
1395 }
1396 }
1397 return this;
1398 }
1399
1400 const TypeFunc* VectorBoxNode::vec_box_type(const TypeInstPtr* box_type) {
1401 const Type** fields = TypeTuple::fields(0);
1402 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms, fields);
1403
1404 fields = TypeTuple::fields(1);
1405 fields[TypeFunc::Parms+0] = box_type;
1406 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
1407
1408 return TypeFunc::make(domain, range);
1409 }
1410
1411 Node* ShiftVNode::Identity(PhaseGVN* phase) {
1412 Node* in2 = in(2);
1413 // Shift by ZERO does nothing
1414 if (is_vshift_cnt(in2) && phase->find_int_type(in2->in(1)) == TypeInt::ZERO) {
1415 return in(1);
1416 }
1417 return this;
1418 }
1419
1420 Node* VectorMaskOpNode::make(Node* mask, const Type* ty, int mopc) {
1421 switch(mopc) {
1422 case Op_VectorMaskTrueCount:
1423 return new VectorMaskTrueCountNode(mask, ty);
1424 case Op_VectorMaskLastTrue:
1425 return new VectorMaskLastTrueNode(mask, ty);
1426 case Op_VectorMaskFirstTrue:
1427 return new VectorMaskFirstTrueNode(mask, ty);
1428 case Op_VectorMaskToLong:
1429 return new VectorMaskToLongNode(mask, ty);
1430 default:
1431 assert(false, "Unhandled operation");
1432 }
1433 return NULL;
1434 }
1435
1436 #ifndef PRODUCT
1437 void VectorBoxAllocateNode::dump_spec(outputStream *st) const {
1438 CallStaticJavaNode::dump_spec(st);
1439 }
1440
1441 #endif // !PRODUCT