1 /*
2 * Copyright (c) 2007, 2022, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #ifndef SHARE_OPTO_VECTORNODE_HPP
25 #define SHARE_OPTO_VECTORNODE_HPP
26
27 #include "opto/callnode.hpp"
28 #include "opto/matcher.hpp"
29 #include "opto/memnode.hpp"
30 #include "opto/node.hpp"
31 #include "opto/opcodes.hpp"
32 #include "prims/vectorSupport.hpp"
33
34 //------------------------------VectorNode-------------------------------------
35 // Vector Operation
36 class VectorNode : public TypeNode {
37 public:
38
39 VectorNode(Node* n1, const TypeVect* vt) : TypeNode(vt, 2) {
40 init_class_id(Class_Vector);
41 init_req(1, n1);
42 }
43 VectorNode(Node* n1, Node* n2, const TypeVect* vt) : TypeNode(vt, 3) {
44 init_class_id(Class_Vector);
45 init_req(1, n1);
46 init_req(2, n2);
47 }
48
49 VectorNode(Node* n1, Node* n2, Node* n3, const TypeVect* vt) : TypeNode(vt, 4) {
50 init_class_id(Class_Vector);
51 init_req(1, n1);
52 init_req(2, n2);
53 init_req(3, n3);
54 }
55
56 VectorNode(Node *n0, Node* n1, Node* n2, Node* n3, const TypeVect* vt) : TypeNode(vt, 5) {
57 init_class_id(Class_Vector);
58 init_req(1, n0);
59 init_req(2, n1);
60 init_req(3, n2);
61 init_req(4, n3);
62 }
63
64 const TypeVect* vect_type() const { return type()->is_vect(); }
65 uint length() const { return vect_type()->length(); } // Vector length
66 uint length_in_bytes() const { return vect_type()->length_in_bytes(); }
67
68 virtual int Opcode() const;
69
70 virtual uint ideal_reg() const {
71 return type()->ideal_reg();
72 }
73
74 static VectorNode* scalar2vector(Node* s, uint vlen, const Type* opd_t, bool is_mask = false);
75 static VectorNode* shift_count(int opc, Node* cnt, uint vlen, BasicType bt);
76 static VectorNode* make(int opc, Node* n1, Node* n2, uint vlen, BasicType bt, bool is_var_shift = false);
77 static VectorNode* make(int vopc, Node* n1, Node* n2, const TypeVect* vt, bool is_mask = false, bool is_var_shift = false);
78 static VectorNode* make(int opc, Node* n1, Node* n2, Node* n3, uint vlen, BasicType bt);
79 static VectorNode* make(int vopc, Node* n1, Node* n2, Node* n3, const TypeVect* vt);
80 static VectorNode* make_mask_node(int vopc, Node* n1, Node* n2, uint vlen, BasicType bt);
81
82 static bool is_shift_opcode(int opc);
83
84 static bool is_vshift_cnt_opcode(int opc);
85
86 static bool is_rotate_opcode(int opc);
87
88 static int opcode(int opc, BasicType bt);
89 static int replicate_opcode(BasicType bt);
90 static bool implemented(int opc, uint vlen, BasicType bt);
91 static bool is_shift(Node* n);
92 static bool is_vshift_cnt(Node* n);
93 static bool is_type_transition_short_to_int(Node* n);
94 static bool is_type_transition_to_int(Node* n);
95 static bool is_muladds2i(Node* n);
96 static bool is_type_transition_long_to_int(Node* n);
97 static bool is_roundopD(Node* n);
98 static bool is_scalar_rotate(Node* n);
99 static bool is_vector_rotate_supported(int opc, uint vlen, BasicType bt);
100 static bool is_vector_integral_negate_supported(int opc, uint vlen, BasicType bt, bool use_predicate);
101 static bool is_invariant_vector(Node* n);
102 static bool is_all_ones_vector(Node* n);
103 static bool is_vector_bitwise_not_pattern(Node* n);
104 static Node* degenerate_vector_rotate(Node* n1, Node* n2, bool is_rotate_left, int vlen,
105 BasicType bt, PhaseGVN* phase);
106
107 // [Start, end) half-open range defining which operands are vectors
108 static void vector_operands(Node* n, uint* start, uint* end);
109
110 static bool is_vector_shift(int opc);
111 static bool is_vector_shift_count(int opc);
112 static bool is_vector_rotate(int opc);
113 static bool is_vector_integral_negate(int opc);
114
115 static bool is_vector_shift(Node* n) {
116 return is_vector_shift(n->Opcode());
117 }
118 static bool is_vector_shift_count(Node* n) {
119 return is_vector_shift_count(n->Opcode());
120 }
121 };
122
123 //===========================Vector=ALU=Operations=============================
124
125 //------------------------------AddVBNode--------------------------------------
126 // Vector add byte
127 class AddVBNode : public VectorNode {
128 public:
129 AddVBNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
130 virtual int Opcode() const;
131 };
132
133 //------------------------------AddVSNode--------------------------------------
134 // Vector add char/short
135 class AddVSNode : public VectorNode {
136 public:
137 AddVSNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
138 virtual int Opcode() const;
139 };
140
141 //------------------------------AddVINode--------------------------------------
142 // Vector add int
143 class AddVINode : public VectorNode {
144 public:
145 AddVINode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
146 virtual int Opcode() const;
147 };
148
149 //------------------------------AddVLNode--------------------------------------
150 // Vector add long
151 class AddVLNode : public VectorNode {
152 public:
153 AddVLNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
154 virtual int Opcode() const;
155 };
156
157 //------------------------------AddVFNode--------------------------------------
158 // Vector add float
159 class AddVFNode : public VectorNode {
160 public:
161 AddVFNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
162 virtual int Opcode() const;
163 };
164
165 //------------------------------AddVDNode--------------------------------------
166 // Vector add double
167 class AddVDNode : public VectorNode {
168 public:
169 AddVDNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
170 virtual int Opcode() const;
171 };
172
173 //------------------------------ReductionNode------------------------------------
174 // Perform reduction of a vector
175 class ReductionNode : public Node {
176 private:
177 const Type* _bottom_type;
178 public:
179 ReductionNode(Node *ctrl, Node* in1, Node* in2) : Node(ctrl, in1, in2),
180 _bottom_type(Type::get_const_basic_type(in1->bottom_type()->basic_type())) {}
181
182 static ReductionNode* make(int opc, Node *ctrl, Node* in1, Node* in2, BasicType bt);
183 static int opcode(int opc, BasicType bt);
184 static bool implemented(int opc, uint vlen, BasicType bt);
185 static Node* make_reduction_input(PhaseGVN& gvn, int opc, BasicType bt);
186
187 virtual const Type* bottom_type() const {
188 return _bottom_type;
189 }
190
191 virtual uint ideal_reg() const {
192 return bottom_type()->ideal_reg();
193 }
194
195 // Needed for proper cloning.
196 virtual uint size_of() const { return sizeof(*this); }
197 };
198
199 //------------------------------AddReductionVINode--------------------------------------
200 // Vector add byte, short and int as a reduction
201 class AddReductionVINode : public ReductionNode {
202 public:
203 AddReductionVINode(Node * ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
204 virtual int Opcode() const;
205 };
206
207 //------------------------------AddReductionVLNode--------------------------------------
208 // Vector add long as a reduction
209 class AddReductionVLNode : public ReductionNode {
210 public:
211 AddReductionVLNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
212 virtual int Opcode() const;
213 };
214
215 //------------------------------AddReductionVFNode--------------------------------------
216 // Vector add float as a reduction
217 class AddReductionVFNode : public ReductionNode {
218 public:
219 AddReductionVFNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
220 virtual int Opcode() const;
221 };
222
223 //------------------------------AddReductionVDNode--------------------------------------
224 // Vector add double as a reduction
225 class AddReductionVDNode : public ReductionNode {
226 public:
227 AddReductionVDNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
228 virtual int Opcode() const;
229 };
230
231 //------------------------------SubVBNode--------------------------------------
232 // Vector subtract byte
233 class SubVBNode : public VectorNode {
234 public:
235 SubVBNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
236 virtual int Opcode() const;
237 };
238
239 //------------------------------SubVSNode--------------------------------------
240 // Vector subtract short
241 class SubVSNode : public VectorNode {
242 public:
243 SubVSNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
244 virtual int Opcode() const;
245 };
246
247 //------------------------------SubVINode--------------------------------------
248 // Vector subtract int
249 class SubVINode : public VectorNode {
250 public:
251 SubVINode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
252 virtual int Opcode() const;
253 };
254
255 //------------------------------SubVLNode--------------------------------------
256 // Vector subtract long
257 class SubVLNode : public VectorNode {
258 public:
259 SubVLNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
260 virtual int Opcode() const;
261 };
262
263 //------------------------------SubVFNode--------------------------------------
264 // Vector subtract float
265 class SubVFNode : public VectorNode {
266 public:
267 SubVFNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
268 virtual int Opcode() const;
269 };
270
271 //------------------------------SubVDNode--------------------------------------
272 // Vector subtract double
273 class SubVDNode : public VectorNode {
274 public:
275 SubVDNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
276 virtual int Opcode() const;
277 };
278
279 //------------------------------MulVBNode--------------------------------------
280 // Vector multiply byte
281 class MulVBNode : public VectorNode {
282 public:
283 MulVBNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
284 virtual int Opcode() const;
285 };
286
287 //------------------------------MulVSNode--------------------------------------
288 // Vector multiply short
289 class MulVSNode : public VectorNode {
290 public:
291 MulVSNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
292 virtual int Opcode() const;
293 };
294
295 //------------------------------MulVINode--------------------------------------
296 // Vector multiply int
297 class MulVINode : public VectorNode {
298 public:
299 MulVINode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
300 virtual int Opcode() const;
301 };
302
303 //------------------------------MulVLNode--------------------------------------
304 // Vector multiply long
305 class MulVLNode : public VectorNode {
306 public:
307 MulVLNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
308 virtual int Opcode() const;
309 };
310
311 //------------------------------MulVFNode--------------------------------------
312 // Vector multiply float
313 class MulVFNode : public VectorNode {
314 public:
315 MulVFNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
316 virtual int Opcode() const;
317 };
318
319 //------------------------------MulVDNode--------------------------------------
320 // Vector multiply double
321 class MulVDNode : public VectorNode {
322 public:
323 MulVDNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
324 virtual int Opcode() const;
325 };
326
327 //------------------------------MulAddVS2VINode--------------------------------
328 // Vector multiply shorts to int and add adjacent ints.
329 class MulAddVS2VINode : public VectorNode {
330 public:
331 MulAddVS2VINode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
332 virtual int Opcode() const;
333 };
334
335 //------------------------------FmaVDNode--------------------------------------
336 // Vector multiply double
337 class FmaVDNode : public VectorNode {
338 public:
339 FmaVDNode(Node* in1, Node* in2, Node* in3, const TypeVect* vt) : VectorNode(in1, in2, in3, vt) {}
340 virtual int Opcode() const;
341 };
342
343 //------------------------------FmaVFNode--------------------------------------
344 // Vector multiply float
345 class FmaVFNode : public VectorNode {
346 public:
347 FmaVFNode(Node* in1, Node* in2, Node* in3, const TypeVect* vt) : VectorNode(in1, in2, in3, vt) {}
348 virtual int Opcode() const;
349 };
350
351 //------------------------------CMoveVFNode--------------------------------------
352 // Vector float conditional move
353 class CMoveVFNode : public VectorNode {
354 public:
355 CMoveVFNode(Node* in1, Node* in2, Node* in3, const TypeVect* vt) : VectorNode(in1, in2, in3, vt) {}
356 virtual int Opcode() const;
357 };
358
359 //------------------------------CMoveVDNode--------------------------------------
360 // Vector double conditional move
361 class CMoveVDNode : public VectorNode {
362 public:
363 CMoveVDNode(Node* in1, Node* in2, Node* in3, const TypeVect* vt) : VectorNode(in1, in2, in3, vt) {}
364 virtual int Opcode() const;
365 };
366
367 //------------------------------MulReductionVINode--------------------------------------
368 // Vector multiply byte, short and int as a reduction
369 class MulReductionVINode : public ReductionNode {
370 public:
371 MulReductionVINode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
372 virtual int Opcode() const;
373 };
374
375 //------------------------------MulReductionVLNode--------------------------------------
376 // Vector multiply int as a reduction
377 class MulReductionVLNode : public ReductionNode {
378 public:
379 MulReductionVLNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
380 virtual int Opcode() const;
381 };
382
383 //------------------------------MulReductionVFNode--------------------------------------
384 // Vector multiply float as a reduction
385 class MulReductionVFNode : public ReductionNode {
386 public:
387 MulReductionVFNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
388 virtual int Opcode() const;
389 };
390
391 //------------------------------MulReductionVDNode--------------------------------------
392 // Vector multiply double as a reduction
393 class MulReductionVDNode : public ReductionNode {
394 public:
395 MulReductionVDNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
396 virtual int Opcode() const;
397 };
398
399 //------------------------------DivVFNode--------------------------------------
400 // Vector divide float
401 class DivVFNode : public VectorNode {
402 public:
403 DivVFNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
404 virtual int Opcode() const;
405 };
406
407 //------------------------------DivVDNode--------------------------------------
408 // Vector Divide double
409 class DivVDNode : public VectorNode {
410 public:
411 DivVDNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
412 virtual int Opcode() const;
413 };
414
415 //------------------------------AbsVBNode--------------------------------------
416 // Vector Abs byte
417 class AbsVBNode : public VectorNode {
418 public:
419 AbsVBNode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {}
420 virtual int Opcode() const;
421 };
422
423 //------------------------------AbsVSNode--------------------------------------
424 // Vector Abs short
425 class AbsVSNode : public VectorNode {
426 public:
427 AbsVSNode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {}
428 virtual int Opcode() const;
429 };
430
431 //------------------------------MinVNode--------------------------------------
432 // Vector Min
433 class MinVNode : public VectorNode {
434 public:
435 MinVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
436 virtual int Opcode() const;
437 };
438
439 //------------------------------MaxVNode--------------------------------------
440 // Vector Max
441 class MaxVNode : public VectorNode {
442 public:
443 MaxVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
444 virtual int Opcode() const;
445 };
446
447 //------------------------------AbsVINode--------------------------------------
448 // Vector Abs int
449 class AbsVINode : public VectorNode {
450 public:
451 AbsVINode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {}
452 virtual int Opcode() const;
453 };
454
455 //------------------------------AbsVLNode--------------------------------------
456 // Vector Abs long
457 class AbsVLNode : public VectorNode {
458 public:
459 AbsVLNode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {}
460 virtual int Opcode() const;
461 };
462
463 //------------------------------AbsVFNode--------------------------------------
464 // Vector Abs float
465 class AbsVFNode : public VectorNode {
466 public:
467 AbsVFNode(Node* in, const TypeVect* vt) : VectorNode(in,vt) {}
468 virtual int Opcode() const;
469 };
470
471 //------------------------------AbsVDNode--------------------------------------
472 // Vector Abs double
473 class AbsVDNode : public VectorNode {
474 public:
475 AbsVDNode(Node* in, const TypeVect* vt) : VectorNode(in,vt) {}
476 virtual int Opcode() const;
477 };
478
479 //------------------------------NegVNode---------------------------------------
480 // Vector Neg parent class (not for code generation).
481 class NegVNode : public VectorNode {
482 public:
483 NegVNode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {}
484 virtual int Opcode() const = 0;
485 virtual Node* Ideal(PhaseGVN* phase, bool can_reshape);
486
487 private:
488 Node* degenerate_integral_negate(PhaseGVN* phase, bool is_predicated);
489 };
490
491 //------------------------------NegVINode--------------------------------------
492 // Vector Neg byte/short/int
493 class NegVINode : public NegVNode {
494 public:
495 NegVINode(Node* in, const TypeVect* vt) : NegVNode(in, vt) {}
496 virtual int Opcode() const;
497 };
498
499 //------------------------------NegVLNode--------------------------------------
500 // Vector Neg long
501 class NegVLNode : public NegVNode {
502 public:
503 NegVLNode(Node* in, const TypeVect* vt) : NegVNode(in, vt) {}
504 virtual int Opcode() const;
505 };
506
507 //------------------------------NegVFNode--------------------------------------
508 // Vector Neg float
509 class NegVFNode : public NegVNode {
510 public:
511 NegVFNode(Node* in, const TypeVect* vt) : NegVNode(in, vt) {}
512 virtual int Opcode() const;
513 };
514
515 //------------------------------NegVDNode--------------------------------------
516 // Vector Neg double
517 class NegVDNode : public NegVNode {
518 public:
519 NegVDNode(Node* in, const TypeVect* vt) : NegVNode(in, vt) {}
520 virtual int Opcode() const;
521 };
522
523 //------------------------------PopCountVINode---------------------------------
524 // Vector popcount integer bits
525 class PopCountVINode : public VectorNode {
526 public:
527 PopCountVINode(Node* in, const TypeVect* vt) : VectorNode(in,vt) {}
528 virtual int Opcode() const;
529 };
530
531 //------------------------------PopCountVLNode---------------------------------
532 // Vector popcount long bits
533 class PopCountVLNode : public VectorNode {
534 public:
535 PopCountVLNode(Node* in, const TypeVect* vt) : VectorNode(in,vt) {}
536 virtual int Opcode() const;
537 };
538
539 //------------------------------SqrtVFNode--------------------------------------
540 // Vector Sqrt float
541 class SqrtVFNode : public VectorNode {
542 public:
543 SqrtVFNode(Node* in, const TypeVect* vt) : VectorNode(in,vt) {}
544 virtual int Opcode() const;
545 };
546 //------------------------------RoundDoubleVNode--------------------------------
547 // Vector round double
548 class RoundDoubleModeVNode : public VectorNode {
549 public:
550 RoundDoubleModeVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
551 virtual int Opcode() const;
552 };
553
554 //------------------------------SqrtVDNode--------------------------------------
555 // Vector Sqrt double
556 class SqrtVDNode : public VectorNode {
557 public:
558 SqrtVDNode(Node* in, const TypeVect* vt) : VectorNode(in,vt) {}
559 virtual int Opcode() const;
560 };
561
562 //------------------------------ShiftVNode-----------------------------------
563 // Class ShiftV functionality. This covers the common behaviors for all kinds
564 // of vector shifts.
565 class ShiftVNode : public VectorNode {
566 private:
567 bool _is_var_shift;
568 public:
569 ShiftVNode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift) :
570 VectorNode(in1,in2,vt), _is_var_shift(is_var_shift) {
571 init_class_id(Class_ShiftV);
572 }
573 virtual Node* Identity(PhaseGVN* phase);
574 virtual int Opcode() const = 0;
575 virtual uint hash() const { return VectorNode::hash() + _is_var_shift; }
576 virtual bool cmp(const Node& n) const {
577 return VectorNode::cmp(n) && _is_var_shift == ((ShiftVNode&)n)._is_var_shift;
578 }
579 bool is_var_shift() { return _is_var_shift;}
580 virtual uint size_of() const { return sizeof(ShiftVNode); }
581 };
582
583 //------------------------------LShiftVBNode-----------------------------------
584 // Vector left shift bytes
585 class LShiftVBNode : public ShiftVNode {
586 public:
587 LShiftVBNode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
588 ShiftVNode(in1,in2,vt,is_var_shift) {}
589 virtual int Opcode() const;
590 };
591
592 //------------------------------LShiftVSNode-----------------------------------
593 // Vector left shift shorts
594 class LShiftVSNode : public ShiftVNode {
595 public:
596 LShiftVSNode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
597 ShiftVNode(in1,in2,vt,is_var_shift) {}
598 virtual int Opcode() const;
599 };
600
601 //------------------------------LShiftVINode-----------------------------------
602 // Vector left shift ints
603 class LShiftVINode : public ShiftVNode {
604 public:
605 LShiftVINode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
606 ShiftVNode(in1,in2,vt,is_var_shift) {}
607 virtual int Opcode() const;
608 };
609
610 //------------------------------LShiftVLNode-----------------------------------
611 // Vector left shift longs
612 class LShiftVLNode : public ShiftVNode {
613 public:
614 LShiftVLNode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
615 ShiftVNode(in1,in2,vt,is_var_shift) {}
616 virtual int Opcode() const;
617 };
618
619 //------------------------------RShiftVBNode-----------------------------------
620 // Vector right arithmetic (signed) shift bytes
621 class RShiftVBNode : public ShiftVNode {
622 public:
623 RShiftVBNode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
624 ShiftVNode(in1,in2,vt,is_var_shift) {}
625 virtual int Opcode() const;
626 };
627
628 //------------------------------RShiftVSNode-----------------------------------
629 // Vector right arithmetic (signed) shift shorts
630 class RShiftVSNode : public ShiftVNode {
631 public:
632 RShiftVSNode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
633 ShiftVNode(in1,in2,vt,is_var_shift) {}
634 virtual int Opcode() const;
635 };
636
637 //------------------------------RShiftVINode-----------------------------------
638 // Vector right arithmetic (signed) shift ints
639 class RShiftVINode : public ShiftVNode {
640 public:
641 RShiftVINode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
642 ShiftVNode(in1,in2,vt,is_var_shift) {}
643 virtual int Opcode() const;
644 };
645
646 //------------------------------RShiftVLNode-----------------------------------
647 // Vector right arithmetic (signed) shift longs
648 class RShiftVLNode : public ShiftVNode {
649 public:
650 RShiftVLNode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
651 ShiftVNode(in1,in2,vt,is_var_shift) {}
652 virtual int Opcode() const;
653 };
654
655 //------------------------------URShiftVBNode----------------------------------
656 // Vector right logical (unsigned) shift bytes
657 class URShiftVBNode : public ShiftVNode {
658 public:
659 URShiftVBNode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
660 ShiftVNode(in1,in2,vt,is_var_shift) {}
661 virtual int Opcode() const;
662 };
663
664 //------------------------------URShiftVSNode----------------------------------
665 // Vector right logical (unsigned) shift shorts
666 class URShiftVSNode : public ShiftVNode {
667 public:
668 URShiftVSNode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
669 ShiftVNode(in1,in2,vt,is_var_shift) {}
670 virtual int Opcode() const;
671 };
672
673 //------------------------------URShiftVINode----------------------------------
674 // Vector right logical (unsigned) shift ints
675 class URShiftVINode : public ShiftVNode {
676 public:
677 URShiftVINode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
678 ShiftVNode(in1,in2,vt,is_var_shift) {}
679 virtual int Opcode() const;
680 };
681
682 //------------------------------URShiftVLNode----------------------------------
683 // Vector right logical (unsigned) shift longs
684 class URShiftVLNode : public ShiftVNode {
685 public:
686 URShiftVLNode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
687 ShiftVNode(in1,in2,vt,is_var_shift) {}
688 virtual int Opcode() const;
689 };
690
691 //------------------------------LShiftCntVNode---------------------------------
692 // Vector left shift count
693 class LShiftCntVNode : public VectorNode {
694 public:
695 LShiftCntVNode(Node* cnt, const TypeVect* vt) : VectorNode(cnt,vt) {}
696 virtual int Opcode() const;
697 };
698
699 //------------------------------RShiftCntVNode---------------------------------
700 // Vector right shift count
701 class RShiftCntVNode : public VectorNode {
702 public:
703 RShiftCntVNode(Node* cnt, const TypeVect* vt) : VectorNode(cnt,vt) {}
704 virtual int Opcode() const;
705 };
706
707 //------------------------------AndVNode---------------------------------------
708 // Vector and integer
709 class AndVNode : public VectorNode {
710 public:
711 AndVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
712 virtual int Opcode() const;
713 };
714
715 //------------------------------AndReductionVNode--------------------------------------
716 // Vector and byte, short, int, long as a reduction
717 class AndReductionVNode : public ReductionNode {
718 public:
719 AndReductionVNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
720 virtual int Opcode() const;
721 };
722
723 //------------------------------OrVNode---------------------------------------
724 // Vector or byte, short, int, long as a reduction
725 class OrVNode : public VectorNode {
726 public:
727 OrVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
728 virtual int Opcode() const;
729 };
730
731 //------------------------------OrReductionVNode--------------------------------------
732 // Vector xor byte, short, int, long as a reduction
733 class OrReductionVNode : public ReductionNode {
734 public:
735 OrReductionVNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
736 virtual int Opcode() const;
737 };
738
739 //------------------------------XorReductionVNode--------------------------------------
740 // Vector and int, long as a reduction
741 class XorReductionVNode : public ReductionNode {
742 public:
743 XorReductionVNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
744 virtual int Opcode() const;
745 };
746
747 //------------------------------XorVNode---------------------------------------
748 // Vector xor integer
749 class XorVNode : public VectorNode {
750 public:
751 XorVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
752 virtual int Opcode() const;
753 };
754
755 //------------------------------MinReductionVNode--------------------------------------
756 // Vector min byte, short, int, long, float, double as a reduction
757 class MinReductionVNode : public ReductionNode {
758 public:
759 MinReductionVNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
760 virtual int Opcode() const;
761 };
762
763 //------------------------------MaxReductionVNode--------------------------------------
764 // Vector min byte, short, int, long, float, double as a reduction
765 class MaxReductionVNode : public ReductionNode {
766 public:
767 MaxReductionVNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
768 virtual int Opcode() const;
769 };
770
771 //------------------------------CompressVNode--------------------------------------
772 // Vector compress
773 class CompressVNode: public VectorNode {
774 public:
775 CompressVNode(Node* vec, Node* mask, const TypeVect* vt) :
776 VectorNode(vec, mask, vt) {
777 init_class_id(Class_CompressV);
778 }
779 virtual int Opcode() const;
780 };
781
782 class CompressMNode: public VectorNode {
783 public:
784 CompressMNode(Node* mask, const TypeVect* vt) :
785 VectorNode(mask, vt) {
786 init_class_id(Class_CompressM);
787 }
788 virtual int Opcode() const;
789 };
790
791 //------------------------------ExpandVNode--------------------------------------
792 // Vector expand
793 class ExpandVNode: public VectorNode {
794 public:
795 ExpandVNode(Node* vec, Node* mask, const TypeVect* vt) :
796 VectorNode(vec, mask, vt) {
797 init_class_id(Class_ExpandV);
798 }
799 virtual int Opcode() const;
800 };
801
802 //================================= M E M O R Y ===============================
803
804 //------------------------------LoadVectorNode---------------------------------
805 // Load Vector from memory
806 class LoadVectorNode : public LoadNode {
807 public:
808 LoadVectorNode(Node* c, Node* mem, Node* adr, const TypePtr* at, const TypeVect* vt, ControlDependency control_dependency = LoadNode::DependsOnlyOnTest)
809 : LoadNode(c, mem, adr, at, vt, MemNode::unordered, control_dependency) {
810 init_class_id(Class_LoadVector);
811 set_mismatched_access();
812 }
813
814 const TypeVect* vect_type() const { return type()->is_vect(); }
815 uint length() const { return vect_type()->length(); } // Vector length
816
817 virtual int Opcode() const;
818
819 virtual uint ideal_reg() const { return Matcher::vector_ideal_reg(memory_size()); }
820 virtual BasicType memory_type() const { return T_VOID; }
821 virtual int memory_size() const { return vect_type()->length_in_bytes(); }
822
823 virtual int store_Opcode() const { return Op_StoreVector; }
824
825 static LoadVectorNode* make(int opc, Node* ctl, Node* mem,
826 Node* adr, const TypePtr* atyp,
827 uint vlen, BasicType bt,
828 ControlDependency control_dependency = LoadNode::DependsOnlyOnTest);
829 uint element_size(void) { return type2aelembytes(vect_type()->element_basic_type()); }
830 };
831
832 //------------------------------LoadVectorGatherNode------------------------------
833 // Load Vector from memory via index map
834 class LoadVectorGatherNode : public LoadVectorNode {
835 public:
836 LoadVectorGatherNode(Node* c, Node* mem, Node* adr, const TypePtr* at, const TypeVect* vt, Node* indices)
837 : LoadVectorNode(c, mem, adr, at, vt) {
838 init_class_id(Class_LoadVectorGather);
839 assert(indices->bottom_type()->is_vect(), "indices must be in vector");
840 add_req(indices);
841 assert(req() == MemNode::ValueIn + 1, "match_edge expects that last input is in MemNode::ValueIn");
842 }
843
844 virtual int Opcode() const;
845 virtual uint match_edge(uint idx) const { return idx == MemNode::Address || idx == MemNode::ValueIn; }
846 };
847
848 //------------------------------StoreVectorNode--------------------------------
849 // Store Vector to memory
850 class StoreVectorNode : public StoreNode {
851 private:
852 const TypeVect* _vect_type;
853 public:
854 StoreVectorNode(Node* c, Node* mem, Node* adr, const TypePtr* at, Node* val)
855 : StoreNode(c, mem, adr, at, val, MemNode::unordered), _vect_type(val->bottom_type()->is_vect()) {
856 init_class_id(Class_StoreVector);
857 set_mismatched_access();
858 }
859
860 const TypeVect* vect_type() const { return _vect_type; }
861 uint length() const { return vect_type()->length(); } // Vector length
862
863 virtual int Opcode() const;
864
865 virtual uint ideal_reg() const { return Matcher::vector_ideal_reg(memory_size()); }
866 virtual BasicType memory_type() const { return T_VOID; }
867 virtual int memory_size() const { return vect_type()->length_in_bytes(); }
868
869 static StoreVectorNode* make(int opc, Node* ctl, Node* mem,
870 Node* adr, const TypePtr* atyp, Node* val,
871 uint vlen);
872
873 uint element_size(void) { return type2aelembytes(vect_type()->element_basic_type()); }
874
875 // Needed for proper cloning.
876 virtual uint size_of() const { return sizeof(*this); }
877 };
878
879 //------------------------------StoreVectorScatterNode------------------------------
880 // Store Vector into memory via index map
881
882 class StoreVectorScatterNode : public StoreVectorNode {
883 public:
884 StoreVectorScatterNode(Node* c, Node* mem, Node* adr, const TypePtr* at, Node* val, Node* indices)
885 : StoreVectorNode(c, mem, adr, at, val) {
886 init_class_id(Class_StoreVectorScatter);
887 assert(indices->bottom_type()->is_vect(), "indices must be in vector");
888 add_req(indices);
889 assert(req() == MemNode::ValueIn + 2, "match_edge expects that last input is in MemNode::ValueIn+1");
890 }
891 virtual int Opcode() const;
892 virtual uint match_edge(uint idx) const { return idx == MemNode::Address ||
893 idx == MemNode::ValueIn ||
894 idx == MemNode::ValueIn + 1; }
895 };
896
897 //------------------------------StoreVectorMaskedNode--------------------------------
898 // Store Vector to memory under the influence of a predicate register(mask).
899 class StoreVectorMaskedNode : public StoreVectorNode {
900 public:
901 StoreVectorMaskedNode(Node* c, Node* mem, Node* dst, Node* src, const TypePtr* at, Node* mask)
902 : StoreVectorNode(c, mem, dst, at, src) {
903 assert(mask->bottom_type()->isa_vectmask(), "sanity");
904 init_class_id(Class_StoreVector);
905 set_mismatched_access();
906 add_req(mask);
907 }
908
909 virtual int Opcode() const;
910
911 virtual uint match_edge(uint idx) const {
912 return idx > 1;
913 }
914 Node* Ideal(PhaseGVN* phase, bool can_reshape);
915 };
916
917 //------------------------------LoadVectorMaskedNode--------------------------------
918 // Load Vector from memory under the influence of a predicate register(mask).
919 class LoadVectorMaskedNode : public LoadVectorNode {
920 public:
921 LoadVectorMaskedNode(Node* c, Node* mem, Node* src, const TypePtr* at, const TypeVect* vt, Node* mask)
922 : LoadVectorNode(c, mem, src, at, vt) {
923 assert(mask->bottom_type()->isa_vectmask(), "sanity");
924 init_class_id(Class_LoadVector);
925 set_mismatched_access();
926 add_req(mask);
927 }
928
929 virtual int Opcode() const;
930
931 virtual uint match_edge(uint idx) const {
932 return idx > 1;
933 }
934 Node* Ideal(PhaseGVN* phase, bool can_reshape);
935 };
936
937 //-------------------------------LoadVectorGatherMaskedNode---------------------------------
938 // Load Vector from memory via index map under the influence of a predicate register(mask).
939 class LoadVectorGatherMaskedNode : public LoadVectorNode {
940 public:
941 LoadVectorGatherMaskedNode(Node* c, Node* mem, Node* adr, const TypePtr* at, const TypeVect* vt, Node* indices, Node* mask)
942 : LoadVectorNode(c, mem, adr, at, vt) {
943 init_class_id(Class_LoadVector);
944 assert(indices->bottom_type()->is_vect(), "indices must be in vector");
945 assert(mask->bottom_type()->isa_vectmask(), "sanity");
946 add_req(indices);
947 add_req(mask);
948 assert(req() == MemNode::ValueIn + 2, "match_edge expects that last input is in MemNode::ValueIn+1");
949 }
950
951 virtual int Opcode() const;
952 virtual uint match_edge(uint idx) const { return idx == MemNode::Address ||
953 idx == MemNode::ValueIn ||
954 idx == MemNode::ValueIn + 1; }
955 };
956
957 //------------------------------StoreVectorScatterMaskedNode--------------------------------
958 // Store Vector into memory via index map under the influence of a predicate register(mask).
959 class StoreVectorScatterMaskedNode : public StoreVectorNode {
960 public:
961 StoreVectorScatterMaskedNode(Node* c, Node* mem, Node* adr, const TypePtr* at, Node* val, Node* indices, Node* mask)
962 : StoreVectorNode(c, mem, adr, at, val) {
963 init_class_id(Class_StoreVector);
964 assert(indices->bottom_type()->is_vect(), "indices must be in vector");
965 assert(mask->bottom_type()->isa_vectmask(), "sanity");
966 add_req(indices);
967 add_req(mask);
968 assert(req() == MemNode::ValueIn + 3, "match_edge expects that last input is in MemNode::ValueIn+2");
969 }
970 virtual int Opcode() const;
971 virtual uint match_edge(uint idx) const { return idx == MemNode::Address ||
972 idx == MemNode::ValueIn ||
973 idx == MemNode::ValueIn + 1 ||
974 idx == MemNode::ValueIn + 2; }
975 };
976
977 //------------------------------VectorCmpMaskedNode--------------------------------
978 // Vector Comparison under the influence of a predicate register(mask).
979 class VectorCmpMaskedNode : public TypeNode {
980 public:
981 VectorCmpMaskedNode(Node* src1, Node* src2, Node* mask, const Type* ty): TypeNode(ty, 4) {
982 init_req(1, src1);
983 init_req(2, src2);
984 init_req(3, mask);
985 }
986
987 virtual int Opcode() const;
988 };
989
990 //------------------------------VectorMaskGenNode----------------------------------
991 class VectorMaskGenNode : public TypeNode {
992 public:
993 VectorMaskGenNode(Node* length, const Type* ty): TypeNode(ty, 2) {
994 init_req(1, length);
995 }
996
997 virtual int Opcode() const;
998 virtual uint ideal_reg() const { return Op_RegVectMask; }
999 static Node* make(Node* length, BasicType vmask_bt);
1000 };
1001
1002 //------------------------------VectorMaskOpNode-----------------------------------
1003 class VectorMaskOpNode : public TypeNode {
1004 public:
1005 VectorMaskOpNode(Node* mask, const Type* ty, int mopc):
1006 TypeNode(ty, 2), _mopc(mopc) {
1007 assert(Matcher::has_predicated_vectors() || mask->bottom_type()->is_vect()->element_basic_type() == T_BOOLEAN, "");
1008 init_req(1, mask);
1009 }
1010
1011 virtual int Opcode() const;
1012 virtual uint size_of() const { return sizeof(VectorMaskOpNode); }
1013 virtual uint ideal_reg() const { return Op_RegI; }
1014 int get_mask_Opcode() const { return _mopc;}
1015 static Node* make(Node* mask, const Type* ty, int mopc);
1016
1017 private:
1018 int _mopc;
1019 };
1020
1021 class VectorMaskTrueCountNode : public VectorMaskOpNode {
1022 public:
1023 VectorMaskTrueCountNode(Node* mask, const Type* ty):
1024 VectorMaskOpNode(mask, ty, Op_VectorMaskTrueCount) {}
1025 virtual int Opcode() const;
1026 };
1027
1028 class VectorMaskFirstTrueNode : public VectorMaskOpNode {
1029 public:
1030 VectorMaskFirstTrueNode(Node* mask, const Type* ty):
1031 VectorMaskOpNode(mask, ty, Op_VectorMaskFirstTrue) {}
1032 virtual int Opcode() const;
1033 };
1034
1035 class VectorMaskLastTrueNode : public VectorMaskOpNode {
1036 public:
1037 VectorMaskLastTrueNode(Node* mask, const Type* ty):
1038 VectorMaskOpNode(mask, ty, Op_VectorMaskLastTrue) {}
1039 virtual int Opcode() const;
1040 };
1041
1042 class VectorMaskToLongNode : public VectorMaskOpNode {
1043 public:
1044 VectorMaskToLongNode(Node* mask, const Type* ty):
1045 VectorMaskOpNode(mask, ty, Op_VectorMaskToLong) {}
1046 virtual int Opcode() const;
1047 virtual uint ideal_reg() const { return Op_RegL; }
1048 virtual Node* Identity(PhaseGVN* phase);
1049 };
1050
1051 class VectorLongToMaskNode : public VectorNode {
1052 public:
1053 VectorLongToMaskNode(Node* mask, const TypeVect* ty):
1054 VectorNode(mask, ty) {
1055 }
1056 virtual int Opcode() const;
1057 Node* Ideal(PhaseGVN* phase, bool can_reshape);
1058 };
1059
1060 //-------------------------- Vector mask broadcast -----------------------------------
1061 class MaskAllNode : public VectorNode {
1062 public:
1063 MaskAllNode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {}
1064 virtual int Opcode() const;
1065 };
1066
1067 //--------------------------- Vector mask logical and --------------------------------
1068 class AndVMaskNode : public VectorNode {
1069 public:
1070 AndVMaskNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
1071 virtual int Opcode() const;
1072 };
1073
1074 //--------------------------- Vector mask logical or ---------------------------------
1075 class OrVMaskNode : public VectorNode {
1076 public:
1077 OrVMaskNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
1078 virtual int Opcode() const;
1079 };
1080
1081 //--------------------------- Vector mask logical xor --------------------------------
1082 class XorVMaskNode : public VectorNode {
1083 public:
1084 XorVMaskNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
1085 virtual int Opcode() const;
1086 };
1087
1088 //=========================Promote_Scalar_to_Vector============================
1089
1090 //------------------------------ReplicateBNode---------------------------------
1091 // Replicate byte scalar to be vector
1092 class ReplicateBNode : public VectorNode {
1093 public:
1094 ReplicateBNode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
1095 virtual int Opcode() const;
1096 };
1097
1098 //------------------------------ReplicateSNode---------------------------------
1099 // Replicate short scalar to be vector
1100 class ReplicateSNode : public VectorNode {
1101 public:
1102 ReplicateSNode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
1103 virtual int Opcode() const;
1104 };
1105
1106 //------------------------------ReplicateINode---------------------------------
1107 // Replicate int scalar to be vector
1108 class ReplicateINode : public VectorNode {
1109 public:
1110 ReplicateINode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
1111 virtual int Opcode() const;
1112 };
1113
1114 //------------------------------ReplicateLNode---------------------------------
1115 // Replicate long scalar to be vector
1116 class ReplicateLNode : public VectorNode {
1117 public:
1118 ReplicateLNode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
1119 virtual int Opcode() const;
1120 };
1121
1122 //------------------------------ReplicateFNode---------------------------------
1123 // Replicate float scalar to be vector
1124 class ReplicateFNode : public VectorNode {
1125 public:
1126 ReplicateFNode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
1127 virtual int Opcode() const;
1128 };
1129
1130 //------------------------------ReplicateDNode---------------------------------
1131 // Replicate double scalar to be vector
1132 class ReplicateDNode : public VectorNode {
1133 public:
1134 ReplicateDNode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
1135 virtual int Opcode() const;
1136 };
1137
1138 //========================Pack_Scalars_into_a_Vector===========================
1139
1140 //------------------------------PackNode---------------------------------------
1141 // Pack parent class (not for code generation).
1142 class PackNode : public VectorNode {
1143 public:
1144 PackNode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
1145 PackNode(Node* in1, Node* n2, const TypeVect* vt) : VectorNode(in1, n2, vt) {}
1146 virtual int Opcode() const;
1147
1148 void add_opd(Node* n) {
1149 add_req(n);
1150 }
1151
1152 // Create a binary tree form for Packs. [lo, hi) (half-open) range
1153 PackNode* binary_tree_pack(int lo, int hi);
1154
1155 static PackNode* make(Node* s, uint vlen, BasicType bt);
1156 };
1157
1158 //------------------------------PackBNode--------------------------------------
1159 // Pack byte scalars into vector
1160 class PackBNode : public PackNode {
1161 public:
1162 PackBNode(Node* in1, const TypeVect* vt) : PackNode(in1, vt) {}
1163 virtual int Opcode() const;
1164 };
1165
1166 //------------------------------PackSNode--------------------------------------
1167 // Pack short scalars into a vector
1168 class PackSNode : public PackNode {
1169 public:
1170 PackSNode(Node* in1, const TypeVect* vt) : PackNode(in1, vt) {}
1171 PackSNode(Node* in1, Node* in2, const TypeVect* vt) : PackNode(in1, in2, vt) {}
1172 virtual int Opcode() const;
1173 };
1174
1175 //------------------------------PackINode--------------------------------------
1176 // Pack integer scalars into a vector
1177 class PackINode : public PackNode {
1178 public:
1179 PackINode(Node* in1, const TypeVect* vt) : PackNode(in1, vt) {}
1180 PackINode(Node* in1, Node* in2, const TypeVect* vt) : PackNode(in1, in2, vt) {}
1181 virtual int Opcode() const;
1182 };
1183
1184 //------------------------------PackLNode--------------------------------------
1185 // Pack long scalars into a vector
1186 class PackLNode : public PackNode {
1187 public:
1188 PackLNode(Node* in1, const TypeVect* vt) : PackNode(in1, vt) {}
1189 PackLNode(Node* in1, Node* in2, const TypeVect* vt) : PackNode(in1, in2, vt) {}
1190 virtual int Opcode() const;
1191 };
1192
1193 //------------------------------Pack2LNode-------------------------------------
1194 // Pack 2 long scalars into a vector
1195 class Pack2LNode : public PackNode {
1196 public:
1197 Pack2LNode(Node* in1, Node* in2, const TypeVect* vt) : PackNode(in1, in2, vt) {}
1198 virtual int Opcode() const;
1199 };
1200
1201 //------------------------------PackFNode--------------------------------------
1202 // Pack float scalars into vector
1203 class PackFNode : public PackNode {
1204 public:
1205 PackFNode(Node* in1, const TypeVect* vt) : PackNode(in1, vt) {}
1206 PackFNode(Node* in1, Node* in2, const TypeVect* vt) : PackNode(in1, in2, vt) {}
1207 virtual int Opcode() const;
1208 };
1209
1210 //------------------------------PackDNode--------------------------------------
1211 // Pack double scalars into a vector
1212 class PackDNode : public PackNode {
1213 public:
1214 PackDNode(Node* in1, const TypeVect* vt) : PackNode(in1, vt) {}
1215 PackDNode(Node* in1, Node* in2, const TypeVect* vt) : PackNode(in1, in2, vt) {}
1216 virtual int Opcode() const;
1217 };
1218
1219 //------------------------------Pack2DNode-------------------------------------
1220 // Pack 2 double scalars into a vector
1221 class Pack2DNode : public PackNode {
1222 public:
1223 Pack2DNode(Node* in1, Node* in2, const TypeVect* vt) : PackNode(in1, in2, vt) {}
1224 virtual int Opcode() const;
1225 };
1226
1227
1228 class VectorLoadConstNode : public VectorNode {
1229 public:
1230 VectorLoadConstNode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
1231 virtual int Opcode() const;
1232 };
1233
1234 //========================Extract_Scalar_from_Vector===========================
1235
1236 //------------------------------ExtractNode------------------------------------
1237 // Extract a scalar from a vector at position "pos"
1238 class ExtractNode : public Node {
1239 public:
1240 ExtractNode(Node* src, ConINode* pos) : Node(NULL, src, (Node*)pos) {
1241 assert(in(2)->get_int() >= 0, "positive constants");
1242 }
1243 virtual int Opcode() const;
1244 uint pos() const { return in(2)->get_int(); }
1245
1246 static Node* make(Node* v, uint position, BasicType bt);
1247 static int opcode(BasicType bt);
1248 };
1249
1250 //------------------------------ExtractBNode-----------------------------------
1251 // Extract a byte from a vector at position "pos"
1252 class ExtractBNode : public ExtractNode {
1253 public:
1254 ExtractBNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1255 virtual int Opcode() const;
1256 virtual const Type *bottom_type() const { return TypeInt::INT; }
1257 virtual uint ideal_reg() const { return Op_RegI; }
1258 };
1259
1260 //------------------------------ExtractUBNode----------------------------------
1261 // Extract a boolean from a vector at position "pos"
1262 class ExtractUBNode : public ExtractNode {
1263 public:
1264 ExtractUBNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1265 virtual int Opcode() const;
1266 virtual const Type *bottom_type() const { return TypeInt::INT; }
1267 virtual uint ideal_reg() const { return Op_RegI; }
1268 };
1269
1270 //------------------------------ExtractCNode-----------------------------------
1271 // Extract a char from a vector at position "pos"
1272 class ExtractCNode : public ExtractNode {
1273 public:
1274 ExtractCNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1275 virtual int Opcode() const;
1276 virtual const Type *bottom_type() const { return TypeInt::CHAR; }
1277 virtual uint ideal_reg() const { return Op_RegI; }
1278 };
1279
1280 //------------------------------ExtractSNode-----------------------------------
1281 // Extract a short from a vector at position "pos"
1282 class ExtractSNode : public ExtractNode {
1283 public:
1284 ExtractSNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1285 virtual int Opcode() const;
1286 virtual const Type *bottom_type() const { return TypeInt::SHORT; }
1287 virtual uint ideal_reg() const { return Op_RegI; }
1288 };
1289
1290 //------------------------------ExtractINode-----------------------------------
1291 // Extract an int from a vector at position "pos"
1292 class ExtractINode : public ExtractNode {
1293 public:
1294 ExtractINode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1295 virtual int Opcode() const;
1296 virtual const Type *bottom_type() const { return TypeInt::INT; }
1297 virtual uint ideal_reg() const { return Op_RegI; }
1298 };
1299
1300 //------------------------------ExtractLNode-----------------------------------
1301 // Extract a long from a vector at position "pos"
1302 class ExtractLNode : public ExtractNode {
1303 public:
1304 ExtractLNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1305 virtual int Opcode() const;
1306 virtual const Type *bottom_type() const { return TypeLong::LONG; }
1307 virtual uint ideal_reg() const { return Op_RegL; }
1308 };
1309
1310 //------------------------------ExtractFNode-----------------------------------
1311 // Extract a float from a vector at position "pos"
1312 class ExtractFNode : public ExtractNode {
1313 public:
1314 ExtractFNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1315 virtual int Opcode() const;
1316 virtual const Type *bottom_type() const { return Type::FLOAT; }
1317 virtual uint ideal_reg() const { return Op_RegF; }
1318 };
1319
1320 //------------------------------ExtractDNode-----------------------------------
1321 // Extract a double from a vector at position "pos"
1322 class ExtractDNode : public ExtractNode {
1323 public:
1324 ExtractDNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1325 virtual int Opcode() const;
1326 virtual const Type *bottom_type() const { return Type::DOUBLE; }
1327 virtual uint ideal_reg() const { return Op_RegD; }
1328 };
1329
1330 //------------------------------MacroLogicVNode-------------------------------
1331 // Vector logical operations packing node.
1332 class MacroLogicVNode : public VectorNode {
1333 private:
1334 MacroLogicVNode(Node* in1, Node* in2, Node* in3, Node* fn, Node* mask, const TypeVect* vt)
1335 : VectorNode(in1, in2, in3, fn, vt) {
1336 if (mask) {
1337 this->add_req(mask);
1338 this->add_flag(Node::Flag_is_predicated_vector);
1339 }
1340 }
1341
1342 public:
1343 virtual int Opcode() const;
1344
1345 static MacroLogicVNode* make(PhaseGVN& igvn, Node* in1, Node* in2, Node* in3,
1346 Node* mask, uint truth_table, const TypeVect* vt);
1347 };
1348
1349 class VectorMaskCmpNode : public VectorNode {
1350 private:
1351 BoolTest::mask _predicate;
1352
1353 protected:
1354 virtual uint size_of() const { return sizeof(VectorMaskCmpNode); }
1355
1356 public:
1357 VectorMaskCmpNode(BoolTest::mask predicate, Node* in1, Node* in2, ConINode* predicate_node, const TypeVect* vt) :
1358 VectorNode(in1, in2, predicate_node, vt),
1359 _predicate(predicate) {
1360 assert(in1->bottom_type()->is_vect()->element_basic_type() == in2->bottom_type()->is_vect()->element_basic_type(),
1361 "VectorMaskCmp inputs must have same type for elements");
1362 assert(in1->bottom_type()->is_vect()->length() == in2->bottom_type()->is_vect()->length(),
1363 "VectorMaskCmp inputs must have same number of elements");
1364 assert((BoolTest::mask)predicate_node->get_int() == predicate, "Unmatched predicates");
1365 init_class_id(Class_VectorMaskCmp);
1366 }
1367
1368 virtual int Opcode() const;
1369 virtual uint hash() const { return VectorNode::hash() + _predicate; }
1370 virtual bool cmp( const Node &n ) const {
1371 return VectorNode::cmp(n) && _predicate == ((VectorMaskCmpNode&)n)._predicate;
1372 }
1373 BoolTest::mask get_predicate() { return _predicate; }
1374 #ifndef PRODUCT
1375 virtual void dump_spec(outputStream *st) const;
1376 #endif // !PRODUCT
1377 };
1378
1379 // Used to wrap other vector nodes in order to add masking functionality.
1380 class VectorMaskWrapperNode : public VectorNode {
1381 public:
1382 VectorMaskWrapperNode(Node* vector, Node* mask)
1383 : VectorNode(vector, mask, vector->bottom_type()->is_vect()) {
1384 assert(mask->is_VectorMaskCmp(), "VectorMaskWrapper requires that second argument be a mask");
1385 }
1386
1387 virtual int Opcode() const;
1388 Node* vector_val() const { return in(1); }
1389 Node* vector_mask() const { return in(2); }
1390 };
1391
1392 class VectorTestNode : public Node {
1393 private:
1394 BoolTest::mask _predicate;
1395
1396 protected:
1397 uint size_of() const { return sizeof(*this); }
1398
1399 public:
1400 VectorTestNode(Node* in1, Node* in2, BoolTest::mask predicate) : Node(NULL, in1, in2), _predicate(predicate) {
1401 assert(in2->bottom_type()->is_vect() == in2->bottom_type()->is_vect(), "same vector type");
1402 }
1403 virtual int Opcode() const;
1404 virtual uint hash() const { return Node::hash() + _predicate; }
1405 virtual bool cmp( const Node &n ) const {
1406 return Node::cmp(n) && _predicate == ((VectorTestNode&)n)._predicate;
1407 }
1408 virtual const Type *bottom_type() const { return TypeInt::BOOL; }
1409 virtual uint ideal_reg() const { return Op_RegI; } // TODO Should be RegFlags but due to missing comparison flags for BoolTest
1410 // in middle-end, we make it boolean result directly.
1411 BoolTest::mask get_predicate() const { return _predicate; }
1412 };
1413
1414 class VectorBlendNode : public VectorNode {
1415 public:
1416 VectorBlendNode(Node* vec1, Node* vec2, Node* mask)
1417 : VectorNode(vec1, vec2, mask, vec1->bottom_type()->is_vect()) {
1418 }
1419
1420 virtual int Opcode() const;
1421 Node* vec1() const { return in(1); }
1422 Node* vec2() const { return in(2); }
1423 Node* vec_mask() const { return in(3); }
1424 };
1425
1426 class VectorRearrangeNode : public VectorNode {
1427 public:
1428 VectorRearrangeNode(Node* vec1, Node* shuffle)
1429 : VectorNode(vec1, shuffle, vec1->bottom_type()->is_vect()) {
1430 // assert(mask->is_VectorMask(), "VectorBlendNode requires that third argument be a mask");
1431 }
1432
1433 virtual int Opcode() const;
1434 Node* vec1() const { return in(1); }
1435 Node* vec_shuffle() const { return in(2); }
1436 };
1437
1438 class VectorLoadShuffleNode : public VectorNode {
1439 public:
1440 VectorLoadShuffleNode(Node* in, const TypeVect* vt)
1441 : VectorNode(in, vt) {
1442 assert(in->bottom_type()->is_vect()->element_basic_type() == T_BYTE, "must be BYTE");
1443 }
1444
1445 int GetOutShuffleSize() const { return type2aelembytes(vect_type()->element_basic_type()); }
1446 virtual int Opcode() const;
1447 };
1448
1449 class VectorLoadMaskNode : public VectorNode {
1450 public:
1451 VectorLoadMaskNode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {
1452 assert(in->bottom_type()->is_vect()->element_basic_type() == T_BOOLEAN, "must be boolean");
1453 }
1454
1455 virtual int Opcode() const;
1456 virtual Node* Identity(PhaseGVN* phase);
1457 };
1458
1459 class VectorStoreMaskNode : public VectorNode {
1460 protected:
1461 VectorStoreMaskNode(Node* in1, ConINode* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
1462
1463 public:
1464 virtual int Opcode() const;
1465 virtual Node* Identity(PhaseGVN* phase);
1466
1467 static VectorStoreMaskNode* make(PhaseGVN& gvn, Node* in, BasicType in_type, uint num_elem);
1468 };
1469
1470 class VectorMaskCastNode : public VectorNode {
1471 public:
1472 VectorMaskCastNode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {
1473 const TypeVect* in_vt = in->bottom_type()->is_vect();
1474 assert(in_vt->length() == vt->length(), "vector length must match");
1475 }
1476 static Node* makeCastNode(PhaseGVN* phase, Node* in1, const TypeVect * vt);
1477 virtual int Opcode() const;
1478 };
1479
1480 // This is intended for use as a simple reinterpret node that has no cast.
1481 class VectorReinterpretNode : public VectorNode {
1482 private:
1483 const TypeVect* _src_vt;
1484
1485 protected:
1486 uint size_of() const { return sizeof(VectorReinterpretNode); }
1487 public:
1488 VectorReinterpretNode(Node* in, const TypeVect* src_vt, const TypeVect* dst_vt)
1489 : VectorNode(in, dst_vt), _src_vt(src_vt) {
1490 assert((!dst_vt->isa_vectmask() && !src_vt->isa_vectmask()) ||
1491 (type2aelembytes(src_vt->element_basic_type()) >= type2aelembytes(dst_vt->element_basic_type())),
1492 "unsupported mask widening reinterpretation");
1493 init_class_id(Class_VectorReinterpret);
1494 }
1495
1496 const TypeVect* src_type() { return _src_vt; }
1497 virtual uint hash() const { return VectorNode::hash() + _src_vt->hash(); }
1498 virtual bool cmp( const Node &n ) const {
1499 return VectorNode::cmp(n) && !Type::cmp(_src_vt,((VectorReinterpretNode&)n)._src_vt);
1500 }
1501 virtual Node* Identity(PhaseGVN* phase);
1502
1503 virtual int Opcode() const;
1504 };
1505
1506 class VectorCastNode : public VectorNode {
1507 public:
1508 VectorCastNode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {}
1509 virtual int Opcode() const;
1510
1511 static VectorCastNode* make(int vopc, Node* n1, BasicType bt, uint vlen);
1512 static int opcode(BasicType bt, bool is_signed = true);
1513 static bool implemented(BasicType bt, uint vlen);
1514
1515 virtual Node* Identity(PhaseGVN* phase);
1516 };
1517
1518 class VectorCastB2XNode : public VectorCastNode {
1519 public:
1520 VectorCastB2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) {
1521 assert(in->bottom_type()->is_vect()->element_basic_type() == T_BYTE, "must be byte");
1522 }
1523 virtual int Opcode() const;
1524 };
1525
1526 class VectorCastS2XNode : public VectorCastNode {
1527 public:
1528 VectorCastS2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) {
1529 assert(in->bottom_type()->is_vect()->element_basic_type() == T_SHORT, "must be short");
1530 }
1531 virtual int Opcode() const;
1532 };
1533
1534 class VectorCastI2XNode : public VectorCastNode {
1535 public:
1536 VectorCastI2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) {
1537 assert(in->bottom_type()->is_vect()->element_basic_type() == T_INT, "must be int");
1538 }
1539 virtual int Opcode() const;
1540 };
1541
1542 class VectorCastL2XNode : public VectorCastNode {
1543 public:
1544 VectorCastL2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) {
1545 assert(in->bottom_type()->is_vect()->element_basic_type() == T_LONG, "must be long");
1546 }
1547 virtual int Opcode() const;
1548 };
1549
1550 class VectorCastF2XNode : public VectorCastNode {
1551 public:
1552 VectorCastF2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) {
1553 assert(in->bottom_type()->is_vect()->element_basic_type() == T_FLOAT, "must be float");
1554 }
1555 virtual int Opcode() const;
1556 };
1557
1558 class VectorCastD2XNode : public VectorCastNode {
1559 public:
1560 VectorCastD2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) {
1561 assert(in->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE, "must be double");
1562 }
1563 virtual int Opcode() const;
1564 };
1565
1566 class RoundVFNode : public VectorNode {
1567 public:
1568 RoundVFNode(Node* in, const TypeVect* vt) :VectorNode(in, vt) {
1569 assert(in->bottom_type()->is_vect()->element_basic_type() == T_FLOAT, "must be float");
1570 }
1571 virtual int Opcode() const;
1572 };
1573
1574 class VectorUCastB2XNode : public VectorCastNode {
1575 public:
1576 VectorUCastB2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) {
1577 assert(in->bottom_type()->is_vect()->element_basic_type() == T_BYTE, "must be byte");
1578 }
1579 virtual int Opcode() const;
1580 };
1581
1582 class RoundVDNode : public VectorNode {
1583 public:
1584 RoundVDNode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {
1585 assert(in->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE, "must be double");
1586 }
1587 virtual int Opcode() const;
1588 };
1589
1590 class VectorUCastS2XNode : public VectorCastNode {
1591 public:
1592 VectorUCastS2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) {
1593 assert(in->bottom_type()->is_vect()->element_basic_type() == T_SHORT, "must be short");
1594 }
1595 virtual int Opcode() const;
1596 };
1597
1598 class VectorUCastI2XNode : public VectorCastNode {
1599 public:
1600 VectorUCastI2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) {
1601 assert(in->bottom_type()->is_vect()->element_basic_type() == T_INT, "must be int");
1602 }
1603 virtual int Opcode() const;
1604 };
1605
1606 class VectorInsertNode : public VectorNode {
1607 public:
1608 VectorInsertNode(Node* vsrc, Node* new_val, ConINode* pos, const TypeVect* vt) : VectorNode(vsrc, new_val, (Node*)pos, vt) {
1609 assert(pos->get_int() >= 0, "positive constants");
1610 assert(pos->get_int() < (int)vt->length(), "index must be less than vector length");
1611 assert(Type::cmp(vt, vsrc->bottom_type()) == 0, "input and output must be same type");
1612 }
1613 virtual int Opcode() const;
1614 uint pos() const { return in(3)->get_int(); }
1615
1616 static Node* make(Node* vec, Node* new_val, int position);
1617 };
1618
1619 class VectorBoxNode : public Node {
1620 private:
1621 const TypeInstPtr* const _box_type;
1622 const TypeVect* const _vec_type;
1623 public:
1624 enum {
1625 Box = 1,
1626 Value = 2
1627 };
1628 VectorBoxNode(Compile* C, Node* box, Node* val,
1629 const TypeInstPtr* box_type, const TypeVect* vt)
1630 : Node(NULL, box, val), _box_type(box_type), _vec_type(vt) {
1631 init_flags(Flag_is_macro);
1632 C->add_macro_node(this);
1633 }
1634
1635 const TypeInstPtr* box_type() const { assert(_box_type != NULL, ""); return _box_type; };
1636 const TypeVect* vec_type() const { assert(_vec_type != NULL, ""); return _vec_type; };
1637
1638 virtual int Opcode() const;
1639 virtual const Type* bottom_type() const { return _box_type; }
1640 virtual uint ideal_reg() const { return box_type()->ideal_reg(); }
1641 virtual uint size_of() const { return sizeof(*this); }
1642
1643 static const TypeFunc* vec_box_type(const TypeInstPtr* box_type);
1644 };
1645
1646 class VectorBoxAllocateNode : public CallStaticJavaNode {
1647 public:
1648 VectorBoxAllocateNode(Compile* C, const TypeInstPtr* vbox_type)
1649 : CallStaticJavaNode(C, VectorBoxNode::vec_box_type(vbox_type), NULL, NULL) {
1650 init_flags(Flag_is_macro);
1651 C->add_macro_node(this);
1652 }
1653
1654 virtual int Opcode() const;
1655 #ifndef PRODUCT
1656 virtual void dump_spec(outputStream *st) const;
1657 #endif // !PRODUCT
1658 };
1659
1660 class VectorUnboxNode : public VectorNode {
1661 private:
1662 bool _shuffle_to_vector;
1663 protected:
1664 uint size_of() const { return sizeof(*this); }
1665 public:
1666 VectorUnboxNode(Compile* C, const TypeVect* vec_type, Node* obj, Node* mem, bool shuffle_to_vector)
1667 : VectorNode(mem, obj, vec_type) {
1668 _shuffle_to_vector = shuffle_to_vector;
1669 init_class_id(Class_VectorUnbox);
1670 init_flags(Flag_is_macro);
1671 C->add_macro_node(this);
1672 }
1673
1674 virtual int Opcode() const;
1675 Node* obj() const { return in(2); }
1676 Node* mem() const { return in(1); }
1677 virtual Node* Identity(PhaseGVN* phase);
1678 Node* Ideal(PhaseGVN* phase, bool can_reshape);
1679 bool is_shuffle_to_vector() { return _shuffle_to_vector; }
1680 };
1681
1682 class RotateRightVNode : public VectorNode {
1683 public:
1684 RotateRightVNode(Node* in1, Node* in2, const TypeVect* vt)
1685 : VectorNode(in1, in2, vt) {}
1686
1687 virtual int Opcode() const;
1688 Node* Ideal(PhaseGVN* phase, bool can_reshape);
1689 };
1690
1691 class RotateLeftVNode : public VectorNode {
1692 public:
1693 RotateLeftVNode(Node* in1, Node* in2, const TypeVect* vt)
1694 : VectorNode(in1, in2, vt) {}
1695
1696 virtual int Opcode() const;
1697 Node* Ideal(PhaseGVN* phase, bool can_reshape);
1698 };
1699
1700 class CountLeadingZerosVNode : public VectorNode {
1701 public:
1702 CountLeadingZerosVNode(Node* in, const TypeVect* vt)
1703 : VectorNode(in, vt) {}
1704
1705 virtual int Opcode() const;
1706 };
1707
1708 class CountTrailingZerosVNode : public VectorNode {
1709 public:
1710 CountTrailingZerosVNode(Node* in, const TypeVect* vt)
1711 : VectorNode(in, vt) {}
1712
1713 virtual int Opcode() const;
1714 };
1715
1716 class ReverseVNode : public VectorNode {
1717 public:
1718 ReverseVNode(Node* in, const TypeVect* vt)
1719 : VectorNode(in, vt) {}
1720
1721 virtual int Opcode() const;
1722 };
1723
1724 class ReverseBytesVNode : public VectorNode {
1725 public:
1726 ReverseBytesVNode(Node* in, const TypeVect* vt)
1727 : VectorNode(in, vt) {}
1728
1729 virtual int Opcode() const;
1730 };
1731 #endif // SHARE_OPTO_VECTORNODE_HPP