1 /*
   2  * Copyright (c) 2007, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 #ifndef SHARE_OPTO_VECTORNODE_HPP
  25 #define SHARE_OPTO_VECTORNODE_HPP
  26 
  27 #include "opto/callnode.hpp"
  28 #include "opto/matcher.hpp"
  29 #include "opto/memnode.hpp"
  30 #include "opto/node.hpp"
  31 #include "opto/opcodes.hpp"
  32 #include "prims/vectorSupport.hpp"
  33 
  34 //------------------------------VectorNode-------------------------------------
  35 // Vector Operation
  36 class VectorNode : public TypeNode {
  37  public:
  38 
  39   VectorNode(Node* n1, const TypeVect* vt) : TypeNode(vt, 2) {
  40     init_class_id(Class_Vector);
  41     init_req(1, n1);
  42   }
  43   VectorNode(Node* n1, Node* n2, const TypeVect* vt) : TypeNode(vt, 3) {
  44     init_class_id(Class_Vector);
  45     init_req(1, n1);
  46     init_req(2, n2);
  47   }
  48 
  49   VectorNode(Node* n1, Node* n2, Node* n3, const TypeVect* vt) : TypeNode(vt, 4) {
  50     init_class_id(Class_Vector);
  51     init_req(1, n1);
  52     init_req(2, n2);
  53     init_req(3, n3);
  54   }
  55 
  56   VectorNode(Node *n0, Node* n1, Node* n2, Node* n3, const TypeVect* vt) : TypeNode(vt, 5) {
  57     init_class_id(Class_Vector);
  58     init_req(1, n0);
  59     init_req(2, n1);
  60     init_req(3, n2);
  61     init_req(4, n3);
  62   }
  63 
  64   const TypeVect* vect_type() const { return type()->is_vect(); }
  65   uint length() const { return vect_type()->length(); } // Vector length
  66   uint length_in_bytes() const { return vect_type()->length_in_bytes(); }
  67 
  68   virtual int Opcode() const;
  69 
  70   virtual uint ideal_reg() const {
  71     return type()->ideal_reg();
  72   }
  73 
  74   static VectorNode* scalar2vector(Node* s, uint vlen, const Type* opd_t, bool is_mask = false);
  75   static VectorNode* shift_count(int opc, Node* cnt, uint vlen, BasicType bt);
  76   static VectorNode* make(int opc, Node* n1, Node* n2, uint vlen, BasicType bt, bool is_var_shift = false);
  77   static VectorNode* make(int vopc, Node* n1, Node* n2, const TypeVect* vt, bool is_mask = false, bool is_var_shift = false);
  78   static VectorNode* make(int opc, Node* n1, Node* n2, Node* n3, uint vlen, BasicType bt);
  79   static VectorNode* make(int vopc, Node* n1, Node* n2, Node* n3, const TypeVect* vt);
  80   static VectorNode* make_mask_node(int vopc, Node* n1, Node* n2, uint vlen, BasicType bt);
  81 
  82   static bool is_shift_opcode(int opc);
  83 
  84   static bool is_vshift_cnt_opcode(int opc);
  85 
  86   static bool is_rotate_opcode(int opc);
  87 
  88   static int  opcode(int opc, BasicType bt);
  89   static int replicate_opcode(BasicType bt);
  90   static bool implemented(int opc, uint vlen, BasicType bt);
  91   static bool is_shift(Node* n);
  92   static bool is_vshift_cnt(Node* n);
  93   static bool is_type_transition_short_to_int(Node* n);
  94   static bool is_type_transition_to_int(Node* n);
  95   static bool is_muladds2i(Node* n);
  96   static bool is_roundopD(Node* n);
  97   static bool is_scalar_rotate(Node* n);
  98   static bool is_vector_rotate_supported(int opc, uint vlen, BasicType bt);
  99   static bool is_invariant_vector(Node* n);
 100   static bool is_all_ones_vector(Node* n);
 101   static bool is_vector_bitwise_not_pattern(Node* n);
 102   static Node* degenerate_vector_rotate(Node* n1, Node* n2, bool is_rotate_left, int vlen,
 103                                         BasicType bt, PhaseGVN* phase);
 104 
 105   // [Start, end) half-open range defining which operands are vectors
 106   static void vector_operands(Node* n, uint* start, uint* end);
 107 
 108   static bool is_vector_shift(int opc);
 109   static bool is_vector_shift_count(int opc);
 110   static bool is_vector_rotate(int opc);
 111 
 112   static bool is_vector_shift(Node* n) {
 113     return is_vector_shift(n->Opcode());
 114   }
 115   static bool is_vector_shift_count(Node* n) {
 116     return is_vector_shift_count(n->Opcode());
 117   }
 118 };
 119 
 120 //===========================Vector=ALU=Operations=============================
 121 
 122 //------------------------------AddVBNode--------------------------------------
 123 // Vector add byte
 124 class AddVBNode : public VectorNode {
 125  public:
 126   AddVBNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
 127   virtual int Opcode() const;
 128 };
 129 
 130 //------------------------------AddVSNode--------------------------------------
 131 // Vector add char/short
 132 class AddVSNode : public VectorNode {
 133  public:
 134   AddVSNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
 135   virtual int Opcode() const;
 136 };
 137 
 138 //------------------------------AddVINode--------------------------------------
 139 // Vector add int
 140 class AddVINode : public VectorNode {
 141  public:
 142   AddVINode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
 143   virtual int Opcode() const;
 144 };
 145 
 146 //------------------------------AddVLNode--------------------------------------
 147 // Vector add long
 148 class AddVLNode : public VectorNode {
 149 public:
 150   AddVLNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
 151   virtual int Opcode() const;
 152 };
 153 
 154 //------------------------------AddVFNode--------------------------------------
 155 // Vector add float
 156 class AddVFNode : public VectorNode {
 157 public:
 158   AddVFNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
 159   virtual int Opcode() const;
 160 };
 161 
 162 //------------------------------AddVDNode--------------------------------------
 163 // Vector add double
 164 class AddVDNode : public VectorNode {
 165 public:
 166   AddVDNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
 167   virtual int Opcode() const;
 168 };
 169 
 170 //------------------------------ReductionNode------------------------------------
 171 // Perform reduction of a vector
 172 class ReductionNode : public Node {
 173  private:
 174   const Type* _bottom_type;
 175  public:
 176   ReductionNode(Node *ctrl, Node* in1, Node* in2) : Node(ctrl, in1, in2),
 177                _bottom_type(Type::get_const_basic_type(in1->bottom_type()->basic_type())) {}
 178 
 179   static ReductionNode* make(int opc, Node *ctrl, Node* in1, Node* in2, BasicType bt);
 180   static int  opcode(int opc, BasicType bt);
 181   static bool implemented(int opc, uint vlen, BasicType bt);
 182   static Node* make_reduction_input(PhaseGVN& gvn, int opc, BasicType bt);
 183 
 184   virtual const Type* bottom_type() const {
 185     return _bottom_type;
 186   }
 187 
 188   virtual uint ideal_reg() const {
 189     return bottom_type()->ideal_reg();
 190   }
 191 
 192   // Needed for proper cloning.
 193   virtual uint size_of() const { return sizeof(*this); }
 194 };
 195 
 196 //------------------------------AddReductionVINode--------------------------------------
 197 // Vector add byte, short and int as a reduction
 198 class AddReductionVINode : public ReductionNode {
 199 public:
 200   AddReductionVINode(Node * ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
 201   virtual int Opcode() const;
 202 };
 203 
 204 //------------------------------AddReductionVLNode--------------------------------------
 205 // Vector add long as a reduction
 206 class AddReductionVLNode : public ReductionNode {
 207 public:
 208   AddReductionVLNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
 209   virtual int Opcode() const;
 210 };
 211 
 212 //------------------------------AddReductionVFNode--------------------------------------
 213 // Vector add float as a reduction
 214 class AddReductionVFNode : public ReductionNode {
 215 public:
 216   AddReductionVFNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
 217   virtual int Opcode() const;
 218 };
 219 
 220 //------------------------------AddReductionVDNode--------------------------------------
 221 // Vector add double as a reduction
 222 class AddReductionVDNode : public ReductionNode {
 223 public:
 224   AddReductionVDNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
 225   virtual int Opcode() const;
 226 };
 227 
 228 //------------------------------SubVBNode--------------------------------------
 229 // Vector subtract byte
 230 class SubVBNode : public VectorNode {
 231  public:
 232   SubVBNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
 233   virtual int Opcode() const;
 234 };
 235 
 236 //------------------------------SubVSNode--------------------------------------
 237 // Vector subtract short
 238 class SubVSNode : public VectorNode {
 239  public:
 240   SubVSNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
 241   virtual int Opcode() const;
 242 };
 243 
 244 //------------------------------SubVINode--------------------------------------
 245 // Vector subtract int
 246 class SubVINode : public VectorNode {
 247  public:
 248   SubVINode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
 249   virtual int Opcode() const;
 250 };
 251 
 252 //------------------------------SubVLNode--------------------------------------
 253 // Vector subtract long
 254 class SubVLNode : public VectorNode {
 255  public:
 256   SubVLNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
 257   virtual int Opcode() const;
 258 };
 259 
 260 //------------------------------SubVFNode--------------------------------------
 261 // Vector subtract float
 262 class SubVFNode : public VectorNode {
 263  public:
 264   SubVFNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
 265   virtual int Opcode() const;
 266 };
 267 
 268 //------------------------------SubVDNode--------------------------------------
 269 // Vector subtract double
 270 class SubVDNode : public VectorNode {
 271  public:
 272   SubVDNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
 273   virtual int Opcode() const;
 274 };
 275 
 276 //------------------------------MulVBNode--------------------------------------
 277 // Vector multiply byte
 278 class MulVBNode : public VectorNode {
 279  public:
 280   MulVBNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
 281   virtual int Opcode() const;
 282 };
 283 
 284 //------------------------------MulVSNode--------------------------------------
 285 // Vector multiply short
 286 class MulVSNode : public VectorNode {
 287  public:
 288   MulVSNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
 289   virtual int Opcode() const;
 290 };
 291 
 292 //------------------------------MulVINode--------------------------------------
 293 // Vector multiply int
 294 class MulVINode : public VectorNode {
 295  public:
 296   MulVINode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
 297   virtual int Opcode() const;
 298 };
 299 
 300 //------------------------------MulVLNode--------------------------------------
 301 // Vector multiply long
 302 class MulVLNode : public VectorNode {
 303 public:
 304   MulVLNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
 305   virtual int Opcode() const;
 306 };
 307 
 308 //------------------------------MulVFNode--------------------------------------
 309 // Vector multiply float
 310 class MulVFNode : public VectorNode {
 311 public:
 312   MulVFNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
 313   virtual int Opcode() const;
 314 };
 315 
 316 //------------------------------MulVDNode--------------------------------------
 317 // Vector multiply double
 318 class MulVDNode : public VectorNode {
 319 public:
 320   MulVDNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
 321   virtual int Opcode() const;
 322 };
 323 
 324 //------------------------------MulAddVS2VINode--------------------------------
 325 // Vector multiply shorts to int and add adjacent ints.
 326 class MulAddVS2VINode : public VectorNode {
 327   public:
 328     MulAddVS2VINode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
 329     virtual int Opcode() const;
 330 };
 331 
 332 //------------------------------FmaVDNode--------------------------------------
 333 // Vector multiply double
 334 class FmaVDNode : public VectorNode {
 335 public:
 336   FmaVDNode(Node* in1, Node* in2, Node* in3, const TypeVect* vt) : VectorNode(in1, in2, in3, vt) {}
 337   virtual int Opcode() const;
 338 };
 339 
 340 //------------------------------FmaVFNode--------------------------------------
 341 // Vector multiply float
 342 class FmaVFNode : public VectorNode {
 343 public:
 344   FmaVFNode(Node* in1, Node* in2, Node* in3, const TypeVect* vt) : VectorNode(in1, in2, in3, vt) {}
 345   virtual int Opcode() const;
 346 };
 347 
 348 //------------------------------CMoveVFNode--------------------------------------
 349 // Vector float conditional move
 350 class CMoveVFNode : public VectorNode {
 351 public:
 352   CMoveVFNode(Node* in1, Node* in2, Node* in3, const TypeVect* vt) : VectorNode(in1, in2, in3, vt) {}
 353   virtual int Opcode() const;
 354 };
 355 
 356 //------------------------------CMoveVDNode--------------------------------------
 357 // Vector double conditional move
 358 class CMoveVDNode : public VectorNode {
 359 public:
 360   CMoveVDNode(Node* in1, Node* in2, Node* in3, const TypeVect* vt) : VectorNode(in1, in2, in3, vt) {}
 361   virtual int Opcode() const;
 362 };
 363 
 364 //------------------------------MulReductionVINode--------------------------------------
 365 // Vector multiply byte, short and int as a reduction
 366 class MulReductionVINode : public ReductionNode {
 367 public:
 368   MulReductionVINode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
 369   virtual int Opcode() const;
 370 };
 371 
 372 //------------------------------MulReductionVLNode--------------------------------------
 373 // Vector multiply int as a reduction
 374 class MulReductionVLNode : public ReductionNode {
 375 public:
 376   MulReductionVLNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
 377   virtual int Opcode() const;
 378 };
 379 
 380 //------------------------------MulReductionVFNode--------------------------------------
 381 // Vector multiply float as a reduction
 382 class MulReductionVFNode : public ReductionNode {
 383 public:
 384   MulReductionVFNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
 385   virtual int Opcode() const;
 386 };
 387 
 388 //------------------------------MulReductionVDNode--------------------------------------
 389 // Vector multiply double as a reduction
 390 class MulReductionVDNode : public ReductionNode {
 391 public:
 392   MulReductionVDNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
 393   virtual int Opcode() const;
 394 };
 395 
 396 //------------------------------DivVFNode--------------------------------------
 397 // Vector divide float
 398 class DivVFNode : public VectorNode {
 399  public:
 400   DivVFNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
 401   virtual int Opcode() const;
 402 };
 403 
 404 //------------------------------DivVDNode--------------------------------------
 405 // Vector Divide double
 406 class DivVDNode : public VectorNode {
 407  public:
 408   DivVDNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
 409   virtual int Opcode() const;
 410 };
 411 
 412 //------------------------------AbsVBNode--------------------------------------
 413 // Vector Abs byte
 414 class AbsVBNode : public VectorNode {
 415 public:
 416   AbsVBNode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {}
 417   virtual int Opcode() const;
 418 };
 419 
 420 //------------------------------AbsVSNode--------------------------------------
 421 // Vector Abs short
 422 class AbsVSNode : public VectorNode {
 423 public:
 424   AbsVSNode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {}
 425   virtual int Opcode() const;
 426 };
 427 
 428 //------------------------------MinVNode--------------------------------------
 429 // Vector Min
 430 class MinVNode : public VectorNode {
 431 public:
 432   MinVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
 433   virtual int Opcode() const;
 434 };
 435 
 436 //------------------------------MaxVNode--------------------------------------
 437 // Vector Max
 438 class MaxVNode : public VectorNode {
 439  public:
 440   MaxVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
 441   virtual int Opcode() const;
 442 };
 443 
 444 //------------------------------AbsVINode--------------------------------------
 445 // Vector Abs int
 446 class AbsVINode : public VectorNode {
 447  public:
 448   AbsVINode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {}
 449   virtual int Opcode() const;
 450 };
 451 
 452 //------------------------------AbsVLNode--------------------------------------
 453 // Vector Abs long
 454 class AbsVLNode : public VectorNode {
 455 public:
 456   AbsVLNode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {}
 457   virtual int Opcode() const;
 458 };
 459 
 460 //------------------------------AbsVFNode--------------------------------------
 461 // Vector Abs float
 462 class AbsVFNode : public VectorNode {
 463  public:
 464   AbsVFNode(Node* in, const TypeVect* vt) : VectorNode(in,vt) {}
 465   virtual int Opcode() const;
 466 };
 467 
 468 //------------------------------AbsVDNode--------------------------------------
 469 // Vector Abs double
 470 class AbsVDNode : public VectorNode {
 471  public:
 472   AbsVDNode(Node* in, const TypeVect* vt) : VectorNode(in,vt) {}
 473   virtual int Opcode() const;
 474 };
 475 
 476 //------------------------------NegVINode--------------------------------------
 477 // Vector Neg int
 478 class NegVINode : public VectorNode {
 479  public:
 480   NegVINode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {}
 481   virtual int Opcode() const;
 482 };
 483 
 484 //------------------------------NegVFNode--------------------------------------
 485 // Vector Neg float
 486 class NegVFNode : public VectorNode {
 487  public:
 488   NegVFNode(Node* in, const TypeVect* vt) : VectorNode(in,vt) {}
 489   virtual int Opcode() const;
 490 };
 491 
 492 //------------------------------NegVDNode--------------------------------------
 493 // Vector Neg double
 494 class NegVDNode : public VectorNode {
 495  public:
 496   NegVDNode(Node* in, const TypeVect* vt) : VectorNode(in,vt) {}
 497   virtual int Opcode() const;
 498 };
 499 
 500 //------------------------------PopCountVINode---------------------------------
 501 // Vector popcount integer bits
 502 class PopCountVINode : public VectorNode {
 503  public:
 504   PopCountVINode(Node* in, const TypeVect* vt) : VectorNode(in,vt) {}
 505   virtual int Opcode() const;
 506 };
 507 
 508 //------------------------------SqrtVFNode--------------------------------------
 509 // Vector Sqrt float
 510 class SqrtVFNode : public VectorNode {
 511  public:
 512   SqrtVFNode(Node* in, const TypeVect* vt) : VectorNode(in,vt) {}
 513   virtual int Opcode() const;
 514 };
 515 //------------------------------RoundDoubleVNode--------------------------------
 516 // Vector round double
 517 class RoundDoubleModeVNode : public VectorNode {
 518  public:
 519   RoundDoubleModeVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
 520   virtual int Opcode() const;
 521 };
 522 
 523 //------------------------------SqrtVDNode--------------------------------------
 524 // Vector Sqrt double
 525 class SqrtVDNode : public VectorNode {
 526  public:
 527   SqrtVDNode(Node* in, const TypeVect* vt) : VectorNode(in,vt) {}
 528   virtual int Opcode() const;
 529 };
 530 
 531 //------------------------------ShiftVNode-----------------------------------
 532 // Class ShiftV functionality.  This covers the common behaviors for all kinds
 533 // of vector shifts.
 534 class ShiftVNode : public VectorNode {
 535  bool _is_var_shift;
 536  public:
 537   ShiftVNode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift) :
 538     VectorNode(in1,in2,vt), _is_var_shift(is_var_shift) {
 539     init_class_id(Class_ShiftV);
 540   }
 541   virtual Node* Identity(PhaseGVN* phase);
 542   virtual int Opcode() const = 0;
 543   bool is_var_shift() { return _is_var_shift;}
 544   virtual  uint  size_of() const { return sizeof(ShiftVNode); }
 545 };
 546 
 547 //------------------------------LShiftVBNode-----------------------------------
 548 // Vector left shift bytes
 549 class LShiftVBNode : public ShiftVNode {
 550  public:
 551   LShiftVBNode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
 552     ShiftVNode(in1,in2,vt,is_var_shift) {}
 553   virtual int Opcode() const;
 554 };
 555 
 556 //------------------------------LShiftVSNode-----------------------------------
 557 // Vector left shift shorts
 558 class LShiftVSNode : public ShiftVNode {
 559  public:
 560   LShiftVSNode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
 561     ShiftVNode(in1,in2,vt,is_var_shift) {}
 562   virtual int Opcode() const;
 563 };
 564 
 565 //------------------------------LShiftVINode-----------------------------------
 566 // Vector left shift ints
 567 class LShiftVINode : public ShiftVNode {
 568  public:
 569   LShiftVINode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
 570     ShiftVNode(in1,in2,vt,is_var_shift) {}
 571   virtual int Opcode() const;
 572 };
 573 
 574 //------------------------------LShiftVLNode-----------------------------------
 575 // Vector left shift longs
 576 class LShiftVLNode : public ShiftVNode {
 577  public:
 578   LShiftVLNode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
 579     ShiftVNode(in1,in2,vt,is_var_shift) {}
 580   virtual int Opcode() const;
 581 };
 582 
 583 //------------------------------RShiftVBNode-----------------------------------
 584 // Vector right arithmetic (signed) shift bytes
 585 class RShiftVBNode : public ShiftVNode {
 586  public:
 587   RShiftVBNode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
 588     ShiftVNode(in1,in2,vt,is_var_shift) {}
 589   virtual int Opcode() const;
 590 };
 591 
 592 //------------------------------RShiftVSNode-----------------------------------
 593 // Vector right arithmetic (signed) shift shorts
 594 class RShiftVSNode : public ShiftVNode {
 595  public:
 596   RShiftVSNode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
 597     ShiftVNode(in1,in2,vt,is_var_shift) {}
 598   virtual int Opcode() const;
 599 };
 600 
 601 //------------------------------RShiftVINode-----------------------------------
 602 // Vector right arithmetic (signed) shift ints
 603 class RShiftVINode : public ShiftVNode {
 604  public:
 605   RShiftVINode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
 606     ShiftVNode(in1,in2,vt,is_var_shift) {}
 607   virtual int Opcode() const;
 608 };
 609 
 610 //------------------------------RShiftVLNode-----------------------------------
 611 // Vector right arithmetic (signed) shift longs
 612 class RShiftVLNode : public ShiftVNode {
 613  public:
 614   RShiftVLNode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
 615     ShiftVNode(in1,in2,vt,is_var_shift) {}
 616   virtual int Opcode() const;
 617 };
 618 
 619 //------------------------------URShiftVBNode----------------------------------
 620 // Vector right logical (unsigned) shift bytes
 621 class URShiftVBNode : public ShiftVNode {
 622  public:
 623   URShiftVBNode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
 624     ShiftVNode(in1,in2,vt,is_var_shift) {}
 625   virtual int Opcode() const;
 626 };
 627 
 628 //------------------------------URShiftVSNode----------------------------------
 629 // Vector right logical (unsigned) shift shorts
 630 class URShiftVSNode : public ShiftVNode {
 631  public:
 632   URShiftVSNode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
 633     ShiftVNode(in1,in2,vt,is_var_shift) {}
 634   virtual int Opcode() const;
 635 };
 636 
 637 //------------------------------URShiftVINode----------------------------------
 638 // Vector right logical (unsigned) shift ints
 639 class URShiftVINode : public ShiftVNode {
 640  public:
 641   URShiftVINode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
 642     ShiftVNode(in1,in2,vt,is_var_shift) {}
 643   virtual int Opcode() const;
 644 };
 645 
 646 //------------------------------URShiftVLNode----------------------------------
 647 // Vector right logical (unsigned) shift longs
 648 class URShiftVLNode : public ShiftVNode {
 649  public:
 650   URShiftVLNode(Node* in1, Node* in2, const TypeVect* vt, bool is_var_shift=false) :
 651      ShiftVNode(in1,in2,vt,is_var_shift) {}
 652   virtual int Opcode() const;
 653 };
 654 
 655 //------------------------------LShiftCntVNode---------------------------------
 656 // Vector left shift count
 657 class LShiftCntVNode : public VectorNode {
 658  public:
 659   LShiftCntVNode(Node* cnt, const TypeVect* vt) : VectorNode(cnt,vt) {}
 660   virtual int Opcode() const;
 661 };
 662 
 663 //------------------------------RShiftCntVNode---------------------------------
 664 // Vector right shift count
 665 class RShiftCntVNode : public VectorNode {
 666  public:
 667   RShiftCntVNode(Node* cnt, const TypeVect* vt) : VectorNode(cnt,vt) {}
 668   virtual int Opcode() const;
 669 };
 670 
 671 //------------------------------AndVNode---------------------------------------
 672 // Vector and integer
 673 class AndVNode : public VectorNode {
 674  public:
 675   AndVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
 676   virtual int Opcode() const;
 677 };
 678 
 679 //------------------------------AndReductionVNode--------------------------------------
 680 // Vector and byte, short, int, long as a reduction
 681 class AndReductionVNode : public ReductionNode {
 682  public:
 683   AndReductionVNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
 684   virtual int Opcode() const;
 685 };
 686 
 687 //------------------------------OrVNode---------------------------------------
 688 // Vector or byte, short, int, long as a reduction
 689 class OrVNode : public VectorNode {
 690  public:
 691   OrVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
 692   virtual int Opcode() const;
 693 };
 694 
 695 //------------------------------OrReductionVNode--------------------------------------
 696 // Vector xor byte, short, int, long as a reduction
 697 class OrReductionVNode : public ReductionNode {
 698  public:
 699   OrReductionVNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
 700   virtual int Opcode() const;
 701 };
 702 
 703 //------------------------------XorReductionVNode--------------------------------------
 704 // Vector and int, long as a reduction
 705 class XorReductionVNode : public ReductionNode {
 706  public:
 707   XorReductionVNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
 708   virtual int Opcode() const;
 709 };
 710 
 711 //------------------------------XorVNode---------------------------------------
 712 // Vector xor integer
 713 class XorVNode : public VectorNode {
 714  public:
 715   XorVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1,in2,vt) {}
 716   virtual int Opcode() const;
 717 };
 718 
 719 //------------------------------MinReductionVNode--------------------------------------
 720 // Vector min byte, short, int, long, float, double as a reduction
 721 class MinReductionVNode : public ReductionNode {
 722 public:
 723   MinReductionVNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
 724   virtual int Opcode() const;
 725 };
 726 
 727 //------------------------------MaxReductionVNode--------------------------------------
 728 // Vector min byte, short, int, long, float, double as a reduction
 729 class MaxReductionVNode : public ReductionNode {
 730 public:
 731   MaxReductionVNode(Node *ctrl, Node* in1, Node* in2) : ReductionNode(ctrl, in1, in2) {}
 732   virtual int Opcode() const;
 733 };
 734 
 735 //================================= M E M O R Y ===============================
 736 
 737 //------------------------------LoadVectorNode---------------------------------
 738 // Load Vector from memory
 739 class LoadVectorNode : public LoadNode {
 740  public:
 741   LoadVectorNode(Node* c, Node* mem, Node* adr, const TypePtr* at, const TypeVect* vt, ControlDependency control_dependency = LoadNode::DependsOnlyOnTest)
 742     : LoadNode(c, mem, adr, at, vt, MemNode::unordered, control_dependency) {
 743     init_class_id(Class_LoadVector);
 744     set_mismatched_access();
 745   }
 746 
 747   const TypeVect* vect_type() const { return type()->is_vect(); }
 748   uint length() const { return vect_type()->length(); } // Vector length
 749 
 750   virtual int Opcode() const;
 751 
 752   virtual uint ideal_reg() const  { return Matcher::vector_ideal_reg(memory_size()); }
 753   virtual BasicType memory_type() const { return T_VOID; }
 754   virtual int memory_size() const { return vect_type()->length_in_bytes(); }
 755 
 756   virtual int store_Opcode() const { return Op_StoreVector; }
 757 
 758   static LoadVectorNode* make(int opc, Node* ctl, Node* mem,
 759                               Node* adr, const TypePtr* atyp,
 760                               uint vlen, BasicType bt,
 761                               ControlDependency control_dependency = LoadNode::DependsOnlyOnTest);
 762   uint element_size(void) { return type2aelembytes(vect_type()->element_basic_type()); }
 763 };
 764 
 765 //------------------------------LoadVectorGatherNode------------------------------
 766 // Load Vector from memory via index map
 767 class LoadVectorGatherNode : public LoadVectorNode {
 768  public:
 769   LoadVectorGatherNode(Node* c, Node* mem, Node* adr, const TypePtr* at, const TypeVect* vt, Node* indices)
 770     : LoadVectorNode(c, mem, adr, at, vt) {
 771     init_class_id(Class_LoadVectorGather);
 772     assert(indices->bottom_type()->is_vect(), "indices must be in vector");
 773     add_req(indices);
 774     assert(req() == MemNode::ValueIn + 1, "match_edge expects that last input is in MemNode::ValueIn");
 775   }
 776 
 777   virtual int Opcode() const;
 778   virtual uint match_edge(uint idx) const { return idx == MemNode::Address || idx == MemNode::ValueIn; }
 779 };
 780 
 781 //------------------------------StoreVectorNode--------------------------------
 782 // Store Vector to memory
 783 class StoreVectorNode : public StoreNode {
 784  private:
 785   const TypeVect* _vect_type;
 786  public:
 787   StoreVectorNode(Node* c, Node* mem, Node* adr, const TypePtr* at, Node* val)
 788     : StoreNode(c, mem, adr, at, val, MemNode::unordered), _vect_type(val->bottom_type()->is_vect()) {
 789     init_class_id(Class_StoreVector);
 790     set_mismatched_access();
 791   }
 792 
 793   const TypeVect* vect_type() const { return _vect_type; }
 794   uint length() const { return vect_type()->length(); } // Vector length
 795 
 796   virtual int Opcode() const;
 797 
 798   virtual uint ideal_reg() const  { return Matcher::vector_ideal_reg(memory_size()); }
 799   virtual BasicType memory_type() const { return T_VOID; }
 800   virtual int memory_size() const { return vect_type()->length_in_bytes(); }
 801 
 802   static StoreVectorNode* make(int opc, Node* ctl, Node* mem,
 803                                Node* adr, const TypePtr* atyp, Node* val,
 804                                uint vlen);
 805 
 806   uint element_size(void) { return type2aelembytes(vect_type()->element_basic_type()); }
 807 
 808   // Needed for proper cloning.
 809   virtual uint size_of() const { return sizeof(*this); }
 810 };
 811 
 812 //------------------------------StoreVectorScatterNode------------------------------
 813 // Store Vector into memory via index map
 814 
 815  class StoreVectorScatterNode : public StoreVectorNode {
 816   public:
 817    StoreVectorScatterNode(Node* c, Node* mem, Node* adr, const TypePtr* at, Node* val, Node* indices)
 818      : StoreVectorNode(c, mem, adr, at, val) {
 819      init_class_id(Class_StoreVectorScatter);
 820      assert(indices->bottom_type()->is_vect(), "indices must be in vector");
 821      add_req(indices);
 822      assert(req() == MemNode::ValueIn + 2, "match_edge expects that last input is in MemNode::ValueIn+1");
 823    }
 824    virtual int Opcode() const;
 825    virtual uint match_edge(uint idx) const { return idx == MemNode::Address ||
 826                                                     idx == MemNode::ValueIn ||
 827                                                     idx == MemNode::ValueIn + 1; }
 828 };
 829 
 830 //------------------------------StoreVectorMaskedNode--------------------------------
 831 // Store Vector to memory under the influence of a predicate register(mask).
 832 class StoreVectorMaskedNode : public StoreVectorNode {
 833  public:
 834   StoreVectorMaskedNode(Node* c, Node* mem, Node* dst, Node* src, const TypePtr* at, Node* mask)
 835    : StoreVectorNode(c, mem, dst, at, src) {
 836     assert(mask->bottom_type()->isa_vectmask(), "sanity");
 837     init_class_id(Class_StoreVector);
 838     set_mismatched_access();
 839     add_req(mask);
 840   }
 841 
 842   virtual int Opcode() const;
 843 
 844   virtual uint match_edge(uint idx) const {
 845     return idx > 1;
 846   }
 847   Node* Ideal(PhaseGVN* phase, bool can_reshape);
 848 };
 849 
 850 //------------------------------LoadVectorMaskedNode--------------------------------
 851 // Load Vector from memory under the influence of a predicate register(mask).
 852 class LoadVectorMaskedNode : public LoadVectorNode {
 853  public:
 854   LoadVectorMaskedNode(Node* c, Node* mem, Node* src, const TypePtr* at, const TypeVect* vt, Node* mask)
 855    : LoadVectorNode(c, mem, src, at, vt) {
 856     assert(mask->bottom_type()->isa_vectmask(), "sanity");
 857     init_class_id(Class_LoadVector);
 858     set_mismatched_access();
 859     add_req(mask);
 860   }
 861 
 862   virtual int Opcode() const;
 863 
 864   virtual uint match_edge(uint idx) const {
 865     return idx > 1;
 866   }
 867   Node* Ideal(PhaseGVN* phase, bool can_reshape);
 868 };
 869 
 870 //-------------------------------LoadVectorGatherMaskedNode---------------------------------
 871 // Load Vector from memory via index map under the influence of a predicate register(mask).
 872 class LoadVectorGatherMaskedNode : public LoadVectorNode {
 873  public:
 874   LoadVectorGatherMaskedNode(Node* c, Node* mem, Node* adr, const TypePtr* at, const TypeVect* vt, Node* indices, Node* mask)
 875     : LoadVectorNode(c, mem, adr, at, vt) {
 876     init_class_id(Class_LoadVector);
 877     assert(indices->bottom_type()->is_vect(), "indices must be in vector");
 878     assert(mask->bottom_type()->isa_vectmask(), "sanity");
 879     add_req(indices);
 880     add_req(mask);
 881     assert(req() == MemNode::ValueIn + 2, "match_edge expects that last input is in MemNode::ValueIn+1");
 882   }
 883 
 884   virtual int Opcode() const;
 885   virtual uint match_edge(uint idx) const { return idx == MemNode::Address ||
 886                                                    idx == MemNode::ValueIn ||
 887                                                    idx == MemNode::ValueIn + 1; }
 888 };
 889 
 890 //------------------------------StoreVectorScatterMaskedNode--------------------------------
 891 // Store Vector into memory via index map under the influence of a predicate register(mask).
 892 class StoreVectorScatterMaskedNode : public StoreVectorNode {
 893   public:
 894    StoreVectorScatterMaskedNode(Node* c, Node* mem, Node* adr, const TypePtr* at, Node* val, Node* indices, Node* mask)
 895      : StoreVectorNode(c, mem, adr, at, val) {
 896      init_class_id(Class_StoreVector);
 897      assert(indices->bottom_type()->is_vect(), "indices must be in vector");
 898      assert(mask->bottom_type()->isa_vectmask(), "sanity");
 899      add_req(indices);
 900      add_req(mask);
 901      assert(req() == MemNode::ValueIn + 3, "match_edge expects that last input is in MemNode::ValueIn+2");
 902    }
 903    virtual int Opcode() const;
 904    virtual uint match_edge(uint idx) const { return idx == MemNode::Address ||
 905                                                     idx == MemNode::ValueIn ||
 906                                                     idx == MemNode::ValueIn + 1 ||
 907                                                     idx == MemNode::ValueIn + 2; }
 908 };
 909 
 910 //------------------------------VectorCmpMaskedNode--------------------------------
 911 // Vector Comparison under the influence of a predicate register(mask).
 912 class VectorCmpMaskedNode : public TypeNode {
 913   public:
 914    VectorCmpMaskedNode(Node* src1, Node* src2, Node* mask, const Type* ty): TypeNode(ty, 4)  {
 915      init_req(1, src1);
 916      init_req(2, src2);
 917      init_req(3, mask);
 918    }
 919 
 920   virtual int Opcode() const;
 921 };
 922 
 923 //------------------------------VectorMaskGenNode----------------------------------
 924 class VectorMaskGenNode : public TypeNode {
 925  public:
 926   VectorMaskGenNode(Node* length, const Type* ty): TypeNode(ty, 2) {
 927     init_req(1, length);
 928   }
 929 
 930   virtual int Opcode() const;
 931   virtual uint ideal_reg() const { return Op_RegVectMask; }
 932   static Node* make(Node* length, BasicType vmask_bt);
 933 };
 934 
 935 //------------------------------VectorMaskOpNode-----------------------------------
 936 class VectorMaskOpNode : public TypeNode {
 937  public:
 938   VectorMaskOpNode(Node* mask, const Type* ty, int mopc):
 939     TypeNode(ty, 2), _mopc(mopc) {
 940     assert(Matcher::has_predicated_vectors() || mask->bottom_type()->is_vect()->element_basic_type() == T_BOOLEAN, "");
 941     init_req(1, mask);
 942   }
 943 
 944   virtual int Opcode() const;
 945   virtual  uint  size_of() const { return sizeof(VectorMaskOpNode); }
 946   virtual uint  ideal_reg() const { return Op_RegI; }
 947   int get_mask_Opcode() const { return _mopc;}
 948   static Node* make(Node* mask, const Type* ty, int mopc);
 949 
 950   private:
 951     int _mopc;
 952 };
 953 
 954 class VectorMaskTrueCountNode : public VectorMaskOpNode {
 955  public:
 956   VectorMaskTrueCountNode(Node* mask, const Type* ty):
 957     VectorMaskOpNode(mask, ty, Op_VectorMaskTrueCount) {}
 958   virtual int Opcode() const;
 959 };
 960 
 961 class VectorMaskFirstTrueNode : public VectorMaskOpNode {
 962  public:
 963   VectorMaskFirstTrueNode(Node* mask, const Type* ty):
 964     VectorMaskOpNode(mask, ty, Op_VectorMaskFirstTrue) {}
 965   virtual int Opcode() const;
 966 };
 967 
 968 class VectorMaskLastTrueNode : public VectorMaskOpNode {
 969  public:
 970   VectorMaskLastTrueNode(Node* mask, const Type* ty):
 971     VectorMaskOpNode(mask, ty, Op_VectorMaskLastTrue) {}
 972   virtual int Opcode() const;
 973 };
 974 
 975 class VectorMaskToLongNode : public VectorMaskOpNode {
 976  public:
 977   VectorMaskToLongNode(Node* mask, const Type* ty):
 978     VectorMaskOpNode(mask, ty, Op_VectorMaskToLong) {}
 979   virtual int Opcode() const;
 980   virtual uint  ideal_reg() const { return Op_RegL; }
 981   virtual Node* Identity(PhaseGVN* phase);
 982 };
 983 
 984 class VectorLongToMaskNode : public VectorNode {
 985  public:
 986   VectorLongToMaskNode(Node* mask, const TypeVect* ty):
 987     VectorNode(mask, ty) {
 988   }
 989   virtual int Opcode() const;
 990   Node* Ideal(PhaseGVN* phase, bool can_reshape);
 991 };
 992 
 993 //-------------------------- Vector mask broadcast -----------------------------------
 994 class MaskAllNode : public VectorNode {
 995  public:
 996   MaskAllNode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {}
 997   virtual int Opcode() const;
 998 };
 999 
1000 //--------------------------- Vector mask logical and --------------------------------
1001 class AndVMaskNode : public VectorNode {
1002  public:
1003   AndVMaskNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
1004   virtual int Opcode() const;
1005 };
1006 
1007 //--------------------------- Vector mask logical or ---------------------------------
1008 class OrVMaskNode : public VectorNode {
1009  public:
1010   OrVMaskNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
1011   virtual int Opcode() const;
1012 };
1013 
1014 //--------------------------- Vector mask logical xor --------------------------------
1015 class XorVMaskNode : public VectorNode {
1016  public:
1017   XorVMaskNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
1018   virtual int Opcode() const;
1019 };
1020 
1021 //=========================Promote_Scalar_to_Vector============================
1022 
1023 //------------------------------ReplicateBNode---------------------------------
1024 // Replicate byte scalar to be vector
1025 class ReplicateBNode : public VectorNode {
1026  public:
1027   ReplicateBNode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
1028   virtual int Opcode() const;
1029 };
1030 
1031 //------------------------------ReplicateSNode---------------------------------
1032 // Replicate short scalar to be vector
1033 class ReplicateSNode : public VectorNode {
1034  public:
1035   ReplicateSNode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
1036   virtual int Opcode() const;
1037 };
1038 
1039 //------------------------------ReplicateINode---------------------------------
1040 // Replicate int scalar to be vector
1041 class ReplicateINode : public VectorNode {
1042  public:
1043   ReplicateINode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
1044   virtual int Opcode() const;
1045 };
1046 
1047 //------------------------------ReplicateLNode---------------------------------
1048 // Replicate long scalar to be vector
1049 class ReplicateLNode : public VectorNode {
1050  public:
1051   ReplicateLNode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
1052   virtual int Opcode() const;
1053 };
1054 
1055 //------------------------------ReplicateFNode---------------------------------
1056 // Replicate float scalar to be vector
1057 class ReplicateFNode : public VectorNode {
1058  public:
1059   ReplicateFNode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
1060   virtual int Opcode() const;
1061 };
1062 
1063 //------------------------------ReplicateDNode---------------------------------
1064 // Replicate double scalar to be vector
1065 class ReplicateDNode : public VectorNode {
1066  public:
1067   ReplicateDNode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
1068   virtual int Opcode() const;
1069 };
1070 
1071 //========================Pack_Scalars_into_a_Vector===========================
1072 
1073 //------------------------------PackNode---------------------------------------
1074 // Pack parent class (not for code generation).
1075 class PackNode : public VectorNode {
1076  public:
1077   PackNode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
1078   PackNode(Node* in1, Node* n2, const TypeVect* vt) : VectorNode(in1, n2, vt) {}
1079   virtual int Opcode() const;
1080 
1081   void add_opd(Node* n) {
1082     add_req(n);
1083   }
1084 
1085   // Create a binary tree form for Packs. [lo, hi) (half-open) range
1086   PackNode* binary_tree_pack(int lo, int hi);
1087 
1088   static PackNode* make(Node* s, uint vlen, BasicType bt);
1089 };
1090 
1091 //------------------------------PackBNode--------------------------------------
1092 // Pack byte scalars into vector
1093 class PackBNode : public PackNode {
1094  public:
1095   PackBNode(Node* in1, const TypeVect* vt)  : PackNode(in1, vt) {}
1096   virtual int Opcode() const;
1097 };
1098 
1099 //------------------------------PackSNode--------------------------------------
1100 // Pack short scalars into a vector
1101 class PackSNode : public PackNode {
1102  public:
1103   PackSNode(Node* in1, const TypeVect* vt)  : PackNode(in1, vt) {}
1104   PackSNode(Node* in1, Node* in2, const TypeVect* vt) : PackNode(in1, in2, vt) {}
1105   virtual int Opcode() const;
1106 };
1107 
1108 //------------------------------PackINode--------------------------------------
1109 // Pack integer scalars into a vector
1110 class PackINode : public PackNode {
1111  public:
1112   PackINode(Node* in1, const TypeVect* vt)  : PackNode(in1, vt) {}
1113   PackINode(Node* in1, Node* in2, const TypeVect* vt) : PackNode(in1, in2, vt) {}
1114   virtual int Opcode() const;
1115 };
1116 
1117 //------------------------------PackLNode--------------------------------------
1118 // Pack long scalars into a vector
1119 class PackLNode : public PackNode {
1120  public:
1121   PackLNode(Node* in1, const TypeVect* vt)  : PackNode(in1, vt) {}
1122   PackLNode(Node* in1, Node* in2, const TypeVect* vt) : PackNode(in1, in2, vt) {}
1123   virtual int Opcode() const;
1124 };
1125 
1126 //------------------------------Pack2LNode-------------------------------------
1127 // Pack 2 long scalars into a vector
1128 class Pack2LNode : public PackNode {
1129  public:
1130   Pack2LNode(Node* in1, Node* in2, const TypeVect* vt) : PackNode(in1, in2, vt) {}
1131   virtual int Opcode() const;
1132 };
1133 
1134 //------------------------------PackFNode--------------------------------------
1135 // Pack float scalars into vector
1136 class PackFNode : public PackNode {
1137  public:
1138   PackFNode(Node* in1, const TypeVect* vt)  : PackNode(in1, vt) {}
1139   PackFNode(Node* in1, Node* in2, const TypeVect* vt) : PackNode(in1, in2, vt) {}
1140   virtual int Opcode() const;
1141 };
1142 
1143 //------------------------------PackDNode--------------------------------------
1144 // Pack double scalars into a vector
1145 class PackDNode : public PackNode {
1146  public:
1147   PackDNode(Node* in1, const TypeVect* vt) : PackNode(in1, vt) {}
1148   PackDNode(Node* in1, Node* in2, const TypeVect* vt) : PackNode(in1, in2, vt) {}
1149   virtual int Opcode() const;
1150 };
1151 
1152 //------------------------------Pack2DNode-------------------------------------
1153 // Pack 2 double scalars into a vector
1154 class Pack2DNode : public PackNode {
1155  public:
1156   Pack2DNode(Node* in1, Node* in2, const TypeVect* vt) : PackNode(in1, in2, vt) {}
1157   virtual int Opcode() const;
1158 };
1159 
1160 
1161 class VectorLoadConstNode : public VectorNode {
1162  public:
1163   VectorLoadConstNode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
1164   virtual int Opcode() const;
1165 };
1166 
1167 //========================Extract_Scalar_from_Vector===========================
1168 
1169 //------------------------------ExtractNode------------------------------------
1170 // Extract a scalar from a vector at position "pos"
1171 class ExtractNode : public Node {
1172  public:
1173   ExtractNode(Node* src, ConINode* pos) : Node(NULL, src, (Node*)pos) {
1174     assert(in(2)->get_int() >= 0, "positive constants");
1175   }
1176   virtual int Opcode() const;
1177   uint  pos() const { return in(2)->get_int(); }
1178 
1179   static Node* make(Node* v, uint position, BasicType bt);
1180   static int opcode(BasicType bt);
1181 };
1182 
1183 //------------------------------ExtractBNode-----------------------------------
1184 // Extract a byte from a vector at position "pos"
1185 class ExtractBNode : public ExtractNode {
1186  public:
1187   ExtractBNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1188   virtual int Opcode() const;
1189   virtual const Type *bottom_type() const { return TypeInt::INT; }
1190   virtual uint ideal_reg() const { return Op_RegI; }
1191 };
1192 
1193 //------------------------------ExtractUBNode----------------------------------
1194 // Extract a boolean from a vector at position "pos"
1195 class ExtractUBNode : public ExtractNode {
1196  public:
1197   ExtractUBNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1198   virtual int Opcode() const;
1199   virtual const Type *bottom_type() const { return TypeInt::INT; }
1200   virtual uint ideal_reg() const { return Op_RegI; }
1201 };
1202 
1203 //------------------------------ExtractCNode-----------------------------------
1204 // Extract a char from a vector at position "pos"
1205 class ExtractCNode : public ExtractNode {
1206  public:
1207   ExtractCNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1208   virtual int Opcode() const;
1209   virtual const Type *bottom_type() const { return TypeInt::CHAR; }
1210   virtual uint ideal_reg() const { return Op_RegI; }
1211 };
1212 
1213 //------------------------------ExtractSNode-----------------------------------
1214 // Extract a short from a vector at position "pos"
1215 class ExtractSNode : public ExtractNode {
1216  public:
1217   ExtractSNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1218   virtual int Opcode() const;
1219   virtual const Type *bottom_type() const { return TypeInt::SHORT; }
1220   virtual uint ideal_reg() const { return Op_RegI; }
1221 };
1222 
1223 //------------------------------ExtractINode-----------------------------------
1224 // Extract an int from a vector at position "pos"
1225 class ExtractINode : public ExtractNode {
1226  public:
1227   ExtractINode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1228   virtual int Opcode() const;
1229   virtual const Type *bottom_type() const { return TypeInt::INT; }
1230   virtual uint ideal_reg() const { return Op_RegI; }
1231 };
1232 
1233 //------------------------------ExtractLNode-----------------------------------
1234 // Extract a long from a vector at position "pos"
1235 class ExtractLNode : public ExtractNode {
1236  public:
1237   ExtractLNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1238   virtual int Opcode() const;
1239   virtual const Type *bottom_type() const { return TypeLong::LONG; }
1240   virtual uint ideal_reg() const { return Op_RegL; }
1241 };
1242 
1243 //------------------------------ExtractFNode-----------------------------------
1244 // Extract a float from a vector at position "pos"
1245 class ExtractFNode : public ExtractNode {
1246  public:
1247   ExtractFNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1248   virtual int Opcode() const;
1249   virtual const Type *bottom_type() const { return Type::FLOAT; }
1250   virtual uint ideal_reg() const { return Op_RegF; }
1251 };
1252 
1253 //------------------------------ExtractDNode-----------------------------------
1254 // Extract a double from a vector at position "pos"
1255 class ExtractDNode : public ExtractNode {
1256  public:
1257   ExtractDNode(Node* src, ConINode* pos) : ExtractNode(src, pos) {}
1258   virtual int Opcode() const;
1259   virtual const Type *bottom_type() const { return Type::DOUBLE; }
1260   virtual uint ideal_reg() const { return Op_RegD; }
1261 };
1262 
1263 //------------------------------SetVectMaskINode-------------------------------
1264 // Provide a mask for a vector predicate machine
1265 class SetVectMaskINode : public Node {
1266 public:
1267   SetVectMaskINode(Node *c, Node *in1) : Node(c, in1) {}
1268   virtual int Opcode() const;
1269   const Type *bottom_type() const { return TypeInt::INT; }
1270   virtual uint ideal_reg() const { return Op_RegI; }
1271   virtual const Type *Value(PhaseGVN *phase) const { return TypeInt::INT; }
1272 };
1273 
1274 //------------------------------MacroLogicVNode-------------------------------
1275 // Vector logical operations packing node.
1276 class MacroLogicVNode : public VectorNode {
1277 private:
1278   MacroLogicVNode(Node* in1, Node* in2, Node* in3, Node* fn, const TypeVect* vt)
1279   : VectorNode(in1, in2, in3, fn, vt) {}
1280 
1281 public:
1282   virtual int Opcode() const;
1283 
1284   static MacroLogicVNode* make(PhaseGVN& igvn, Node* in1, Node* in2, Node* in3, uint truth_table, const TypeVect* vt);
1285 };
1286 
1287 class VectorMaskCmpNode : public VectorNode {
1288  private:
1289   BoolTest::mask _predicate;
1290 
1291  protected:
1292   virtual  uint size_of() const { return sizeof(VectorMaskCmpNode); }
1293 
1294  public:
1295   VectorMaskCmpNode(BoolTest::mask predicate, Node* in1, Node* in2, ConINode* predicate_node, const TypeVect* vt) :
1296       VectorNode(in1, in2, predicate_node, vt),
1297       _predicate(predicate) {
1298     assert(in1->bottom_type()->is_vect()->element_basic_type() == in2->bottom_type()->is_vect()->element_basic_type(),
1299            "VectorMaskCmp inputs must have same type for elements");
1300     assert(in1->bottom_type()->is_vect()->length() == in2->bottom_type()->is_vect()->length(),
1301            "VectorMaskCmp inputs must have same number of elements");
1302     assert((BoolTest::mask)predicate_node->get_int() == predicate, "Unmatched predicates");
1303     init_class_id(Class_VectorMaskCmp);
1304   }
1305 
1306   virtual int Opcode() const;
1307   virtual uint hash() const { return VectorNode::hash() + _predicate; }
1308   virtual bool cmp( const Node &n ) const {
1309     return VectorNode::cmp(n) && _predicate == ((VectorMaskCmpNode&)n)._predicate;
1310   }
1311   BoolTest::mask get_predicate() { return _predicate; }
1312 #ifndef PRODUCT
1313   virtual void dump_spec(outputStream *st) const;
1314 #endif // !PRODUCT
1315 };
1316 
1317 // Used to wrap other vector nodes in order to add masking functionality.
1318 class VectorMaskWrapperNode : public VectorNode {
1319  public:
1320   VectorMaskWrapperNode(Node* vector, Node* mask)
1321     : VectorNode(vector, mask, vector->bottom_type()->is_vect()) {
1322     assert(mask->is_VectorMaskCmp(), "VectorMaskWrapper requires that second argument be a mask");
1323   }
1324 
1325   virtual int Opcode() const;
1326   Node* vector_val() const { return in(1); }
1327   Node* vector_mask() const { return in(2); }
1328 };
1329 
1330 class VectorTestNode : public Node {
1331  private:
1332   BoolTest::mask _predicate;
1333 
1334  protected:
1335   uint size_of() const { return sizeof(*this); }
1336 
1337  public:
1338   VectorTestNode(Node* in1, Node* in2, BoolTest::mask predicate) : Node(NULL, in1, in2), _predicate(predicate) {
1339     assert(in2->bottom_type()->is_vect() == in2->bottom_type()->is_vect(), "same vector type");
1340   }
1341   virtual int Opcode() const;
1342   virtual uint hash() const { return Node::hash() + _predicate; }
1343   virtual bool cmp( const Node &n ) const {
1344     return Node::cmp(n) && _predicate == ((VectorTestNode&)n)._predicate;
1345   }
1346   virtual const Type *bottom_type() const { return TypeInt::BOOL; }
1347   virtual uint ideal_reg() const { return Op_RegI; }  // TODO Should be RegFlags but due to missing comparison flags for BoolTest
1348                                                       // in middle-end, we make it boolean result directly.
1349   BoolTest::mask get_predicate() const { return _predicate; }
1350 };
1351 
1352 class VectorBlendNode : public VectorNode {
1353  public:
1354   VectorBlendNode(Node* vec1, Node* vec2, Node* mask)
1355     : VectorNode(vec1, vec2, mask, vec1->bottom_type()->is_vect()) {
1356     // assert(mask->is_VectorMask(), "VectorBlendNode requires that third argument be a mask");
1357   }
1358 
1359   virtual int Opcode() const;
1360   Node* vec1() const { return in(1); }
1361   Node* vec2() const { return in(2); }
1362   Node* vec_mask() const { return in(3); }
1363 };
1364 
1365 class VectorRearrangeNode : public VectorNode {
1366  public:
1367   VectorRearrangeNode(Node* vec1, Node* shuffle)
1368     : VectorNode(vec1, shuffle, vec1->bottom_type()->is_vect()) {
1369     // assert(mask->is_VectorMask(), "VectorBlendNode requires that third argument be a mask");
1370   }
1371 
1372   virtual int Opcode() const;
1373   Node* vec1() const { return in(1); }
1374   Node* vec_shuffle() const { return in(2); }
1375 };
1376 
1377 class VectorLoadShuffleNode : public VectorNode {
1378  public:
1379   VectorLoadShuffleNode(Node* in, const TypeVect* vt)
1380     : VectorNode(in, vt) {
1381     assert(in->bottom_type()->is_vect()->element_basic_type() == T_BYTE, "must be BYTE");
1382   }
1383 
1384   int GetOutShuffleSize() const { return type2aelembytes(vect_type()->element_basic_type()); }
1385   virtual int Opcode() const;
1386 };
1387 
1388 class VectorLoadMaskNode : public VectorNode {
1389  public:
1390   VectorLoadMaskNode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {
1391     assert(in->bottom_type()->is_vect()->element_basic_type() == T_BOOLEAN, "must be boolean");
1392   }
1393 
1394   virtual int Opcode() const;
1395   virtual Node* Identity(PhaseGVN* phase);
1396 };
1397 
1398 class VectorStoreMaskNode : public VectorNode {
1399  protected:
1400   VectorStoreMaskNode(Node* in1, ConINode* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) {}
1401 
1402  public:
1403   virtual int Opcode() const;
1404   virtual Node* Identity(PhaseGVN* phase);
1405 
1406   static VectorStoreMaskNode* make(PhaseGVN& gvn, Node* in, BasicType in_type, uint num_elem);
1407 };
1408 
1409 class VectorMaskCastNode : public VectorNode {
1410  public:
1411   VectorMaskCastNode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {
1412     const TypeVect* in_vt = in->bottom_type()->is_vect();
1413     assert(in_vt->length() == vt->length(), "vector length must match");
1414   }
1415   static Node* makeCastNode(PhaseGVN* phase, Node* in1, const TypeVect * vt);
1416   virtual int Opcode() const;
1417 };
1418 
1419 // This is intended for use as a simple reinterpret node that has no cast.
1420 class VectorReinterpretNode : public VectorNode {
1421  private:
1422   const TypeVect* _src_vt;
1423 
1424  protected:
1425   uint size_of() const { return sizeof(VectorReinterpretNode); }
1426  public:
1427   VectorReinterpretNode(Node* in, const TypeVect* src_vt, const TypeVect* dst_vt)
1428      : VectorNode(in, dst_vt), _src_vt(src_vt) {
1429      assert((!dst_vt->isa_vectmask() && !src_vt->isa_vectmask()) ||
1430             (type2aelembytes(src_vt->element_basic_type()) >= type2aelembytes(dst_vt->element_basic_type())),
1431             "unsupported mask widening reinterpretation");
1432      init_class_id(Class_VectorReinterpret);
1433   }
1434 
1435   const TypeVect* src_type() { return _src_vt; }
1436   virtual uint hash() const { return VectorNode::hash() + _src_vt->hash(); }
1437   virtual bool cmp( const Node &n ) const {
1438     return VectorNode::cmp(n) && !Type::cmp(_src_vt,((VectorReinterpretNode&)n)._src_vt);
1439   }
1440   virtual Node* Identity(PhaseGVN* phase);
1441 
1442   virtual int Opcode() const;
1443 };
1444 
1445 class VectorCastNode : public VectorNode {
1446  public:
1447   VectorCastNode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {}
1448   virtual int Opcode() const;
1449 
1450   static VectorCastNode* make(int vopc, Node* n1, BasicType bt, uint vlen);
1451   static int  opcode(BasicType bt);
1452   static bool implemented(BasicType bt, uint vlen);
1453 
1454   virtual Node* Identity(PhaseGVN* phase);
1455 };
1456 
1457 class VectorCastB2XNode : public VectorCastNode {
1458  public:
1459   VectorCastB2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) {
1460     assert(in->bottom_type()->is_vect()->element_basic_type() == T_BYTE, "must be byte");
1461   }
1462   virtual int Opcode() const;
1463 };
1464 
1465 class VectorCastS2XNode : public VectorCastNode {
1466  public:
1467   VectorCastS2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) {
1468     assert(in->bottom_type()->is_vect()->element_basic_type() == T_SHORT, "must be short");
1469   }
1470   virtual int Opcode() const;
1471 };
1472 
1473 class VectorCastI2XNode : public VectorCastNode {
1474  public:
1475   VectorCastI2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) {
1476     assert(in->bottom_type()->is_vect()->element_basic_type() == T_INT, "must be int");
1477   }
1478   virtual int Opcode() const;
1479 };
1480 
1481 class VectorCastL2XNode : public VectorCastNode {
1482  public:
1483   VectorCastL2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) {
1484     assert(in->bottom_type()->is_vect()->element_basic_type() == T_LONG, "must be long");
1485   }
1486   virtual int Opcode() const;
1487 };
1488 
1489 class VectorCastF2XNode : public VectorCastNode {
1490  public:
1491   VectorCastF2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) {
1492     assert(in->bottom_type()->is_vect()->element_basic_type() == T_FLOAT, "must be float");
1493   }
1494   virtual int Opcode() const;
1495 };
1496 
1497 class VectorCastD2XNode : public VectorCastNode {
1498  public:
1499   VectorCastD2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) {
1500     assert(in->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE, "must be double");
1501   }
1502   virtual int Opcode() const;
1503 };
1504 
1505 class VectorInsertNode : public VectorNode {
1506  public:
1507   VectorInsertNode(Node* vsrc, Node* new_val, ConINode* pos, const TypeVect* vt) : VectorNode(vsrc, new_val, (Node*)pos, vt) {
1508    assert(pos->get_int() >= 0, "positive constants");
1509    assert(pos->get_int() < (int)vt->length(), "index must be less than vector length");
1510    assert(Type::cmp(vt, vsrc->bottom_type()) == 0, "input and output must be same type");
1511   }
1512   virtual int Opcode() const;
1513   uint pos() const { return in(3)->get_int(); }
1514 
1515   static Node* make(Node* vec, Node* new_val, int position);
1516 };
1517 
1518 class VectorBoxNode : public Node {
1519  private:
1520   const TypeInstPtr* const _box_type;
1521   const TypeVect*    const _vec_type;
1522  public:
1523   enum {
1524      Box   = 1,
1525      Value = 2
1526   };
1527   VectorBoxNode(Compile* C, Node* box, Node* val,
1528                 const TypeInstPtr* box_type, const TypeVect* vt)
1529     : Node(NULL, box, val), _box_type(box_type), _vec_type(vt) {
1530     init_flags(Flag_is_macro);
1531     C->add_macro_node(this);
1532   }
1533 
1534   const  TypeInstPtr* box_type() const { assert(_box_type != NULL, ""); return _box_type; };
1535   const  TypeVect*    vec_type() const { assert(_vec_type != NULL, ""); return _vec_type; };
1536 
1537   virtual int Opcode() const;
1538   virtual const Type* bottom_type() const { return _box_type; }
1539   virtual       uint  ideal_reg() const { return box_type()->ideal_reg(); }
1540   virtual       uint  size_of() const { return sizeof(*this); }
1541 
1542   static const TypeFunc* vec_box_type(const TypeInstPtr* box_type);
1543 };
1544 
1545 class VectorBoxAllocateNode : public CallStaticJavaNode {
1546  public:
1547   VectorBoxAllocateNode(Compile* C, const TypeInstPtr* vbox_type)
1548     : CallStaticJavaNode(C, VectorBoxNode::vec_box_type(vbox_type), NULL, NULL) {
1549     init_flags(Flag_is_macro);
1550     C->add_macro_node(this);
1551   }
1552 
1553   virtual int Opcode() const;
1554 #ifndef PRODUCT
1555   virtual void dump_spec(outputStream *st) const;
1556 #endif // !PRODUCT
1557 };
1558 
1559 class VectorUnboxNode : public VectorNode {
1560  private:
1561   bool _shuffle_to_vector;
1562  protected:
1563   uint size_of() const { return sizeof(*this); }
1564  public:
1565   VectorUnboxNode(Compile* C, const TypeVect* vec_type, Node* obj, Node* mem, bool shuffle_to_vector)
1566     : VectorNode(mem, obj, vec_type) {
1567     _shuffle_to_vector = shuffle_to_vector;
1568     init_class_id(Class_VectorUnbox);
1569     init_flags(Flag_is_macro);
1570     C->add_macro_node(this);
1571   }
1572 
1573   virtual int Opcode() const;
1574   Node* obj() const { return in(2); }
1575   Node* mem() const { return in(1); }
1576   virtual Node* Identity(PhaseGVN* phase);
1577   Node* Ideal(PhaseGVN* phase, bool can_reshape);
1578   bool is_shuffle_to_vector() { return _shuffle_to_vector; }
1579 };
1580 
1581 class RotateRightVNode : public VectorNode {
1582 public:
1583   RotateRightVNode(Node* in1, Node* in2, const TypeVect* vt)
1584   : VectorNode(in1, in2, vt) {}
1585 
1586   virtual int Opcode() const;
1587   Node* Ideal(PhaseGVN* phase, bool can_reshape);
1588 };
1589 
1590 class RotateLeftVNode : public VectorNode {
1591 public:
1592   RotateLeftVNode(Node* in1, Node* in2, const TypeVect* vt)
1593   : VectorNode(in1, in2, vt) {}
1594 
1595   virtual int Opcode() const;
1596   Node* Ideal(PhaseGVN* phase, bool can_reshape);
1597 };
1598 #endif // SHARE_OPTO_VECTORNODE_HPP