1 /*
   2  * Copyright (c) 2007, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 #include "precompiled.hpp"
  25 #include "memory/allocation.inline.hpp"
  26 #include "opto/connode.hpp"
  27 #include "opto/mulnode.hpp"
  28 #include "opto/subnode.hpp"
  29 #include "opto/vectornode.hpp"
  30 #include "opto/convertnode.hpp"
  31 #include "utilities/powerOfTwo.hpp"
  32 #include "utilities/globalDefinitions.hpp"
  33 
  34 //------------------------------VectorNode--------------------------------------
  35 
  36 // Return the vector operator for the specified scalar operation
  37 // and vector length.
  38 int VectorNode::opcode(int sopc, BasicType bt) {
  39   switch (sopc) {
  40   case Op_AddI:
  41     switch (bt) {
  42     case T_BOOLEAN:
  43     case T_BYTE:      return Op_AddVB;
  44     case T_CHAR:
  45     case T_SHORT:     return Op_AddVS;
  46     case T_INT:       return Op_AddVI;
  47     default:          return 0;
  48     }
  49   case Op_AddL: return (bt == T_LONG   ? Op_AddVL : 0);
  50   case Op_AddF: return (bt == T_FLOAT  ? Op_AddVF : 0);
  51   case Op_AddD: return (bt == T_DOUBLE ? Op_AddVD : 0);
  52 
  53   case Op_SubI:
  54     switch (bt) {
  55     case T_BOOLEAN:
  56     case T_BYTE:   return Op_SubVB;
  57     case T_CHAR:
  58     case T_SHORT:  return Op_SubVS;
  59     case T_INT:    return Op_SubVI;
  60     default:       return 0;
  61     }
  62   case Op_SubL: return (bt == T_LONG   ? Op_SubVL : 0);
  63   case Op_SubF: return (bt == T_FLOAT  ? Op_SubVF : 0);
  64   case Op_SubD: return (bt == T_DOUBLE ? Op_SubVD : 0);
  65 
  66   case Op_MulI:
  67     switch (bt) {
  68     case T_BOOLEAN:return 0;
  69     case T_BYTE:   return Op_MulVB;
  70     case T_CHAR:
  71     case T_SHORT:  return Op_MulVS;
  72     case T_INT:    return Op_MulVI;
  73     default:       return 0;
  74     }
  75   case Op_MulL: return (bt == T_LONG ? Op_MulVL : 0);
  76   case Op_MulF:
  77     return (bt == T_FLOAT ? Op_MulVF : 0);
  78   case Op_MulD:
  79     return (bt == T_DOUBLE ? Op_MulVD : 0);
  80   case Op_FmaD:
  81     return (bt == T_DOUBLE ? Op_FmaVD : 0);
  82   case Op_FmaF:
  83     return (bt == T_FLOAT ? Op_FmaVF : 0);
  84   case Op_CMoveF:
  85     return (bt == T_FLOAT ? Op_CMoveVF : 0);
  86   case Op_CMoveD:
  87     return (bt == T_DOUBLE ? Op_CMoveVD : 0);
  88   case Op_DivF:
  89     return (bt == T_FLOAT ? Op_DivVF : 0);
  90   case Op_DivD:
  91     return (bt == T_DOUBLE ? Op_DivVD : 0);
  92   case Op_AbsI:
  93     switch (bt) {
  94     case T_BOOLEAN:
  95     case T_CHAR:  return 0; // abs does not make sense for unsigned
  96     case T_BYTE:  return Op_AbsVB;
  97     case T_SHORT: return Op_AbsVS;
  98     case T_INT:   return Op_AbsVI;
  99     default:      return 0;
 100     }
 101   case Op_AbsL:
 102     return (bt == T_LONG ? Op_AbsVL : 0);
 103   case Op_MinI:
 104     switch (bt) {
 105     case T_BOOLEAN:
 106     case T_CHAR:   return 0;
 107     case T_BYTE:
 108     case T_SHORT:
 109     case T_INT:    return Op_MinV;
 110     default:       return 0;
 111     }
 112   case Op_MinL:
 113     return (bt == T_LONG ? Op_MinV : 0);
 114   case Op_MinF:
 115     return (bt == T_FLOAT ? Op_MinV : 0);
 116   case Op_MinD:
 117     return (bt == T_DOUBLE ? Op_MinV : 0);
 118   case Op_MaxI:
 119     switch (bt) {
 120     case T_BOOLEAN:
 121     case T_CHAR:   return 0;
 122     case T_BYTE:
 123     case T_SHORT:
 124     case T_INT:    return Op_MaxV;
 125     default:       return 0;
 126     }
 127   case Op_MaxL:
 128     return (bt == T_LONG ? Op_MaxV : 0);
 129   case Op_MaxF:
 130     return (bt == T_FLOAT ? Op_MaxV : 0);
 131   case Op_MaxD:
 132     return (bt == T_DOUBLE ? Op_MaxV : 0);
 133   case Op_AbsF:
 134     return (bt == T_FLOAT ? Op_AbsVF : 0);
 135   case Op_AbsD:
 136     return (bt == T_DOUBLE ? Op_AbsVD : 0);
 137   case Op_NegI:
 138     return (bt == T_INT ? Op_NegVI : 0);
 139   case Op_NegF:
 140     return (bt == T_FLOAT ? Op_NegVF : 0);
 141   case Op_NegD:
 142     return (bt == T_DOUBLE ? Op_NegVD : 0);
 143   case Op_RoundDoubleMode:
 144     return (bt == T_DOUBLE ? Op_RoundDoubleModeV : 0);
 145   case Op_RotateLeft:
 146     return (is_integral_type(bt) ? Op_RotateLeftV : 0);
 147   case Op_RotateRight:
 148     return (is_integral_type(bt) ? Op_RotateRightV : 0);
 149   case Op_SqrtF:
 150     return (bt == T_FLOAT ? Op_SqrtVF : 0);
 151   case Op_SqrtD:
 152     return (bt == T_DOUBLE ? Op_SqrtVD : 0);
 153   case Op_PopCountI:
 154     // Unimplemented for subword types since bit count changes
 155     // depending on size of lane (and sign bit).
 156     return (bt == T_INT ? Op_PopCountVI : 0);
 157   case Op_LShiftI:
 158     switch (bt) {
 159     case T_BOOLEAN:
 160     case T_BYTE:   return Op_LShiftVB;
 161     case T_CHAR:
 162     case T_SHORT:  return Op_LShiftVS;
 163     case T_INT:    return Op_LShiftVI;
 164     default:       return 0;
 165     }
 166   case Op_LShiftL:
 167     return (bt == T_LONG ? Op_LShiftVL : 0);
 168   case Op_RShiftI:
 169     switch (bt) {
 170     case T_BOOLEAN:return Op_URShiftVB; // boolean is unsigned value
 171     case T_CHAR:   return Op_URShiftVS; // char is unsigned value
 172     case T_BYTE:   return Op_RShiftVB;
 173     case T_SHORT:  return Op_RShiftVS;
 174     case T_INT:    return Op_RShiftVI;
 175     default:       return 0;
 176     }
 177   case Op_RShiftL:
 178     return (bt == T_LONG ? Op_RShiftVL : 0);
 179   case Op_URShiftB:
 180     return (bt == T_BYTE ? Op_URShiftVB : 0);
 181   case Op_URShiftS:
 182     return (bt == T_SHORT ? Op_URShiftVS : 0);
 183   case Op_URShiftI:
 184     switch (bt) {
 185     case T_BOOLEAN:return Op_URShiftVB;
 186     case T_CHAR:   return Op_URShiftVS;
 187     case T_BYTE:
 188     case T_SHORT:  return 0; // Vector logical right shift for signed short
 189                              // values produces incorrect Java result for
 190                              // negative data because java code should convert
 191                              // a short value into int value with sign
 192                              // extension before a shift.
 193     case T_INT:    return Op_URShiftVI;
 194     default:       return 0;
 195     }
 196   case Op_URShiftL:
 197     return (bt == T_LONG ? Op_URShiftVL : 0);
 198   case Op_AndI:
 199   case Op_AndL:
 200     return Op_AndV;
 201   case Op_OrI:
 202   case Op_OrL:
 203     return Op_OrV;
 204   case Op_XorI:
 205   case Op_XorL:
 206     return Op_XorV;
 207 
 208   case Op_LoadB:
 209   case Op_LoadUB:
 210   case Op_LoadUS:
 211   case Op_LoadS:
 212   case Op_LoadI:
 213   case Op_LoadL:
 214   case Op_LoadF:
 215   case Op_LoadD:
 216     return Op_LoadVector;
 217 
 218   case Op_StoreB:
 219   case Op_StoreC:
 220   case Op_StoreI:
 221   case Op_StoreL:
 222   case Op_StoreF:
 223   case Op_StoreD:
 224     return Op_StoreVector;
 225   case Op_MulAddS2I:
 226     return Op_MulAddVS2VI;
 227 
 228   default:
 229     return 0; // Unimplemented
 230   }
 231 }
 232 
 233 int VectorNode::replicate_opcode(BasicType bt) {
 234   switch(bt) {
 235     case T_BOOLEAN:
 236     case T_BYTE:
 237       return Op_ReplicateB;
 238     case T_SHORT:
 239     case T_CHAR:
 240       return Op_ReplicateS;
 241     case T_INT:
 242       return Op_ReplicateI;
 243     case T_LONG:
 244       return Op_ReplicateL;
 245     case T_FLOAT:
 246       return Op_ReplicateF;
 247     case T_DOUBLE:
 248       return Op_ReplicateD;
 249     default:
 250       assert(false, "wrong type: %s", type2name(bt));
 251       return 0;
 252   }
 253 }
 254 
 255 // Also used to check if the code generator
 256 // supports the vector operation.
 257 bool VectorNode::implemented(int opc, uint vlen, BasicType bt) {
 258   if (is_java_primitive(bt) &&
 259       (vlen > 1) && is_power_of_2(vlen) &&
 260       Matcher::vector_size_supported(bt, vlen)) {
 261     int vopc = VectorNode::opcode(opc, bt);
 262     // For rotate operation we will do a lazy de-generation into
 263     // OrV/LShiftV/URShiftV pattern if the target does not support
 264     // vector rotation instruction.
 265     if (VectorNode::is_vector_rotate(vopc)) {
 266       return is_vector_rotate_supported(vopc, vlen, bt);
 267     }
 268     return vopc > 0 && Matcher::match_rule_supported_vector(vopc, vlen, bt);
 269   }
 270   return false;
 271 }
 272 
 273 bool VectorNode::is_type_transition_short_to_int(Node* n) {
 274   switch (n->Opcode()) {
 275   case Op_MulAddS2I:
 276     return true;
 277   }
 278   return false;
 279 }
 280 
 281 bool VectorNode::is_type_transition_to_int(Node* n) {
 282   return is_type_transition_short_to_int(n);
 283 }
 284 
 285 bool VectorNode::is_muladds2i(Node* n) {
 286   if (n->Opcode() == Op_MulAddS2I) {
 287     return true;
 288   }
 289   return false;
 290 }
 291 
 292 bool VectorNode::is_roundopD(Node* n) {
 293   if (n->Opcode() == Op_RoundDoubleMode) {
 294     return true;
 295   }
 296   return false;
 297 }
 298 
 299 bool VectorNode::is_vector_rotate_supported(int vopc, uint vlen, BasicType bt) {
 300   assert(VectorNode::is_vector_rotate(vopc), "wrong opcode");
 301 
 302   // If target defines vector rotation patterns then no
 303   // need for degeneration.
 304   if (Matcher::match_rule_supported_vector(vopc, vlen, bt)) {
 305     return true;
 306   }
 307 
 308   // If target does not support variable shift operations then no point
 309   // in creating a rotate vector node since it will not be disintegratable.
 310   // Adding a pessimistic check to avoid complex pattern mathing which
 311   // may not be full proof.
 312   if (!Matcher::supports_vector_variable_shifts()) {
 313      return false;
 314   }
 315 
 316   // Validate existence of nodes created in case of rotate degeneration.
 317   switch (bt) {
 318     case T_INT:
 319       return Matcher::match_rule_supported_vector(Op_OrV,       vlen, bt) &&
 320              Matcher::match_rule_supported_vector(Op_LShiftVI,  vlen, bt) &&
 321              Matcher::match_rule_supported_vector(Op_URShiftVI, vlen, bt);
 322     case T_LONG:
 323       return Matcher::match_rule_supported_vector(Op_OrV,       vlen, bt) &&
 324              Matcher::match_rule_supported_vector(Op_LShiftVL,  vlen, bt) &&
 325              Matcher::match_rule_supported_vector(Op_URShiftVL, vlen, bt);
 326     default:
 327       assert(false, "not supported: %s", type2name(bt));
 328       return false;
 329   }
 330 }
 331 
 332 bool VectorNode::is_shift_opcode(int opc) {
 333   switch (opc) {
 334   case Op_LShiftI:
 335   case Op_LShiftL:
 336   case Op_RShiftI:
 337   case Op_RShiftL:
 338   case Op_URShiftB:
 339   case Op_URShiftS:
 340   case Op_URShiftI:
 341   case Op_URShiftL:
 342     return true;
 343   default:
 344     return false;
 345   }
 346 }
 347 
 348 bool VectorNode::is_shift(Node* n) {
 349   return is_shift_opcode(n->Opcode());
 350 }
 351 
 352 bool VectorNode::is_rotate_opcode(int opc) {
 353   switch (opc) {
 354   case Op_RotateRight:
 355   case Op_RotateLeft:
 356     return true;
 357   default:
 358     return false;
 359   }
 360 }
 361 
 362 bool VectorNode::is_scalar_rotate(Node* n) {
 363   if (is_rotate_opcode(n->Opcode())) {
 364     return true;
 365   }
 366   return false;
 367 }
 368 
 369 bool VectorNode::is_vshift_cnt_opcode(int opc) {
 370   switch (opc) {
 371   case Op_LShiftCntV:
 372   case Op_RShiftCntV:
 373     return true;
 374   default:
 375     return false;
 376   }
 377 }
 378 
 379 bool VectorNode::is_vshift_cnt(Node* n) {
 380   return is_vshift_cnt_opcode(n->Opcode());
 381 }
 382 
 383 // Check if input is loop invariant vector.
 384 bool VectorNode::is_invariant_vector(Node* n) {
 385   // Only Replicate vector nodes are loop invariant for now.
 386   switch (n->Opcode()) {
 387   case Op_ReplicateB:
 388   case Op_ReplicateS:
 389   case Op_ReplicateI:
 390   case Op_ReplicateL:
 391   case Op_ReplicateF:
 392   case Op_ReplicateD:
 393     return true;
 394   default:
 395     return false;
 396   }
 397 }
 398 
 399 // [Start, end) half-open range defining which operands are vectors
 400 void VectorNode::vector_operands(Node* n, uint* start, uint* end) {
 401   switch (n->Opcode()) {
 402   case Op_LoadB:   case Op_LoadUB:
 403   case Op_LoadS:   case Op_LoadUS:
 404   case Op_LoadI:   case Op_LoadL:
 405   case Op_LoadF:   case Op_LoadD:
 406   case Op_LoadP:   case Op_LoadN:
 407     *start = 0;
 408     *end   = 0; // no vector operands
 409     break;
 410   case Op_StoreB:  case Op_StoreC:
 411   case Op_StoreI:  case Op_StoreL:
 412   case Op_StoreF:  case Op_StoreD:
 413   case Op_StoreP:  case Op_StoreN:
 414     *start = MemNode::ValueIn;
 415     *end   = MemNode::ValueIn + 1; // 1 vector operand
 416     break;
 417   case Op_LShiftI:  case Op_LShiftL:
 418   case Op_RShiftI:  case Op_RShiftL:
 419   case Op_URShiftI: case Op_URShiftL:
 420     *start = 1;
 421     *end   = 2; // 1 vector operand
 422     break;
 423   case Op_AddI: case Op_AddL: case Op_AddF: case Op_AddD:
 424   case Op_SubI: case Op_SubL: case Op_SubF: case Op_SubD:
 425   case Op_MulI: case Op_MulL: case Op_MulF: case Op_MulD:
 426   case Op_DivF: case Op_DivD:
 427   case Op_AndI: case Op_AndL:
 428   case Op_OrI:  case Op_OrL:
 429   case Op_XorI: case Op_XorL:
 430   case Op_MulAddS2I:
 431     *start = 1;
 432     *end   = 3; // 2 vector operands
 433     break;
 434   case Op_CMoveI:  case Op_CMoveL:  case Op_CMoveF:  case Op_CMoveD:
 435     *start = 2;
 436     *end   = n->req();
 437     break;
 438   case Op_FmaD:
 439   case Op_FmaF:
 440     *start = 1;
 441     *end   = 4; // 3 vector operands
 442     break;
 443   default:
 444     *start = 1;
 445     *end   = n->req(); // default is all operands
 446   }
 447 }
 448 
 449 VectorNode* VectorNode::make_mask_node(int vopc, Node* n1, Node* n2, uint vlen, BasicType bt) {
 450   guarantee(vopc > 0, "vopc must be > 0");
 451   const TypeVect* vmask_type = TypeVect::makemask(bt, vlen);
 452   switch (vopc) {
 453     case Op_AndV:
 454       if (Matcher::match_rule_supported_vector_masked(Op_AndVMask, vlen, bt)) {
 455         return new AndVMaskNode(n1, n2, vmask_type);
 456       }
 457       return new AndVNode(n1, n2, vmask_type);
 458     case Op_OrV:
 459       if (Matcher::match_rule_supported_vector_masked(Op_OrVMask, vlen, bt)) {
 460         return new OrVMaskNode(n1, n2, vmask_type);
 461       }
 462       return new OrVNode(n1, n2, vmask_type);
 463     case Op_XorV:
 464       if (Matcher::match_rule_supported_vector_masked(Op_XorVMask, vlen, bt)) {
 465         return new XorVMaskNode(n1, n2, vmask_type);
 466       }
 467       return new XorVNode(n1, n2, vmask_type);
 468     default:
 469       fatal("Unsupported mask vector creation for '%s'", NodeClassNames[vopc]);
 470       return NULL;
 471   }
 472 }
 473 
 474 // Make a vector node for binary operation
 475 VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, const TypeVect* vt, bool is_mask) {
 476   // This method should not be called for unimplemented vectors.
 477   guarantee(vopc > 0, "vopc must be > 0");
 478 
 479   if (is_mask) {
 480     return make_mask_node(vopc, n1, n2, vt->length(), vt->element_basic_type());
 481   }
 482 
 483   switch (vopc) {
 484   case Op_AddVB: return new AddVBNode(n1, n2, vt);
 485   case Op_AddVS: return new AddVSNode(n1, n2, vt);
 486   case Op_AddVI: return new AddVINode(n1, n2, vt);
 487   case Op_AddVL: return new AddVLNode(n1, n2, vt);
 488   case Op_AddVF: return new AddVFNode(n1, n2, vt);
 489   case Op_AddVD: return new AddVDNode(n1, n2, vt);
 490 
 491   case Op_SubVB: return new SubVBNode(n1, n2, vt);
 492   case Op_SubVS: return new SubVSNode(n1, n2, vt);
 493   case Op_SubVI: return new SubVINode(n1, n2, vt);
 494   case Op_SubVL: return new SubVLNode(n1, n2, vt);
 495   case Op_SubVF: return new SubVFNode(n1, n2, vt);
 496   case Op_SubVD: return new SubVDNode(n1, n2, vt);
 497 
 498   case Op_MulVB: return new MulVBNode(n1, n2, vt);
 499   case Op_MulVS: return new MulVSNode(n1, n2, vt);
 500   case Op_MulVI: return new MulVINode(n1, n2, vt);
 501   case Op_MulVL: return new MulVLNode(n1, n2, vt);
 502   case Op_MulVF: return new MulVFNode(n1, n2, vt);
 503   case Op_MulVD: return new MulVDNode(n1, n2, vt);
 504 
 505   case Op_DivVF: return new DivVFNode(n1, n2, vt);
 506   case Op_DivVD: return new DivVDNode(n1, n2, vt);
 507 
 508   case Op_MinV: return new MinVNode(n1, n2, vt);
 509   case Op_MaxV: return new MaxVNode(n1, n2, vt);
 510 
 511   case Op_AbsVF: return new AbsVFNode(n1, vt);
 512   case Op_AbsVD: return new AbsVDNode(n1, vt);
 513   case Op_AbsVB: return new AbsVBNode(n1, vt);
 514   case Op_AbsVS: return new AbsVSNode(n1, vt);
 515   case Op_AbsVI: return new AbsVINode(n1, vt);
 516   case Op_AbsVL: return new AbsVLNode(n1, vt);
 517 
 518   case Op_NegVI: return new NegVINode(n1, vt);
 519   case Op_NegVF: return new NegVFNode(n1, vt);
 520   case Op_NegVD: return new NegVDNode(n1, vt);
 521 
 522   case Op_SqrtVF: return new SqrtVFNode(n1, vt);
 523   case Op_SqrtVD: return new SqrtVDNode(n1, vt);
 524 
 525   case Op_PopCountVI: return new PopCountVINode(n1, vt);
 526   case Op_RotateLeftV: return new RotateLeftVNode(n1, n2, vt);
 527   case Op_RotateRightV: return new RotateRightVNode(n1, n2, vt);
 528 
 529   case Op_LShiftVB: return new LShiftVBNode(n1, n2, vt);
 530   case Op_LShiftVS: return new LShiftVSNode(n1, n2, vt);
 531   case Op_LShiftVI: return new LShiftVINode(n1, n2, vt);
 532   case Op_LShiftVL: return new LShiftVLNode(n1, n2, vt);
 533 
 534   case Op_RShiftVB: return new RShiftVBNode(n1, n2, vt);
 535   case Op_RShiftVS: return new RShiftVSNode(n1, n2, vt);
 536   case Op_RShiftVI: return new RShiftVINode(n1, n2, vt);
 537   case Op_RShiftVL: return new RShiftVLNode(n1, n2, vt);
 538 
 539   case Op_URShiftVB: return new URShiftVBNode(n1, n2, vt);
 540   case Op_URShiftVS: return new URShiftVSNode(n1, n2, vt);
 541   case Op_URShiftVI: return new URShiftVINode(n1, n2, vt);
 542   case Op_URShiftVL: return new URShiftVLNode(n1, n2, vt);
 543 
 544   case Op_AndV: return new AndVNode(n1, n2, vt);
 545   case Op_OrV:  return new OrVNode (n1, n2, vt);
 546   case Op_XorV: return new XorVNode(n1, n2, vt);
 547 
 548   case Op_RoundDoubleModeV: return new RoundDoubleModeVNode(n1, n2, vt);
 549 
 550   case Op_MulAddVS2VI: return new MulAddVS2VINode(n1, n2, vt);
 551   default:
 552     fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
 553     return NULL;
 554   }
 555 }
 556 
 557 // Return the vector version of a scalar binary operation node.
 558 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, uint vlen, BasicType bt) {
 559   const TypeVect* vt = TypeVect::make(bt, vlen);
 560   int vopc = VectorNode::opcode(opc, bt);
 561   // This method should not be called for unimplemented vectors.
 562   guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]);
 563   return make(vopc, n1, n2, vt);
 564 }
 565 
 566 // Make a vector node for ternary operation
 567 VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, Node* n3, const TypeVect* vt) {
 568   // This method should not be called for unimplemented vectors.
 569   guarantee(vopc > 0, "vopc must be > 0");
 570   switch (vopc) {
 571   case Op_FmaVD: return new FmaVDNode(n1, n2, n3, vt);
 572   case Op_FmaVF: return new FmaVFNode(n1, n2, n3, vt);
 573   default:
 574     fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
 575     return NULL;
 576   }
 577 }
 578 
 579 // Return the vector version of a scalar ternary operation node.
 580 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, Node* n3, uint vlen, BasicType bt) {
 581   const TypeVect* vt = TypeVect::make(bt, vlen);
 582   int vopc = VectorNode::opcode(opc, bt);
 583   // This method should not be called for unimplemented vectors.
 584   guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]);
 585   return make(vopc, n1, n2, n3, vt);
 586 }
 587 
 588 // Scalar promotion
 589 VectorNode* VectorNode::scalar2vector(Node* s, uint vlen, const Type* opd_t, bool is_mask) {
 590   BasicType bt = opd_t->array_element_basic_type();
 591   const TypeVect* vt = opd_t->singleton() ? TypeVect::make(opd_t, vlen, is_mask)
 592                                           : TypeVect::make(bt, vlen, is_mask);
 593 
 594   if (is_mask && Matcher::match_rule_supported_vector(Op_MaskAll, vlen, bt)) {
 595     return new MaskAllNode(s, vt);
 596   }
 597 
 598   switch (bt) {
 599   case T_BOOLEAN:
 600   case T_BYTE:
 601     return new ReplicateBNode(s, vt);
 602   case T_CHAR:
 603   case T_SHORT:
 604     return new ReplicateSNode(s, vt);
 605   case T_INT:
 606     return new ReplicateINode(s, vt);
 607   case T_LONG:
 608     return new ReplicateLNode(s, vt);
 609   case T_FLOAT:
 610     return new ReplicateFNode(s, vt);
 611   case T_DOUBLE:
 612     return new ReplicateDNode(s, vt);
 613   default:
 614     fatal("Type '%s' is not supported for vectors", type2name(bt));
 615     return NULL;
 616   }
 617 }
 618 
 619 VectorNode* VectorNode::shift_count(int opc, Node* cnt, uint vlen, BasicType bt) {
 620   // Match shift count type with shift vector type.
 621   const TypeVect* vt = TypeVect::make(bt, vlen);
 622   switch (opc) {
 623   case Op_LShiftI:
 624   case Op_LShiftL:
 625     return new LShiftCntVNode(cnt, vt);
 626   case Op_RShiftI:
 627   case Op_RShiftL:
 628   case Op_URShiftB:
 629   case Op_URShiftS:
 630   case Op_URShiftI:
 631   case Op_URShiftL:
 632     return new RShiftCntVNode(cnt, vt);
 633   default:
 634     fatal("Missed vector creation for '%s'", NodeClassNames[opc]);
 635     return NULL;
 636   }
 637 }
 638 
 639 bool VectorNode::is_vector_rotate(int opc) {
 640   switch (opc) {
 641   case Op_RotateLeftV:
 642   case Op_RotateRightV:
 643     return true;
 644   default:
 645     return false;
 646   }
 647 }
 648 
 649 bool VectorNode::is_vector_shift(int opc) {
 650   assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode");
 651   switch (opc) {
 652   case Op_LShiftVB:
 653   case Op_LShiftVS:
 654   case Op_LShiftVI:
 655   case Op_LShiftVL:
 656   case Op_RShiftVB:
 657   case Op_RShiftVS:
 658   case Op_RShiftVI:
 659   case Op_RShiftVL:
 660   case Op_URShiftVB:
 661   case Op_URShiftVS:
 662   case Op_URShiftVI:
 663   case Op_URShiftVL:
 664     return true;
 665   default:
 666     return false;
 667   }
 668 }
 669 
 670 bool VectorNode::is_vector_shift_count(int opc) {
 671   assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode");
 672   switch (opc) {
 673   case Op_RShiftCntV:
 674   case Op_LShiftCntV:
 675     return true;
 676   default:
 677     return false;
 678   }
 679 }
 680 
 681 static bool is_con_M1(Node* n) {
 682   if (n->is_Con()) {
 683     const Type* t = n->bottom_type();
 684     if (t->isa_int() && t->is_int()->get_con() == -1) {
 685       return true;
 686     }
 687     if (t->isa_long() && t->is_long()->get_con() == -1) {
 688       return true;
 689     }
 690   }
 691   return false;
 692 }
 693 
 694 bool VectorNode::is_all_ones_vector(Node* n) {
 695   switch (n->Opcode()) {
 696   case Op_ReplicateB:
 697   case Op_ReplicateS:
 698   case Op_ReplicateI:
 699   case Op_ReplicateL:
 700     return is_con_M1(n->in(1));
 701   default:
 702     return false;
 703   }
 704 }
 705 
 706 bool VectorNode::is_vector_bitwise_not_pattern(Node* n) {
 707   if (n->Opcode() == Op_XorV) {
 708     return is_all_ones_vector(n->in(1)) ||
 709            is_all_ones_vector(n->in(2));
 710   }
 711   return false;
 712 }
 713 
 714 // Return initial Pack node. Additional operands added with add_opd() calls.
 715 PackNode* PackNode::make(Node* s, uint vlen, BasicType bt) {
 716   const TypeVect* vt = TypeVect::make(bt, vlen);
 717   switch (bt) {
 718   case T_BOOLEAN:
 719   case T_BYTE:
 720     return new PackBNode(s, vt);
 721   case T_CHAR:
 722   case T_SHORT:
 723     return new PackSNode(s, vt);
 724   case T_INT:
 725     return new PackINode(s, vt);
 726   case T_LONG:
 727     return new PackLNode(s, vt);
 728   case T_FLOAT:
 729     return new PackFNode(s, vt);
 730   case T_DOUBLE:
 731     return new PackDNode(s, vt);
 732   default:
 733     fatal("Type '%s' is not supported for vectors", type2name(bt));
 734     return NULL;
 735   }
 736 }
 737 
 738 // Create a binary tree form for Packs. [lo, hi) (half-open) range
 739 PackNode* PackNode::binary_tree_pack(int lo, int hi) {
 740   int ct = hi - lo;
 741   assert(is_power_of_2(ct), "power of 2");
 742   if (ct == 2) {
 743     PackNode* pk = PackNode::make(in(lo), 2, vect_type()->element_basic_type());
 744     pk->add_opd(in(lo+1));
 745     return pk;
 746   } else {
 747     int mid = lo + ct/2;
 748     PackNode* n1 = binary_tree_pack(lo,  mid);
 749     PackNode* n2 = binary_tree_pack(mid, hi );
 750 
 751     BasicType bt = n1->vect_type()->element_basic_type();
 752     assert(bt == n2->vect_type()->element_basic_type(), "should be the same");
 753     switch (bt) {
 754     case T_BOOLEAN:
 755     case T_BYTE:
 756       return new PackSNode(n1, n2, TypeVect::make(T_SHORT, 2));
 757     case T_CHAR:
 758     case T_SHORT:
 759       return new PackINode(n1, n2, TypeVect::make(T_INT, 2));
 760     case T_INT:
 761       return new PackLNode(n1, n2, TypeVect::make(T_LONG, 2));
 762     case T_LONG:
 763       return new Pack2LNode(n1, n2, TypeVect::make(T_LONG, 2));
 764     case T_FLOAT:
 765       return new PackDNode(n1, n2, TypeVect::make(T_DOUBLE, 2));
 766     case T_DOUBLE:
 767       return new Pack2DNode(n1, n2, TypeVect::make(T_DOUBLE, 2));
 768     default:
 769       fatal("Type '%s' is not supported for vectors", type2name(bt));
 770       return NULL;
 771     }
 772   }
 773 }
 774 
 775 // Return the vector version of a scalar load node.
 776 LoadVectorNode* LoadVectorNode::make(int opc, Node* ctl, Node* mem,
 777                                      Node* adr, const TypePtr* atyp,
 778                                      uint vlen, BasicType bt,
 779                                      ControlDependency control_dependency) {
 780   const TypeVect* vt = TypeVect::make(bt, vlen);
 781   return new LoadVectorNode(ctl, mem, adr, atyp, vt, control_dependency);
 782 }
 783 
 784 // Return the vector version of a scalar store node.
 785 StoreVectorNode* StoreVectorNode::make(int opc, Node* ctl, Node* mem,
 786                                        Node* adr, const TypePtr* atyp, Node* val,
 787                                        uint vlen) {
 788   return new StoreVectorNode(ctl, mem, adr, atyp, val);
 789 }
 790 
 791 Node* LoadVectorMaskedNode::Ideal(PhaseGVN* phase, bool can_reshape) {
 792   if (!in(3)->is_top() && in(3)->Opcode() == Op_VectorMaskGen) {
 793     Node* mask_len = in(3)->in(1);
 794     const TypeLong* ty = phase->type(mask_len)->isa_long();
 795     if (ty && ty->is_con()) {
 796       BasicType mask_bt = ((VectorMaskGenNode*)in(3))->get_elem_type();
 797       uint load_sz      = type2aelembytes(mask_bt) * ty->get_con();
 798       if ( load_sz == 32 || load_sz == 64) {
 799         assert(load_sz == 32 || MaxVectorSize > 32, "Unexpected load size");
 800         Node* ctr = in(MemNode::Control);
 801         Node* mem = in(MemNode::Memory);
 802         Node* adr = in(MemNode::Address);
 803         return phase->transform(new LoadVectorNode(ctr, mem, adr, adr_type(), vect_type()));
 804       }
 805     }
 806   }
 807   return NULL;
 808 }
 809 
 810 Node* StoreVectorMaskedNode::Ideal(PhaseGVN* phase, bool can_reshape) {
 811   if (!in(4)->is_top() && in(4)->Opcode() == Op_VectorMaskGen) {
 812     Node* mask_len = in(4)->in(1);
 813     const TypeLong* ty = phase->type(mask_len)->isa_long();
 814     if (ty && ty->is_con()) {
 815       BasicType mask_bt = ((VectorMaskGenNode*)in(4))->get_elem_type();
 816       uint load_sz      = type2aelembytes(mask_bt) * ty->get_con();
 817       if ( load_sz == 32 || load_sz == 64) {
 818         assert(load_sz == 32 || MaxVectorSize > 32, "Unexpected store size");
 819         Node* ctr = in(MemNode::Control);
 820         Node* mem = in(MemNode::Memory);
 821         Node* adr = in(MemNode::Address);
 822         Node* val = in(MemNode::ValueIn);
 823         return phase->transform(new StoreVectorNode(ctr, mem, adr, adr_type(), val));
 824       }
 825     }
 826   }
 827   return NULL;
 828 }
 829 
 830 int ExtractNode::opcode(BasicType bt) {
 831   switch (bt) {
 832     case T_BOOLEAN: return Op_ExtractUB;
 833     case T_BYTE:    return Op_ExtractB;
 834     case T_CHAR:    return Op_ExtractC;
 835     case T_SHORT:   return Op_ExtractS;
 836     case T_INT:     return Op_ExtractI;
 837     case T_LONG:    return Op_ExtractL;
 838     case T_FLOAT:   return Op_ExtractF;
 839     case T_DOUBLE:  return Op_ExtractD;
 840     default:
 841       assert(false, "wrong type: %s", type2name(bt));
 842       return 0;
 843   }
 844 }
 845 
 846 // Extract a scalar element of vector.
 847 Node* ExtractNode::make(Node* v, uint position, BasicType bt) {
 848   assert((int)position < Matcher::max_vector_size(bt), "pos in range");
 849   ConINode* pos = ConINode::make((int)position);
 850   switch (bt) {
 851   case T_BOOLEAN: return new ExtractUBNode(v, pos);
 852   case T_BYTE:    return new ExtractBNode(v, pos);
 853   case T_CHAR:    return new ExtractCNode(v, pos);
 854   case T_SHORT:   return new ExtractSNode(v, pos);
 855   case T_INT:     return new ExtractINode(v, pos);
 856   case T_LONG:    return new ExtractLNode(v, pos);
 857   case T_FLOAT:   return new ExtractFNode(v, pos);
 858   case T_DOUBLE:  return new ExtractDNode(v, pos);
 859   default:
 860     assert(false, "wrong type: %s", type2name(bt));
 861     return NULL;
 862   }
 863 }
 864 
 865 int ReductionNode::opcode(int opc, BasicType bt) {
 866   int vopc = opc;
 867   switch (opc) {
 868     case Op_AddI:
 869       switch (bt) {
 870         case T_BOOLEAN:
 871         case T_CHAR: return 0;
 872         case T_BYTE:
 873         case T_SHORT:
 874         case T_INT:
 875           vopc = Op_AddReductionVI;
 876           break;
 877         default: ShouldNotReachHere(); return 0;
 878       }
 879       break;
 880     case Op_AddL:
 881       assert(bt == T_LONG, "must be");
 882       vopc = Op_AddReductionVL;
 883       break;
 884     case Op_AddF:
 885       assert(bt == T_FLOAT, "must be");
 886       vopc = Op_AddReductionVF;
 887       break;
 888     case Op_AddD:
 889       assert(bt == T_DOUBLE, "must be");
 890       vopc = Op_AddReductionVD;
 891       break;
 892     case Op_MulI:
 893       switch (bt) {
 894         case T_BOOLEAN:
 895         case T_CHAR: return 0;
 896         case T_BYTE:
 897         case T_SHORT:
 898         case T_INT:
 899           vopc = Op_MulReductionVI;
 900           break;
 901         default: ShouldNotReachHere(); return 0;
 902       }
 903       break;
 904     case Op_MulL:
 905       assert(bt == T_LONG, "must be");
 906       vopc = Op_MulReductionVL;
 907       break;
 908     case Op_MulF:
 909       assert(bt == T_FLOAT, "must be");
 910       vopc = Op_MulReductionVF;
 911       break;
 912     case Op_MulD:
 913       assert(bt == T_DOUBLE, "must be");
 914       vopc = Op_MulReductionVD;
 915       break;
 916     case Op_MinI:
 917       switch (bt) {
 918         case T_BOOLEAN:
 919         case T_CHAR: return 0;
 920         case T_BYTE:
 921         case T_SHORT:
 922         case T_INT:
 923           vopc = Op_MinReductionV;
 924           break;
 925         default: ShouldNotReachHere(); return 0;
 926       }
 927       break;
 928     case Op_MinL:
 929       assert(bt == T_LONG, "must be");
 930       vopc = Op_MinReductionV;
 931       break;
 932     case Op_MinF:
 933       assert(bt == T_FLOAT, "must be");
 934       vopc = Op_MinReductionV;
 935       break;
 936     case Op_MinD:
 937       assert(bt == T_DOUBLE, "must be");
 938       vopc = Op_MinReductionV;
 939       break;
 940     case Op_MaxI:
 941       switch (bt) {
 942         case T_BOOLEAN:
 943         case T_CHAR: return 0;
 944         case T_BYTE:
 945         case T_SHORT:
 946         case T_INT:
 947           vopc = Op_MaxReductionV;
 948           break;
 949         default: ShouldNotReachHere(); return 0;
 950       }
 951       break;
 952     case Op_MaxL:
 953       assert(bt == T_LONG, "must be");
 954       vopc = Op_MaxReductionV;
 955       break;
 956     case Op_MaxF:
 957       assert(bt == T_FLOAT, "must be");
 958       vopc = Op_MaxReductionV;
 959       break;
 960     case Op_MaxD:
 961       assert(bt == T_DOUBLE, "must be");
 962       vopc = Op_MaxReductionV;
 963       break;
 964     case Op_AndI:
 965       switch (bt) {
 966       case T_BOOLEAN:
 967       case T_CHAR: return 0;
 968       case T_BYTE:
 969       case T_SHORT:
 970       case T_INT:
 971         vopc = Op_AndReductionV;
 972         break;
 973       default: ShouldNotReachHere(); return 0;
 974       }
 975       break;
 976     case Op_AndL:
 977       assert(bt == T_LONG, "must be");
 978       vopc = Op_AndReductionV;
 979       break;
 980     case Op_OrI:
 981       switch(bt) {
 982       case T_BOOLEAN:
 983       case T_CHAR: return 0;
 984       case T_BYTE:
 985       case T_SHORT:
 986       case T_INT:
 987         vopc = Op_OrReductionV;
 988         break;
 989       default: ShouldNotReachHere(); return 0;
 990       }
 991       break;
 992     case Op_OrL:
 993       assert(bt == T_LONG, "must be");
 994       vopc = Op_OrReductionV;
 995       break;
 996     case Op_XorI:
 997       switch(bt) {
 998       case T_BOOLEAN:
 999       case T_CHAR: return 0;
1000       case T_BYTE:
1001       case T_SHORT:
1002       case T_INT:
1003         vopc = Op_XorReductionV;
1004         break;
1005       default: ShouldNotReachHere(); return 0;
1006       }
1007       break;
1008     case Op_XorL:
1009       assert(bt == T_LONG, "must be");
1010       vopc = Op_XorReductionV;
1011       break;
1012     default:
1013       break;
1014   }
1015   return vopc;
1016 }
1017 
1018 // Return the appropriate reduction node.
1019 ReductionNode* ReductionNode::make(int opc, Node *ctrl, Node* n1, Node* n2, BasicType bt) {
1020 
1021   int vopc = opcode(opc, bt);
1022 
1023   // This method should not be called for unimplemented vectors.
1024   guarantee(vopc != opc, "Vector for '%s' is not implemented", NodeClassNames[opc]);
1025 
1026   switch (vopc) {
1027   case Op_AddReductionVI: return new AddReductionVINode(ctrl, n1, n2);
1028   case Op_AddReductionVL: return new AddReductionVLNode(ctrl, n1, n2);
1029   case Op_AddReductionVF: return new AddReductionVFNode(ctrl, n1, n2);
1030   case Op_AddReductionVD: return new AddReductionVDNode(ctrl, n1, n2);
1031   case Op_MulReductionVI: return new MulReductionVINode(ctrl, n1, n2);
1032   case Op_MulReductionVL: return new MulReductionVLNode(ctrl, n1, n2);
1033   case Op_MulReductionVF: return new MulReductionVFNode(ctrl, n1, n2);
1034   case Op_MulReductionVD: return new MulReductionVDNode(ctrl, n1, n2);
1035   case Op_MinReductionV:  return new MinReductionVNode(ctrl, n1, n2);
1036   case Op_MaxReductionV:  return new MaxReductionVNode(ctrl, n1, n2);
1037   case Op_AndReductionV:  return new AndReductionVNode(ctrl, n1, n2);
1038   case Op_OrReductionV:   return new OrReductionVNode(ctrl, n1, n2);
1039   case Op_XorReductionV:  return new XorReductionVNode(ctrl, n1, n2);
1040   default:
1041     assert(false, "unknown node: %s", NodeClassNames[vopc]);
1042     return NULL;
1043   }
1044 }
1045 
1046 Node* VectorLoadMaskNode::Identity(PhaseGVN* phase) {
1047   BasicType out_bt = type()->is_vect()->element_basic_type();
1048   if (!Matcher::has_predicated_vectors() && out_bt == T_BOOLEAN) {
1049     return in(1); // redundant conversion
1050   }
1051 
1052   return this;
1053 }
1054 
1055 Node* VectorStoreMaskNode::Identity(PhaseGVN* phase) {
1056   // Identity transformation on boolean vectors.
1057   //   VectorStoreMask (VectorLoadMask bv) elem_size ==> bv
1058   //   vector[n]{bool} => vector[n]{t} => vector[n]{bool}
1059   if (in(1)->Opcode() == Op_VectorLoadMask) {
1060     return in(1)->in(1);
1061   }
1062   return this;
1063 }
1064 
1065 VectorStoreMaskNode* VectorStoreMaskNode::make(PhaseGVN& gvn, Node* in, BasicType in_type, uint num_elem) {
1066   assert(in->bottom_type()->isa_vect(), "sanity");
1067   const TypeVect* vt = TypeVect::make(T_BOOLEAN, num_elem);
1068   int elem_size = type2aelembytes(in_type);
1069   return new VectorStoreMaskNode(in, gvn.intcon(elem_size), vt);
1070 }
1071 
1072 VectorCastNode* VectorCastNode::make(int vopc, Node* n1, BasicType bt, uint vlen) {
1073   const TypeVect* vt = TypeVect::make(bt, vlen);
1074   switch (vopc) {
1075     case Op_VectorCastB2X: return new VectorCastB2XNode(n1, vt);
1076     case Op_VectorCastS2X: return new VectorCastS2XNode(n1, vt);
1077     case Op_VectorCastI2X: return new VectorCastI2XNode(n1, vt);
1078     case Op_VectorCastL2X: return new VectorCastL2XNode(n1, vt);
1079     case Op_VectorCastF2X: return new VectorCastF2XNode(n1, vt);
1080     case Op_VectorCastD2X: return new VectorCastD2XNode(n1, vt);
1081     default:
1082       assert(false, "unknown node: %s", NodeClassNames[vopc]);
1083       return NULL;
1084   }
1085 }
1086 
1087 int VectorCastNode::opcode(BasicType bt) {
1088   switch (bt) {
1089     case T_BYTE:   return Op_VectorCastB2X;
1090     case T_SHORT:  return Op_VectorCastS2X;
1091     case T_INT:    return Op_VectorCastI2X;
1092     case T_LONG:   return Op_VectorCastL2X;
1093     case T_FLOAT:  return Op_VectorCastF2X;
1094     case T_DOUBLE: return Op_VectorCastD2X;
1095     default:
1096       assert(false, "unknown type: %s", type2name(bt));
1097       return 0;
1098   }
1099 }
1100 
1101 Node* VectorCastNode::Identity(PhaseGVN* phase) {
1102   if (!in(1)->is_top()) {
1103     BasicType  in_bt = in(1)->bottom_type()->is_vect()->element_basic_type();
1104     BasicType out_bt = vect_type()->element_basic_type();
1105     if (in_bt == out_bt) {
1106       return in(1); // redundant cast
1107     }
1108   }
1109   return this;
1110 }
1111 
1112 Node* ReductionNode::make_reduction_input(PhaseGVN& gvn, int opc, BasicType bt) {
1113   int vopc = opcode(opc, bt);
1114   guarantee(vopc != opc, "Vector reduction for '%s' is not implemented", NodeClassNames[opc]);
1115 
1116   switch (vopc) {
1117     case Op_AndReductionV:
1118       switch (bt) {
1119         case T_BYTE:
1120         case T_SHORT:
1121         case T_INT:
1122           return gvn.makecon(TypeInt::MINUS_1);
1123         case T_LONG:
1124           return gvn.makecon(TypeLong::MINUS_1);
1125         default:
1126           fatal("Missed vector creation for '%s' as the basic type is not correct.", NodeClassNames[vopc]);
1127           return NULL;
1128       }
1129       break;
1130     case Op_AddReductionVI: // fallthrough
1131     case Op_AddReductionVL: // fallthrough
1132     case Op_AddReductionVF: // fallthrough
1133     case Op_AddReductionVD:
1134     case Op_OrReductionV:
1135     case Op_XorReductionV:
1136       return gvn.zerocon(bt);
1137     case Op_MulReductionVI:
1138       return gvn.makecon(TypeInt::ONE);
1139     case Op_MulReductionVL:
1140       return gvn.makecon(TypeLong::ONE);
1141     case Op_MulReductionVF:
1142       return gvn.makecon(TypeF::ONE);
1143     case Op_MulReductionVD:
1144       return gvn.makecon(TypeD::ONE);
1145     case Op_MinReductionV:
1146       switch (bt) {
1147         case T_BYTE:
1148           return gvn.makecon(TypeInt::make(max_jbyte));
1149         case T_SHORT:
1150           return gvn.makecon(TypeInt::make(max_jshort));
1151         case T_INT:
1152           return gvn.makecon(TypeInt::MAX);
1153         case T_LONG:
1154           return gvn.makecon(TypeLong::MAX);
1155         case T_FLOAT:
1156           return gvn.makecon(TypeF::POS_INF);
1157         case T_DOUBLE:
1158           return gvn.makecon(TypeD::POS_INF);
1159           default: Unimplemented(); return NULL;
1160       }
1161       break;
1162     case Op_MaxReductionV:
1163       switch (bt) {
1164         case T_BYTE:
1165           return gvn.makecon(TypeInt::make(min_jbyte));
1166         case T_SHORT:
1167           return gvn.makecon(TypeInt::make(min_jshort));
1168         case T_INT:
1169           return gvn.makecon(TypeInt::MIN);
1170         case T_LONG:
1171           return gvn.makecon(TypeLong::MIN);
1172         case T_FLOAT:
1173           return gvn.makecon(TypeF::NEG_INF);
1174         case T_DOUBLE:
1175           return gvn.makecon(TypeD::NEG_INF);
1176           default: Unimplemented(); return NULL;
1177       }
1178       break;
1179     default:
1180       fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
1181       return NULL;
1182   }
1183 }
1184 
1185 bool ReductionNode::implemented(int opc, uint vlen, BasicType bt) {
1186   if (is_java_primitive(bt) &&
1187       (vlen > 1) && is_power_of_2(vlen) &&
1188       Matcher::vector_size_supported(bt, vlen)) {
1189     int vopc = ReductionNode::opcode(opc, bt);
1190     return vopc != opc && Matcher::match_rule_supported_vector(vopc, vlen, bt);
1191   }
1192   return false;
1193 }
1194 
1195 MacroLogicVNode* MacroLogicVNode::make(PhaseGVN& gvn, Node* in1, Node* in2, Node* in3,
1196                                        uint truth_table, const TypeVect* vt) {
1197   assert(truth_table <= 0xFF, "invalid");
1198   assert(in1->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
1199   assert(in2->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
1200   assert(in3->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
1201   Node* fn = gvn.intcon(truth_table);
1202   return new MacroLogicVNode(in1, in2, in3, fn, vt);
1203 }
1204 
1205 Node* VectorNode::degenerate_vector_rotate(Node* src, Node* cnt, bool is_rotate_left,
1206                                            int vlen, BasicType bt, PhaseGVN* phase) {
1207   assert(is_integral_type(bt), "sanity");
1208   const TypeVect* vt = TypeVect::make(bt, vlen);
1209 
1210   int shift_mask = (type2aelembytes(bt) * 8) - 1;
1211   int shiftLOpc = (bt == T_LONG) ? Op_LShiftL : Op_LShiftI;
1212   auto urshiftopc = [=]() {
1213     switch(bt) {
1214       case T_INT: return Op_URShiftI;
1215       case T_LONG: return Op_URShiftL;
1216       case T_BYTE: return Op_URShiftB;
1217       case T_SHORT: return Op_URShiftS;
1218       default: return (Opcodes)0;
1219     }
1220   };
1221   int shiftROpc = urshiftopc();
1222 
1223   // Compute shift values for right rotation and
1224   // later swap them in case of left rotation.
1225   Node* shiftRCnt = NULL;
1226   Node* shiftLCnt = NULL;
1227   const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
1228   bool is_binary_vector_op = false;
1229   if (cnt_type && cnt_type->is_con()) {
1230     // Constant shift.
1231     int shift = cnt_type->get_con() & shift_mask;
1232     shiftRCnt = phase->intcon(shift);
1233     shiftLCnt = phase->intcon(shift_mask + 1 - shift);
1234   } else if (VectorNode::is_invariant_vector(cnt)) {
1235     // Scalar variable shift, handle replicates generated by auto vectorizer.
1236     cnt = cnt->in(1);
1237     if (bt == T_LONG) {
1238       // Shift count vector for Rotate vector has long elements too.
1239       if (cnt->Opcode() == Op_ConvI2L) {
1240          cnt = cnt->in(1);
1241       } else {
1242          assert(cnt->bottom_type()->isa_long() &&
1243                 cnt->bottom_type()->is_long()->is_con(), "Long constant expected");
1244          cnt = phase->transform(new ConvL2INode(cnt));
1245       }
1246     }
1247     shiftRCnt = phase->transform(new AndINode(cnt, phase->intcon(shift_mask)));
1248     shiftLCnt = phase->transform(new SubINode(phase->intcon(shift_mask + 1), shiftRCnt));
1249   } else {
1250     // Variable vector rotate count.
1251     assert(Matcher::supports_vector_variable_shifts(), "");
1252 
1253     int subVopc = 0;
1254     int addVopc = 0;
1255     Node* shift_mask_node = NULL;
1256     Node* const_one_node = NULL;
1257 
1258     assert(cnt->bottom_type()->isa_vect(), "Unexpected shift");
1259     const Type* elem_ty = Type::get_const_basic_type(bt);
1260 
1261     if (bt == T_LONG) {
1262       shift_mask_node = phase->longcon(shift_mask);
1263       const_one_node = phase->longcon(1L);
1264       subVopc = VectorNode::opcode(Op_SubL, bt);
1265       addVopc = VectorNode::opcode(Op_AddL, bt);
1266     } else {
1267       shift_mask_node = phase->intcon(shift_mask);
1268       const_one_node = phase->intcon(1);
1269       subVopc = VectorNode::opcode(Op_SubI, bt);
1270       addVopc = VectorNode::opcode(Op_AddI, bt);
1271     }
1272     Node* vector_mask = phase->transform(VectorNode::scalar2vector(shift_mask_node, vlen, elem_ty));
1273     Node* vector_one = phase->transform(VectorNode::scalar2vector(const_one_node, vlen, elem_ty));
1274 
1275     shiftRCnt = cnt;
1276     shiftRCnt = phase->transform(VectorNode::make(Op_AndV, shiftRCnt, vector_mask, vt));
1277     vector_mask = phase->transform(VectorNode::make(addVopc, vector_one, vector_mask, vt));
1278     shiftLCnt = phase->transform(VectorNode::make(subVopc, vector_mask, shiftRCnt, vt));
1279     is_binary_vector_op = true;
1280   }
1281 
1282   // Swap the computed left and right shift counts.
1283   if (is_rotate_left) {
1284     swap(shiftRCnt,shiftLCnt);
1285   }
1286 
1287   if (!is_binary_vector_op) {
1288     shiftLCnt = phase->transform(new LShiftCntVNode(shiftLCnt, vt));
1289     shiftRCnt = phase->transform(new RShiftCntVNode(shiftRCnt, vt));
1290   }
1291 
1292   return new OrVNode(phase->transform(VectorNode::make(shiftLOpc, src, shiftLCnt, vlen, bt)),
1293                      phase->transform(VectorNode::make(shiftROpc, src, shiftRCnt, vlen, bt)),
1294                      vt);
1295 }
1296 
1297 Node* RotateLeftVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1298   int vlen = length();
1299   BasicType bt = vect_type()->element_basic_type();
1300   if ((!in(2)->is_Con() && !Matcher::supports_vector_variable_rotates()) ||
1301        !Matcher::match_rule_supported_vector(Op_RotateLeftV, vlen, bt)) {
1302     return VectorNode::degenerate_vector_rotate(in(1), in(2), true, vlen, bt, phase);
1303   }
1304   return NULL;
1305 }
1306 
1307 Node* RotateRightVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1308   int vlen = length();
1309   BasicType bt = vect_type()->element_basic_type();
1310   if ((!in(2)->is_Con() && !Matcher::supports_vector_variable_rotates()) ||
1311        !Matcher::match_rule_supported_vector(Op_RotateRightV, vlen, bt)) {
1312     return VectorNode::degenerate_vector_rotate(in(1), in(2), false, vlen, bt, phase);
1313   }
1314   return NULL;
1315 }
1316 
1317 #ifndef PRODUCT
1318 void VectorMaskCmpNode::dump_spec(outputStream *st) const {
1319   st->print(" %d #", _predicate); _type->dump_on(st);
1320 }
1321 #endif // PRODUCT
1322 
1323 Node* VectorReinterpretNode::Identity(PhaseGVN *phase) {
1324   Node* n = in(1);
1325   if (n->Opcode() == Op_VectorReinterpret) {
1326     // "VectorReinterpret (VectorReinterpret node) ==> node" if:
1327     //   1) Types of 'node' and 'this' are identical
1328     //   2) Truncations are not introduced by the first VectorReinterpret
1329     if (Type::cmp(bottom_type(), n->in(1)->bottom_type()) == 0 &&
1330         length_in_bytes() <= n->bottom_type()->is_vect()->length_in_bytes()) {
1331       return n->in(1);
1332     }
1333   }
1334   return this;
1335 }
1336 
1337 Node* VectorInsertNode::make(Node* vec, Node* new_val, int position) {
1338   assert(position < (int)vec->bottom_type()->is_vect()->length(), "pos in range");
1339   ConINode* pos = ConINode::make(position);
1340   return new VectorInsertNode(vec, new_val, pos, vec->bottom_type()->is_vect());
1341 }
1342 
1343 Node* VectorUnboxNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1344   Node* n = obj()->uncast();
1345   if (EnableVectorReboxing && n->Opcode() == Op_VectorBox) {
1346     if (Type::cmp(bottom_type(), n->in(VectorBoxNode::Value)->bottom_type()) == 0) {
1347       // Handled by VectorUnboxNode::Identity()
1348     } else {
1349       VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
1350       ciKlass* vbox_klass = vbox->box_type()->klass();
1351       const TypeVect* in_vt = vbox->vec_type();
1352       const TypeVect* out_vt = type()->is_vect();
1353 
1354       if (in_vt->length() == out_vt->length()) {
1355         Node* value = vbox->in(VectorBoxNode::Value);
1356 
1357         bool is_vector_mask    = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
1358         bool is_vector_shuffle = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
1359         if (is_vector_mask) {
1360           const TypeVect* vmask_type = TypeVect::makemask(out_vt->element_basic_type(), out_vt->length());
1361           if (in_vt->length_in_bytes() == out_vt->length_in_bytes() &&
1362               Matcher::match_rule_supported_vector(Op_VectorMaskCast, out_vt->length(), out_vt->element_basic_type())) {
1363             // Apply "VectorUnbox (VectorBox vmask) ==> VectorMaskCast (vmask)"
1364             // directly. This could avoid the transformation ordering issue from
1365             // "VectorStoreMask (VectorLoadMask vmask) => vmask".
1366             return new VectorMaskCastNode(value, vmask_type);
1367           }
1368           // VectorUnbox (VectorBox vmask) ==> VectorLoadMask (VectorStoreMask vmask)
1369           value = phase->transform(VectorStoreMaskNode::make(*phase, value, in_vt->element_basic_type(), in_vt->length()));
1370           return new VectorLoadMaskNode(value, vmask_type);
1371         } else if (is_vector_shuffle) {
1372           if (!is_shuffle_to_vector()) {
1373             // VectorUnbox (VectorBox vshuffle) ==> VectorLoadShuffle vshuffle
1374             return new VectorLoadShuffleNode(value, out_vt);
1375           }
1376         } else {
1377           // Vector type mismatch is only supported for masks and shuffles, but sometimes it happens in pathological cases.
1378         }
1379       } else {
1380         // Vector length mismatch.
1381         // Sometimes happen in pathological cases (e.g., when unboxing happens in effectively dead code).
1382       }
1383     }
1384   }
1385   return NULL;
1386 }
1387 
1388 Node* VectorUnboxNode::Identity(PhaseGVN* phase) {
1389   Node* n = obj()->uncast();
1390   if (EnableVectorReboxing && n->Opcode() == Op_VectorBox) {
1391     if (Type::cmp(bottom_type(), n->in(VectorBoxNode::Value)->bottom_type()) == 0) {
1392       return n->in(VectorBoxNode::Value); // VectorUnbox (VectorBox v) ==> v
1393     } else {
1394       // Handled by VectorUnboxNode::Ideal().
1395     }
1396   }
1397   return this;
1398 }
1399 
1400 const TypeFunc* VectorBoxNode::vec_box_type(const TypeInstPtr* box_type) {
1401   const Type** fields = TypeTuple::fields(0);
1402   const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms, fields);
1403 
1404   fields = TypeTuple::fields(1);
1405   fields[TypeFunc::Parms+0] = box_type;
1406   const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
1407 
1408   return TypeFunc::make(domain, range);
1409 }
1410 
1411 Node* ShiftVNode::Identity(PhaseGVN* phase) {
1412   Node* in2 = in(2);
1413   // Shift by ZERO does nothing
1414   if (is_vshift_cnt(in2) && phase->find_int_type(in2->in(1)) == TypeInt::ZERO) {
1415     return in(1);
1416   }
1417   return this;
1418 }
1419 
1420 Node* VectorMaskOpNode::make(Node* mask, const Type* ty, int mopc) {
1421   switch(mopc) {
1422     case Op_VectorMaskTrueCount:
1423       return new VectorMaskTrueCountNode(mask, ty);
1424     case Op_VectorMaskLastTrue:
1425       return new VectorMaskLastTrueNode(mask, ty);
1426     case Op_VectorMaskFirstTrue:
1427       return new VectorMaskFirstTrueNode(mask, ty);
1428     case Op_VectorMaskToLong:
1429       return new VectorMaskToLongNode(mask, ty);
1430     default:
1431       assert(false, "Unhandled operation");
1432   }
1433   return NULL;
1434 }
1435 
1436 #ifndef PRODUCT
1437 void VectorBoxAllocateNode::dump_spec(outputStream *st) const {
1438   CallStaticJavaNode::dump_spec(st);
1439 }
1440 
1441 #endif // !PRODUCT