1 /*
   2  * Copyright (c) 2007, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 #include "precompiled.hpp"
  25 #include "memory/allocation.inline.hpp"
  26 #include "opto/connode.hpp"
  27 #include "opto/mulnode.hpp"
  28 #include "opto/subnode.hpp"
  29 #include "opto/vectornode.hpp"
  30 #include "opto/convertnode.hpp"
  31 #include "utilities/powerOfTwo.hpp"
  32 #include "utilities/globalDefinitions.hpp"
  33 
  34 //------------------------------VectorNode--------------------------------------
  35 
  36 // Return the vector operator for the specified scalar operation
  37 // and vector length.
  38 int VectorNode::opcode(int sopc, BasicType bt) {
  39   switch (sopc) {
  40   case Op_AddI:
  41     switch (bt) {
  42     case T_BOOLEAN:
  43     case T_BYTE:      return Op_AddVB;
  44     case T_CHAR:
  45     case T_SHORT:     return Op_AddVS;
  46     case T_INT:       return Op_AddVI;
  47     default:          return 0;
  48     }
  49   case Op_AddL: return (bt == T_LONG   ? Op_AddVL : 0);
  50   case Op_AddF: return (bt == T_FLOAT  ? Op_AddVF : 0);
  51   case Op_AddD: return (bt == T_DOUBLE ? Op_AddVD : 0);
  52 
  53   case Op_SubI:
  54     switch (bt) {
  55     case T_BOOLEAN:
  56     case T_BYTE:   return Op_SubVB;
  57     case T_CHAR:
  58     case T_SHORT:  return Op_SubVS;
  59     case T_INT:    return Op_SubVI;
  60     default:       return 0;
  61     }
  62   case Op_SubL: return (bt == T_LONG   ? Op_SubVL : 0);
  63   case Op_SubF: return (bt == T_FLOAT  ? Op_SubVF : 0);
  64   case Op_SubD: return (bt == T_DOUBLE ? Op_SubVD : 0);
  65 
  66   case Op_MulI:
  67     switch (bt) {
  68     case T_BOOLEAN:return 0;
  69     case T_BYTE:   return Op_MulVB;
  70     case T_CHAR:
  71     case T_SHORT:  return Op_MulVS;
  72     case T_INT:    return Op_MulVI;
  73     default:       return 0;
  74     }
  75   case Op_MulL: return (bt == T_LONG ? Op_MulVL : 0);
  76   case Op_MulF:
  77     return (bt == T_FLOAT ? Op_MulVF : 0);
  78   case Op_MulD:
  79     return (bt == T_DOUBLE ? Op_MulVD : 0);
  80   case Op_FmaD:
  81     return (bt == T_DOUBLE ? Op_FmaVD : 0);
  82   case Op_FmaF:
  83     return (bt == T_FLOAT ? Op_FmaVF : 0);
  84   case Op_CMoveF:
  85     return (bt == T_FLOAT ? Op_CMoveVF : 0);
  86   case Op_CMoveD:
  87     return (bt == T_DOUBLE ? Op_CMoveVD : 0);
  88   case Op_DivF:
  89     return (bt == T_FLOAT ? Op_DivVF : 0);
  90   case Op_DivD:
  91     return (bt == T_DOUBLE ? Op_DivVD : 0);
  92   case Op_AbsI:
  93     switch (bt) {
  94     case T_BOOLEAN:
  95     case T_CHAR:  return 0; // abs does not make sense for unsigned
  96     case T_BYTE:  return Op_AbsVB;
  97     case T_SHORT: return Op_AbsVS;
  98     case T_INT:   return Op_AbsVI;
  99     default:      return 0;
 100     }
 101   case Op_AbsL:
 102     return (bt == T_LONG ? Op_AbsVL : 0);
 103   case Op_MinI:
 104     switch (bt) {
 105     case T_BOOLEAN:
 106     case T_CHAR:   return 0;
 107     case T_BYTE:
 108     case T_SHORT:
 109     case T_INT:    return Op_MinV;
 110     default:       return 0;
 111     }
 112   case Op_MinL:
 113     return (bt == T_LONG ? Op_MinV : 0);
 114   case Op_MinF:
 115     return (bt == T_FLOAT ? Op_MinV : 0);
 116   case Op_MinD:
 117     return (bt == T_DOUBLE ? Op_MinV : 0);
 118   case Op_MaxI:
 119     switch (bt) {
 120     case T_BOOLEAN:
 121     case T_CHAR:   return 0;
 122     case T_BYTE:
 123     case T_SHORT:
 124     case T_INT:    return Op_MaxV;
 125     default:       return 0;
 126     }
 127   case Op_MaxL:
 128     return (bt == T_LONG ? Op_MaxV : 0);
 129   case Op_MaxF:
 130     return (bt == T_FLOAT ? Op_MaxV : 0);
 131   case Op_MaxD:
 132     return (bt == T_DOUBLE ? Op_MaxV : 0);
 133   case Op_AbsF:
 134     return (bt == T_FLOAT ? Op_AbsVF : 0);
 135   case Op_AbsD:
 136     return (bt == T_DOUBLE ? Op_AbsVD : 0);
 137   case Op_NegI:
 138     return (bt == T_INT ? Op_NegVI : 0);
 139   case Op_NegF:
 140     return (bt == T_FLOAT ? Op_NegVF : 0);
 141   case Op_NegD:
 142     return (bt == T_DOUBLE ? Op_NegVD : 0);
 143   case Op_RoundDoubleMode:
 144     return (bt == T_DOUBLE ? Op_RoundDoubleModeV : 0);
 145   case Op_RotateLeft:
 146     return (is_integral_type(bt) ? Op_RotateLeftV : 0);
 147   case Op_RotateRight:
 148     return (is_integral_type(bt) ? Op_RotateRightV : 0);
 149   case Op_SqrtF:
 150     return (bt == T_FLOAT ? Op_SqrtVF : 0);
 151   case Op_SqrtD:
 152     return (bt == T_DOUBLE ? Op_SqrtVD : 0);
 153   case Op_PopCountI:
 154     // Unimplemented for subword types since bit count changes
 155     // depending on size of lane (and sign bit).
 156     return (bt == T_INT ? Op_PopCountVI : 0);
 157   case Op_LShiftI:
 158     switch (bt) {
 159     case T_BOOLEAN:
 160     case T_BYTE:   return Op_LShiftVB;
 161     case T_CHAR:
 162     case T_SHORT:  return Op_LShiftVS;
 163     case T_INT:    return Op_LShiftVI;
 164     default:       return 0;
 165     }
 166   case Op_LShiftL:
 167     return (bt == T_LONG ? Op_LShiftVL : 0);
 168   case Op_RShiftI:
 169     switch (bt) {
 170     case T_BOOLEAN:return Op_URShiftVB; // boolean is unsigned value
 171     case T_CHAR:   return Op_URShiftVS; // char is unsigned value
 172     case T_BYTE:   return Op_RShiftVB;
 173     case T_SHORT:  return Op_RShiftVS;
 174     case T_INT:    return Op_RShiftVI;
 175     default:       return 0;
 176     }
 177   case Op_RShiftL:
 178     return (bt == T_LONG ? Op_RShiftVL : 0);
 179   case Op_URShiftB:
 180     return (bt == T_BYTE ? Op_URShiftVB : 0);
 181   case Op_URShiftS:
 182     return (bt == T_SHORT ? Op_URShiftVS : 0);
 183   case Op_URShiftI:
 184     switch (bt) {
 185     case T_BOOLEAN:return Op_URShiftVB;
 186     case T_CHAR:   return Op_URShiftVS;
 187     case T_BYTE:
 188     case T_SHORT:  return 0; // Vector logical right shift for signed short
 189                              // values produces incorrect Java result for
 190                              // negative data because java code should convert
 191                              // a short value into int value with sign
 192                              // extension before a shift.
 193     case T_INT:    return Op_URShiftVI;
 194     default:       return 0;
 195     }
 196   case Op_URShiftL:
 197     return (bt == T_LONG ? Op_URShiftVL : 0);
 198   case Op_AndI:
 199   case Op_AndL:
 200     return Op_AndV;
 201   case Op_OrI:
 202   case Op_OrL:
 203     return Op_OrV;
 204   case Op_XorI:
 205   case Op_XorL:
 206     return Op_XorV;
 207 
 208   case Op_LoadB:
 209   case Op_LoadUB:
 210   case Op_LoadUS:
 211   case Op_LoadS:
 212   case Op_LoadI:
 213   case Op_LoadL:
 214   case Op_LoadF:
 215   case Op_LoadD:
 216     return Op_LoadVector;
 217 
 218   case Op_StoreB:
 219   case Op_StoreC:
 220   case Op_StoreI:
 221   case Op_StoreL:
 222   case Op_StoreF:
 223   case Op_StoreD:
 224     return Op_StoreVector;
 225   case Op_MulAddS2I:
 226     return Op_MulAddVS2VI;
 227 
 228   default:
 229     return 0; // Unimplemented
 230   }
 231 }
 232 
 233 int VectorNode::replicate_opcode(BasicType bt) {
 234   switch(bt) {
 235     case T_BOOLEAN:
 236     case T_BYTE:
 237       return Op_ReplicateB;
 238     case T_SHORT:
 239     case T_CHAR:
 240       return Op_ReplicateS;
 241     case T_INT:
 242       return Op_ReplicateI;
 243     case T_LONG:
 244       return Op_ReplicateL;
 245     case T_FLOAT:
 246       return Op_ReplicateF;
 247     case T_DOUBLE:
 248       return Op_ReplicateD;
 249     default:
 250       assert(false, "wrong type: %s", type2name(bt));
 251       return 0;
 252   }
 253 }
 254 
 255 // Also used to check if the code generator
 256 // supports the vector operation.
 257 bool VectorNode::implemented(int opc, uint vlen, BasicType bt) {
 258   if (is_java_primitive(bt) &&
 259       (vlen > 1) && is_power_of_2(vlen) &&
 260       Matcher::vector_size_supported(bt, vlen)) {
 261     int vopc = VectorNode::opcode(opc, bt);
 262     // For rotate operation we will do a lazy de-generation into
 263     // OrV/LShiftV/URShiftV pattern if the target does not support
 264     // vector rotation instruction.
 265     if (VectorNode::is_vector_rotate(vopc)) {
 266       return is_vector_rotate_supported(vopc, vlen, bt);
 267     }
 268     return vopc > 0 && Matcher::match_rule_supported_vector(vopc, vlen, bt);
 269   }
 270   return false;
 271 }
 272 
 273 bool VectorNode::is_type_transition_short_to_int(Node* n) {
 274   switch (n->Opcode()) {
 275   case Op_MulAddS2I:
 276     return true;
 277   }
 278   return false;
 279 }
 280 
 281 bool VectorNode::is_type_transition_to_int(Node* n) {
 282   return is_type_transition_short_to_int(n);
 283 }
 284 
 285 bool VectorNode::is_muladds2i(Node* n) {
 286   if (n->Opcode() == Op_MulAddS2I) {
 287     return true;
 288   }
 289   return false;
 290 }
 291 
 292 bool VectorNode::is_roundopD(Node* n) {
 293   if (n->Opcode() == Op_RoundDoubleMode) {
 294     return true;
 295   }
 296   return false;
 297 }
 298 
 299 bool VectorNode::is_vector_rotate_supported(int vopc, uint vlen, BasicType bt) {
 300   assert(VectorNode::is_vector_rotate(vopc), "wrong opcode");
 301 
 302   // If target defines vector rotation patterns then no
 303   // need for degeneration.
 304   if (Matcher::match_rule_supported_vector(vopc, vlen, bt)) {
 305     return true;
 306   }
 307 
 308   // If target does not support variable shift operations then no point
 309   // in creating a rotate vector node since it will not be disintegratable.
 310   // Adding a pessimistic check to avoid complex pattern mathing which
 311   // may not be full proof.
 312   if (!Matcher::supports_vector_variable_shifts()) {
 313      return false;
 314   }
 315 
 316   // Validate existence of nodes created in case of rotate degeneration.
 317   switch (bt) {
 318     case T_INT:
 319       return Matcher::match_rule_supported_vector(Op_OrV,       vlen, bt) &&
 320              Matcher::match_rule_supported_vector(Op_LShiftVI,  vlen, bt) &&
 321              Matcher::match_rule_supported_vector(Op_URShiftVI, vlen, bt);
 322     case T_LONG:
 323       return Matcher::match_rule_supported_vector(Op_OrV,       vlen, bt) &&
 324              Matcher::match_rule_supported_vector(Op_LShiftVL,  vlen, bt) &&
 325              Matcher::match_rule_supported_vector(Op_URShiftVL, vlen, bt);
 326     default:
 327       assert(false, "not supported: %s", type2name(bt));
 328       return false;
 329   }
 330 }
 331 
 332 bool VectorNode::is_shift_opcode(int opc) {
 333   switch (opc) {
 334   case Op_LShiftI:
 335   case Op_LShiftL:
 336   case Op_RShiftI:
 337   case Op_RShiftL:
 338   case Op_URShiftB:
 339   case Op_URShiftS:
 340   case Op_URShiftI:
 341   case Op_URShiftL:
 342     return true;
 343   default:
 344     return false;
 345   }
 346 }
 347 
 348 bool VectorNode::is_shift(Node* n) {
 349   return is_shift_opcode(n->Opcode());
 350 }
 351 
 352 bool VectorNode::is_rotate_opcode(int opc) {
 353   switch (opc) {
 354   case Op_RotateRight:
 355   case Op_RotateLeft:
 356     return true;
 357   default:
 358     return false;
 359   }
 360 }
 361 
 362 bool VectorNode::is_scalar_rotate(Node* n) {
 363   if (is_rotate_opcode(n->Opcode())) {
 364     return true;
 365   }
 366   return false;
 367 }
 368 
 369 bool VectorNode::is_vshift_cnt(Node* n) {
 370   switch (n->Opcode()) {
 371   case Op_LShiftCntV:
 372   case Op_RShiftCntV:
 373     return true;
 374   default:
 375     return false;
 376   }
 377 }
 378 
 379 // Check if input is loop invariant vector.
 380 bool VectorNode::is_invariant_vector(Node* n) {
 381   // Only Replicate vector nodes are loop invariant for now.
 382   switch (n->Opcode()) {
 383   case Op_ReplicateB:
 384   case Op_ReplicateS:
 385   case Op_ReplicateI:
 386   case Op_ReplicateL:
 387   case Op_ReplicateF:
 388   case Op_ReplicateD:
 389     return true;
 390   default:
 391     return false;
 392   }
 393 }
 394 
 395 // [Start, end) half-open range defining which operands are vectors
 396 void VectorNode::vector_operands(Node* n, uint* start, uint* end) {
 397   switch (n->Opcode()) {
 398   case Op_LoadB:   case Op_LoadUB:
 399   case Op_LoadS:   case Op_LoadUS:
 400   case Op_LoadI:   case Op_LoadL:
 401   case Op_LoadF:   case Op_LoadD:
 402   case Op_LoadP:   case Op_LoadN:
 403     *start = 0;
 404     *end   = 0; // no vector operands
 405     break;
 406   case Op_StoreB:  case Op_StoreC:
 407   case Op_StoreI:  case Op_StoreL:
 408   case Op_StoreF:  case Op_StoreD:
 409   case Op_StoreP:  case Op_StoreN:
 410     *start = MemNode::ValueIn;
 411     *end   = MemNode::ValueIn + 1; // 1 vector operand
 412     break;
 413   case Op_LShiftI:  case Op_LShiftL:
 414   case Op_RShiftI:  case Op_RShiftL:
 415   case Op_URShiftI: case Op_URShiftL:
 416     *start = 1;
 417     *end   = 2; // 1 vector operand
 418     break;
 419   case Op_AddI: case Op_AddL: case Op_AddF: case Op_AddD:
 420   case Op_SubI: case Op_SubL: case Op_SubF: case Op_SubD:
 421   case Op_MulI: case Op_MulL: case Op_MulF: case Op_MulD:
 422   case Op_DivF: case Op_DivD:
 423   case Op_AndI: case Op_AndL:
 424   case Op_OrI:  case Op_OrL:
 425   case Op_XorI: case Op_XorL:
 426   case Op_MulAddS2I:
 427     *start = 1;
 428     *end   = 3; // 2 vector operands
 429     break;
 430   case Op_CMoveI:  case Op_CMoveL:  case Op_CMoveF:  case Op_CMoveD:
 431     *start = 2;
 432     *end   = n->req();
 433     break;
 434   case Op_FmaD:
 435   case Op_FmaF:
 436     *start = 1;
 437     *end   = 4; // 3 vector operands
 438     break;
 439   default:
 440     *start = 1;
 441     *end   = n->req(); // default is all operands
 442   }
 443 }
 444 
 445 // Make a vector node for binary operation
 446 VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, const TypeVect* vt) {
 447   // This method should not be called for unimplemented vectors.
 448   guarantee(vopc > 0, "vopc must be > 0");
 449   switch (vopc) {
 450   case Op_AddVB: return new AddVBNode(n1, n2, vt);
 451   case Op_AddVS: return new AddVSNode(n1, n2, vt);
 452   case Op_AddVI: return new AddVINode(n1, n2, vt);
 453   case Op_AddVL: return new AddVLNode(n1, n2, vt);
 454   case Op_AddVF: return new AddVFNode(n1, n2, vt);
 455   case Op_AddVD: return new AddVDNode(n1, n2, vt);
 456 
 457   case Op_SubVB: return new SubVBNode(n1, n2, vt);
 458   case Op_SubVS: return new SubVSNode(n1, n2, vt);
 459   case Op_SubVI: return new SubVINode(n1, n2, vt);
 460   case Op_SubVL: return new SubVLNode(n1, n2, vt);
 461   case Op_SubVF: return new SubVFNode(n1, n2, vt);
 462   case Op_SubVD: return new SubVDNode(n1, n2, vt);
 463 
 464   case Op_MulVB: return new MulVBNode(n1, n2, vt);
 465   case Op_MulVS: return new MulVSNode(n1, n2, vt);
 466   case Op_MulVI: return new MulVINode(n1, n2, vt);
 467   case Op_MulVL: return new MulVLNode(n1, n2, vt);
 468   case Op_MulVF: return new MulVFNode(n1, n2, vt);
 469   case Op_MulVD: return new MulVDNode(n1, n2, vt);
 470 
 471   case Op_DivVF: return new DivVFNode(n1, n2, vt);
 472   case Op_DivVD: return new DivVDNode(n1, n2, vt);
 473 
 474   case Op_MinV: return new MinVNode(n1, n2, vt);
 475   case Op_MaxV: return new MaxVNode(n1, n2, vt);
 476 
 477   case Op_AbsVF: return new AbsVFNode(n1, vt);
 478   case Op_AbsVD: return new AbsVDNode(n1, vt);
 479   case Op_AbsVB: return new AbsVBNode(n1, vt);
 480   case Op_AbsVS: return new AbsVSNode(n1, vt);
 481   case Op_AbsVI: return new AbsVINode(n1, vt);
 482   case Op_AbsVL: return new AbsVLNode(n1, vt);
 483 
 484   case Op_NegVI: return new NegVINode(n1, vt);
 485   case Op_NegVF: return new NegVFNode(n1, vt);
 486   case Op_NegVD: return new NegVDNode(n1, vt);
 487 
 488   case Op_SqrtVF: return new SqrtVFNode(n1, vt);
 489   case Op_SqrtVD: return new SqrtVDNode(n1, vt);
 490 
 491   case Op_PopCountVI: return new PopCountVINode(n1, vt);
 492   case Op_RotateLeftV: return new RotateLeftVNode(n1, n2, vt);
 493   case Op_RotateRightV: return new RotateRightVNode(n1, n2, vt);
 494 
 495   case Op_LShiftVB: return new LShiftVBNode(n1, n2, vt);
 496   case Op_LShiftVS: return new LShiftVSNode(n1, n2, vt);
 497   case Op_LShiftVI: return new LShiftVINode(n1, n2, vt);
 498   case Op_LShiftVL: return new LShiftVLNode(n1, n2, vt);
 499 
 500   case Op_RShiftVB: return new RShiftVBNode(n1, n2, vt);
 501   case Op_RShiftVS: return new RShiftVSNode(n1, n2, vt);
 502   case Op_RShiftVI: return new RShiftVINode(n1, n2, vt);
 503   case Op_RShiftVL: return new RShiftVLNode(n1, n2, vt);
 504 
 505   case Op_URShiftVB: return new URShiftVBNode(n1, n2, vt);
 506   case Op_URShiftVS: return new URShiftVSNode(n1, n2, vt);
 507   case Op_URShiftVI: return new URShiftVINode(n1, n2, vt);
 508   case Op_URShiftVL: return new URShiftVLNode(n1, n2, vt);
 509 
 510   case Op_AndV: return new AndVNode(n1, n2, vt);
 511   case Op_OrV:  return new OrVNode (n1, n2, vt);
 512   case Op_XorV: return new XorVNode(n1, n2, vt);
 513 
 514   case Op_RoundDoubleModeV: return new RoundDoubleModeVNode(n1, n2, vt);
 515 
 516   case Op_MulAddVS2VI: return new MulAddVS2VINode(n1, n2, vt);
 517   default:
 518     fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
 519     return NULL;
 520   }
 521 }
 522 
 523 // Return the vector version of a scalar binary operation node.
 524 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, uint vlen, BasicType bt) {
 525   const TypeVect* vt = TypeVect::make(bt, vlen);
 526   int vopc = VectorNode::opcode(opc, bt);
 527   // This method should not be called for unimplemented vectors.
 528   guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]);
 529   return make(vopc, n1, n2, vt);
 530 }
 531 
 532 // Make a vector node for ternary operation
 533 VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, Node* n3, const TypeVect* vt) {
 534   // This method should not be called for unimplemented vectors.
 535   guarantee(vopc > 0, "vopc must be > 0");
 536   switch (vopc) {
 537   case Op_FmaVD: return new FmaVDNode(n1, n2, n3, vt);
 538   case Op_FmaVF: return new FmaVFNode(n1, n2, n3, vt);
 539   default:
 540     fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
 541     return NULL;
 542   }
 543 }
 544 
 545 // Return the vector version of a scalar ternary operation node.
 546 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, Node* n3, uint vlen, BasicType bt) {
 547   const TypeVect* vt = TypeVect::make(bt, vlen);
 548   int vopc = VectorNode::opcode(opc, bt);
 549   // This method should not be called for unimplemented vectors.
 550   guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]);
 551   return make(vopc, n1, n2, n3, vt);
 552 }
 553 
 554 // Scalar promotion
 555 VectorNode* VectorNode::scalar2vector(Node* s, uint vlen, const Type* opd_t) {
 556   BasicType bt = opd_t->array_element_basic_type();
 557   const TypeVect* vt = opd_t->singleton() ? TypeVect::make(opd_t, vlen)
 558                                           : TypeVect::make(bt, vlen);
 559   switch (bt) {
 560   case T_BOOLEAN:
 561   case T_BYTE:
 562     return new ReplicateBNode(s, vt);
 563   case T_CHAR:
 564   case T_SHORT:
 565     return new ReplicateSNode(s, vt);
 566   case T_INT:
 567     return new ReplicateINode(s, vt);
 568   case T_LONG:
 569     return new ReplicateLNode(s, vt);
 570   case T_FLOAT:
 571     return new ReplicateFNode(s, vt);
 572   case T_DOUBLE:
 573     return new ReplicateDNode(s, vt);
 574   default:
 575     fatal("Type '%s' is not supported for vectors", type2name(bt));
 576     return NULL;
 577   }
 578 }
 579 
 580 VectorNode* VectorNode::shift_count(int opc, Node* cnt, uint vlen, BasicType bt) {
 581   // Match shift count type with shift vector type.
 582   const TypeVect* vt = TypeVect::make(bt, vlen);
 583   switch (opc) {
 584   case Op_LShiftI:
 585   case Op_LShiftL:
 586     return new LShiftCntVNode(cnt, vt);
 587   case Op_RShiftI:
 588   case Op_RShiftL:
 589   case Op_URShiftB:
 590   case Op_URShiftS:
 591   case Op_URShiftI:
 592   case Op_URShiftL:
 593     return new RShiftCntVNode(cnt, vt);
 594   default:
 595     fatal("Missed vector creation for '%s'", NodeClassNames[opc]);
 596     return NULL;
 597   }
 598 }
 599 
 600 bool VectorNode::is_vector_rotate(int opc) {
 601   switch (opc) {
 602   case Op_RotateLeftV:
 603   case Op_RotateRightV:
 604     return true;
 605   default:
 606     return false;
 607   }
 608 }
 609 
 610 bool VectorNode::is_vector_shift(int opc) {
 611   assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode");
 612   switch (opc) {
 613   case Op_LShiftVB:
 614   case Op_LShiftVS:
 615   case Op_LShiftVI:
 616   case Op_LShiftVL:
 617   case Op_RShiftVB:
 618   case Op_RShiftVS:
 619   case Op_RShiftVI:
 620   case Op_RShiftVL:
 621   case Op_URShiftVB:
 622   case Op_URShiftVS:
 623   case Op_URShiftVI:
 624   case Op_URShiftVL:
 625     return true;
 626   default:
 627     return false;
 628   }
 629 }
 630 
 631 bool VectorNode::is_vector_shift_count(int opc) {
 632   assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode");
 633   switch (opc) {
 634   case Op_RShiftCntV:
 635   case Op_LShiftCntV:
 636     return true;
 637   default:
 638     return false;
 639   }
 640 }
 641 
 642 static bool is_con_M1(Node* n) {
 643   if (n->is_Con()) {
 644     const Type* t = n->bottom_type();
 645     if (t->isa_int() && t->is_int()->get_con() == -1) {
 646       return true;
 647     }
 648     if (t->isa_long() && t->is_long()->get_con() == -1) {
 649       return true;
 650     }
 651   }
 652   return false;
 653 }
 654 
 655 bool VectorNode::is_all_ones_vector(Node* n) {
 656   switch (n->Opcode()) {
 657   case Op_ReplicateB:
 658   case Op_ReplicateS:
 659   case Op_ReplicateI:
 660   case Op_ReplicateL:
 661     return is_con_M1(n->in(1));
 662   default:
 663     return false;
 664   }
 665 }
 666 
 667 bool VectorNode::is_vector_bitwise_not_pattern(Node* n) {
 668   if (n->Opcode() == Op_XorV) {
 669     return is_all_ones_vector(n->in(1)) ||
 670            is_all_ones_vector(n->in(2));
 671   }
 672   return false;
 673 }
 674 
 675 // Return initial Pack node. Additional operands added with add_opd() calls.
 676 PackNode* PackNode::make(Node* s, uint vlen, BasicType bt) {
 677   const TypeVect* vt = TypeVect::make(bt, vlen);
 678   switch (bt) {
 679   case T_BOOLEAN:
 680   case T_BYTE:
 681     return new PackBNode(s, vt);
 682   case T_CHAR:
 683   case T_SHORT:
 684     return new PackSNode(s, vt);
 685   case T_INT:
 686     return new PackINode(s, vt);
 687   case T_LONG:
 688     return new PackLNode(s, vt);
 689   case T_FLOAT:
 690     return new PackFNode(s, vt);
 691   case T_DOUBLE:
 692     return new PackDNode(s, vt);
 693   default:
 694     fatal("Type '%s' is not supported for vectors", type2name(bt));
 695     return NULL;
 696   }
 697 }
 698 
 699 // Create a binary tree form for Packs. [lo, hi) (half-open) range
 700 PackNode* PackNode::binary_tree_pack(int lo, int hi) {
 701   int ct = hi - lo;
 702   assert(is_power_of_2(ct), "power of 2");
 703   if (ct == 2) {
 704     PackNode* pk = PackNode::make(in(lo), 2, vect_type()->element_basic_type());
 705     pk->add_opd(in(lo+1));
 706     return pk;
 707   } else {
 708     int mid = lo + ct/2;
 709     PackNode* n1 = binary_tree_pack(lo,  mid);
 710     PackNode* n2 = binary_tree_pack(mid, hi );
 711 
 712     BasicType bt = n1->vect_type()->element_basic_type();
 713     assert(bt == n2->vect_type()->element_basic_type(), "should be the same");
 714     switch (bt) {
 715     case T_BOOLEAN:
 716     case T_BYTE:
 717       return new PackSNode(n1, n2, TypeVect::make(T_SHORT, 2));
 718     case T_CHAR:
 719     case T_SHORT:
 720       return new PackINode(n1, n2, TypeVect::make(T_INT, 2));
 721     case T_INT:
 722       return new PackLNode(n1, n2, TypeVect::make(T_LONG, 2));
 723     case T_LONG:
 724       return new Pack2LNode(n1, n2, TypeVect::make(T_LONG, 2));
 725     case T_FLOAT:
 726       return new PackDNode(n1, n2, TypeVect::make(T_DOUBLE, 2));
 727     case T_DOUBLE:
 728       return new Pack2DNode(n1, n2, TypeVect::make(T_DOUBLE, 2));
 729     default:
 730       fatal("Type '%s' is not supported for vectors", type2name(bt));
 731       return NULL;
 732     }
 733   }
 734 }
 735 
 736 // Return the vector version of a scalar load node.
 737 LoadVectorNode* LoadVectorNode::make(int opc, Node* ctl, Node* mem,
 738                                      Node* adr, const TypePtr* atyp,
 739                                      uint vlen, BasicType bt,
 740                                      ControlDependency control_dependency) {
 741   const TypeVect* vt = TypeVect::make(bt, vlen);
 742   return new LoadVectorNode(ctl, mem, adr, atyp, vt, control_dependency);
 743 }
 744 
 745 // Return the vector version of a scalar store node.
 746 StoreVectorNode* StoreVectorNode::make(int opc, Node* ctl, Node* mem,
 747                                        Node* adr, const TypePtr* atyp, Node* val,
 748                                        uint vlen) {
 749   return new StoreVectorNode(ctl, mem, adr, atyp, val);
 750 }
 751 
 752 Node* LoadVectorMaskedNode::Ideal(PhaseGVN* phase, bool can_reshape) {
 753   if (!in(3)->is_top() && in(3)->Opcode() == Op_VectorMaskGen) {
 754     Node* mask_len = in(3)->in(1);
 755     const TypeLong* ty = phase->type(mask_len)->isa_long();
 756     if (ty && ty->is_con()) {
 757       BasicType mask_bt = ((VectorMaskGenNode*)in(3))->get_elem_type();
 758       uint load_sz      = type2aelembytes(mask_bt) * ty->get_con();
 759       if ( load_sz == 32 || load_sz == 64) {
 760         assert(load_sz == 32 || MaxVectorSize > 32, "Unexpected load size");
 761         Node* ctr = in(MemNode::Control);
 762         Node* mem = in(MemNode::Memory);
 763         Node* adr = in(MemNode::Address);
 764         return phase->transform(new LoadVectorNode(ctr, mem, adr, adr_type(), vect_type()));
 765       }
 766     }
 767   }
 768   return NULL;
 769 }
 770 
 771 Node* StoreVectorMaskedNode::Ideal(PhaseGVN* phase, bool can_reshape) {
 772   if (!in(4)->is_top() && in(4)->Opcode() == Op_VectorMaskGen) {
 773     Node* mask_len = in(4)->in(1);
 774     const TypeLong* ty = phase->type(mask_len)->isa_long();
 775     if (ty && ty->is_con()) {
 776       BasicType mask_bt = ((VectorMaskGenNode*)in(4))->get_elem_type();
 777       uint load_sz      = type2aelembytes(mask_bt) * ty->get_con();
 778       if ( load_sz == 32 || load_sz == 64) {
 779         assert(load_sz == 32 || MaxVectorSize > 32, "Unexpected store size");
 780         Node* ctr = in(MemNode::Control);
 781         Node* mem = in(MemNode::Memory);
 782         Node* adr = in(MemNode::Address);
 783         Node* val = in(MemNode::ValueIn);
 784         return phase->transform(new StoreVectorNode(ctr, mem, adr, adr_type(), val));
 785       }
 786     }
 787   }
 788   return NULL;
 789 }
 790 
 791 int ExtractNode::opcode(BasicType bt) {
 792   switch (bt) {
 793     case T_BOOLEAN: return Op_ExtractUB;
 794     case T_BYTE:    return Op_ExtractB;
 795     case T_CHAR:    return Op_ExtractC;
 796     case T_SHORT:   return Op_ExtractS;
 797     case T_INT:     return Op_ExtractI;
 798     case T_LONG:    return Op_ExtractL;
 799     case T_FLOAT:   return Op_ExtractF;
 800     case T_DOUBLE:  return Op_ExtractD;
 801     default:
 802       assert(false, "wrong type: %s", type2name(bt));
 803       return 0;
 804   }
 805 }
 806 
 807 // Extract a scalar element of vector.
 808 Node* ExtractNode::make(Node* v, uint position, BasicType bt) {
 809   assert((int)position < Matcher::max_vector_size(bt), "pos in range");
 810   ConINode* pos = ConINode::make((int)position);
 811   switch (bt) {
 812   case T_BOOLEAN: return new ExtractUBNode(v, pos);
 813   case T_BYTE:    return new ExtractBNode(v, pos);
 814   case T_CHAR:    return new ExtractCNode(v, pos);
 815   case T_SHORT:   return new ExtractSNode(v, pos);
 816   case T_INT:     return new ExtractINode(v, pos);
 817   case T_LONG:    return new ExtractLNode(v, pos);
 818   case T_FLOAT:   return new ExtractFNode(v, pos);
 819   case T_DOUBLE:  return new ExtractDNode(v, pos);
 820   default:
 821     assert(false, "wrong type: %s", type2name(bt));
 822     return NULL;
 823   }
 824 }
 825 
 826 int ReductionNode::opcode(int opc, BasicType bt) {
 827   int vopc = opc;
 828   switch (opc) {
 829     case Op_AddI:
 830       switch (bt) {
 831         case T_BOOLEAN:
 832         case T_CHAR: return 0;
 833         case T_BYTE:
 834         case T_SHORT:
 835         case T_INT:
 836           vopc = Op_AddReductionVI;
 837           break;
 838         default: ShouldNotReachHere(); return 0;
 839       }
 840       break;
 841     case Op_AddL:
 842       assert(bt == T_LONG, "must be");
 843       vopc = Op_AddReductionVL;
 844       break;
 845     case Op_AddF:
 846       assert(bt == T_FLOAT, "must be");
 847       vopc = Op_AddReductionVF;
 848       break;
 849     case Op_AddD:
 850       assert(bt == T_DOUBLE, "must be");
 851       vopc = Op_AddReductionVD;
 852       break;
 853     case Op_MulI:
 854       switch (bt) {
 855         case T_BOOLEAN:
 856         case T_CHAR: return 0;
 857         case T_BYTE:
 858         case T_SHORT:
 859         case T_INT:
 860           vopc = Op_MulReductionVI;
 861           break;
 862         default: ShouldNotReachHere(); return 0;
 863       }
 864       break;
 865     case Op_MulL:
 866       assert(bt == T_LONG, "must be");
 867       vopc = Op_MulReductionVL;
 868       break;
 869     case Op_MulF:
 870       assert(bt == T_FLOAT, "must be");
 871       vopc = Op_MulReductionVF;
 872       break;
 873     case Op_MulD:
 874       assert(bt == T_DOUBLE, "must be");
 875       vopc = Op_MulReductionVD;
 876       break;
 877     case Op_MinI:
 878       switch (bt) {
 879         case T_BOOLEAN:
 880         case T_CHAR: return 0;
 881         case T_BYTE:
 882         case T_SHORT:
 883         case T_INT:
 884           vopc = Op_MinReductionV;
 885           break;
 886         default: ShouldNotReachHere(); return 0;
 887       }
 888       break;
 889     case Op_MinL:
 890       assert(bt == T_LONG, "must be");
 891       vopc = Op_MinReductionV;
 892       break;
 893     case Op_MinF:
 894       assert(bt == T_FLOAT, "must be");
 895       vopc = Op_MinReductionV;
 896       break;
 897     case Op_MinD:
 898       assert(bt == T_DOUBLE, "must be");
 899       vopc = Op_MinReductionV;
 900       break;
 901     case Op_MaxI:
 902       switch (bt) {
 903         case T_BOOLEAN:
 904         case T_CHAR: return 0;
 905         case T_BYTE:
 906         case T_SHORT:
 907         case T_INT:
 908           vopc = Op_MaxReductionV;
 909           break;
 910         default: ShouldNotReachHere(); return 0;
 911       }
 912       break;
 913     case Op_MaxL:
 914       assert(bt == T_LONG, "must be");
 915       vopc = Op_MaxReductionV;
 916       break;
 917     case Op_MaxF:
 918       assert(bt == T_FLOAT, "must be");
 919       vopc = Op_MaxReductionV;
 920       break;
 921     case Op_MaxD:
 922       assert(bt == T_DOUBLE, "must be");
 923       vopc = Op_MaxReductionV;
 924       break;
 925     case Op_AndI:
 926       switch (bt) {
 927       case T_BOOLEAN:
 928       case T_CHAR: return 0;
 929       case T_BYTE:
 930       case T_SHORT:
 931       case T_INT:
 932         vopc = Op_AndReductionV;
 933         break;
 934       default: ShouldNotReachHere(); return 0;
 935       }
 936       break;
 937     case Op_AndL:
 938       assert(bt == T_LONG, "must be");
 939       vopc = Op_AndReductionV;
 940       break;
 941     case Op_OrI:
 942       switch(bt) {
 943       case T_BOOLEAN:
 944       case T_CHAR: return 0;
 945       case T_BYTE:
 946       case T_SHORT:
 947       case T_INT:
 948         vopc = Op_OrReductionV;
 949         break;
 950       default: ShouldNotReachHere(); return 0;
 951       }
 952       break;
 953     case Op_OrL:
 954       assert(bt == T_LONG, "must be");
 955       vopc = Op_OrReductionV;
 956       break;
 957     case Op_XorI:
 958       switch(bt) {
 959       case T_BOOLEAN:
 960       case T_CHAR: return 0;
 961       case T_BYTE:
 962       case T_SHORT:
 963       case T_INT:
 964         vopc = Op_XorReductionV;
 965         break;
 966       default: ShouldNotReachHere(); return 0;
 967       }
 968       break;
 969     case Op_XorL:
 970       assert(bt == T_LONG, "must be");
 971       vopc = Op_XorReductionV;
 972       break;
 973     default:
 974       break;
 975   }
 976   return vopc;
 977 }
 978 
 979 // Return the appropriate reduction node.
 980 ReductionNode* ReductionNode::make(int opc, Node *ctrl, Node* n1, Node* n2, BasicType bt) {
 981 
 982   int vopc = opcode(opc, bt);
 983 
 984   // This method should not be called for unimplemented vectors.
 985   guarantee(vopc != opc, "Vector for '%s' is not implemented", NodeClassNames[opc]);
 986 
 987   switch (vopc) {
 988   case Op_AddReductionVI: return new AddReductionVINode(ctrl, n1, n2);
 989   case Op_AddReductionVL: return new AddReductionVLNode(ctrl, n1, n2);
 990   case Op_AddReductionVF: return new AddReductionVFNode(ctrl, n1, n2);
 991   case Op_AddReductionVD: return new AddReductionVDNode(ctrl, n1, n2);
 992   case Op_MulReductionVI: return new MulReductionVINode(ctrl, n1, n2);
 993   case Op_MulReductionVL: return new MulReductionVLNode(ctrl, n1, n2);
 994   case Op_MulReductionVF: return new MulReductionVFNode(ctrl, n1, n2);
 995   case Op_MulReductionVD: return new MulReductionVDNode(ctrl, n1, n2);
 996   case Op_MinReductionV:  return new MinReductionVNode(ctrl, n1, n2);
 997   case Op_MaxReductionV:  return new MaxReductionVNode(ctrl, n1, n2);
 998   case Op_AndReductionV:  return new AndReductionVNode(ctrl, n1, n2);
 999   case Op_OrReductionV:   return new OrReductionVNode(ctrl, n1, n2);
1000   case Op_XorReductionV:  return new XorReductionVNode(ctrl, n1, n2);
1001   default:
1002     assert(false, "unknown node: %s", NodeClassNames[vopc]);
1003     return NULL;
1004   }
1005 }
1006 
1007 Node* VectorLoadMaskNode::Identity(PhaseGVN* phase) {
1008   BasicType out_bt = type()->is_vect()->element_basic_type();
1009   if (out_bt == T_BOOLEAN) {
1010     return in(1); // redundant conversion
1011   }
1012   return this;
1013 }
1014 
1015 Node* VectorStoreMaskNode::Identity(PhaseGVN* phase) {
1016   // Identity transformation on boolean vectors.
1017   //   VectorStoreMask (VectorLoadMask bv) elem_size ==> bv
1018   //   vector[n]{bool} => vector[n]{t} => vector[n]{bool}
1019   if (in(1)->Opcode() == Op_VectorLoadMask) {
1020     return in(1)->in(1);
1021   }
1022   return this;
1023 }
1024 
1025 VectorStoreMaskNode* VectorStoreMaskNode::make(PhaseGVN& gvn, Node* in, BasicType in_type, uint num_elem) {
1026   assert(in->bottom_type()->isa_vect(), "sanity");
1027   const TypeVect* vt = TypeVect::make(T_BOOLEAN, num_elem);
1028   int elem_size = type2aelembytes(in_type);
1029   return new VectorStoreMaskNode(in, gvn.intcon(elem_size), vt);
1030 }
1031 
1032 VectorCastNode* VectorCastNode::make(int vopc, Node* n1, BasicType bt, uint vlen) {
1033   const TypeVect* vt = TypeVect::make(bt, vlen);
1034   switch (vopc) {
1035     case Op_VectorCastB2X: return new VectorCastB2XNode(n1, vt);
1036     case Op_VectorCastS2X: return new VectorCastS2XNode(n1, vt);
1037     case Op_VectorCastI2X: return new VectorCastI2XNode(n1, vt);
1038     case Op_VectorCastL2X: return new VectorCastL2XNode(n1, vt);
1039     case Op_VectorCastF2X: return new VectorCastF2XNode(n1, vt);
1040     case Op_VectorCastD2X: return new VectorCastD2XNode(n1, vt);
1041     default:
1042       assert(false, "unknown node: %s", NodeClassNames[vopc]);
1043       return NULL;
1044   }
1045 }
1046 
1047 int VectorCastNode::opcode(BasicType bt) {
1048   switch (bt) {
1049     case T_BYTE:   return Op_VectorCastB2X;
1050     case T_SHORT:  return Op_VectorCastS2X;
1051     case T_INT:    return Op_VectorCastI2X;
1052     case T_LONG:   return Op_VectorCastL2X;
1053     case T_FLOAT:  return Op_VectorCastF2X;
1054     case T_DOUBLE: return Op_VectorCastD2X;
1055     default:
1056       assert(false, "unknown type: %s", type2name(bt));
1057       return 0;
1058   }
1059 }
1060 
1061 Node* VectorCastNode::Identity(PhaseGVN* phase) {
1062   if (!in(1)->is_top()) {
1063     BasicType  in_bt = in(1)->bottom_type()->is_vect()->element_basic_type();
1064     BasicType out_bt = vect_type()->element_basic_type();
1065     if (in_bt == out_bt) {
1066       return in(1); // redundant cast
1067     }
1068   }
1069   return this;
1070 }
1071 
1072 Node* ReductionNode::make_reduction_input(PhaseGVN& gvn, int opc, BasicType bt) {
1073   int vopc = opcode(opc, bt);
1074   guarantee(vopc != opc, "Vector reduction for '%s' is not implemented", NodeClassNames[opc]);
1075 
1076   switch (vopc) {
1077     case Op_AndReductionV:
1078       switch (bt) {
1079         case T_BYTE:
1080         case T_SHORT:
1081         case T_INT:
1082           return gvn.makecon(TypeInt::MINUS_1);
1083         case T_LONG:
1084           return gvn.makecon(TypeLong::MINUS_1);
1085         default:
1086           fatal("Missed vector creation for '%s' as the basic type is not correct.", NodeClassNames[vopc]);
1087           return NULL;
1088       }
1089       break;
1090     case Op_AddReductionVI: // fallthrough
1091     case Op_AddReductionVL: // fallthrough
1092     case Op_AddReductionVF: // fallthrough
1093     case Op_AddReductionVD:
1094     case Op_OrReductionV:
1095     case Op_XorReductionV:
1096       return gvn.zerocon(bt);
1097     case Op_MulReductionVI:
1098       return gvn.makecon(TypeInt::ONE);
1099     case Op_MulReductionVL:
1100       return gvn.makecon(TypeLong::ONE);
1101     case Op_MulReductionVF:
1102       return gvn.makecon(TypeF::ONE);
1103     case Op_MulReductionVD:
1104       return gvn.makecon(TypeD::ONE);
1105     case Op_MinReductionV:
1106       switch (bt) {
1107         case T_BYTE:
1108         case T_SHORT:
1109         case T_INT:
1110           return gvn.makecon(TypeInt::MAX);
1111         case T_LONG:
1112           return gvn.makecon(TypeLong::MAX);
1113         case T_FLOAT:
1114           return gvn.makecon(TypeF::POS_INF);
1115         case T_DOUBLE:
1116           return gvn.makecon(TypeD::POS_INF);
1117           default: Unimplemented(); return NULL;
1118       }
1119       break;
1120     case Op_MaxReductionV:
1121       switch (bt) {
1122         case T_BYTE:
1123         case T_SHORT:
1124         case T_INT:
1125           return gvn.makecon(TypeInt::MIN);
1126         case T_LONG:
1127           return gvn.makecon(TypeLong::MIN);
1128         case T_FLOAT:
1129           return gvn.makecon(TypeF::NEG_INF);
1130         case T_DOUBLE:
1131           return gvn.makecon(TypeD::NEG_INF);
1132           default: Unimplemented(); return NULL;
1133       }
1134       break;
1135     default:
1136       fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
1137       return NULL;
1138   }
1139 }
1140 
1141 bool ReductionNode::implemented(int opc, uint vlen, BasicType bt) {
1142   if (is_java_primitive(bt) &&
1143       (vlen > 1) && is_power_of_2(vlen) &&
1144       Matcher::vector_size_supported(bt, vlen)) {
1145     int vopc = ReductionNode::opcode(opc, bt);
1146     return vopc != opc && Matcher::match_rule_supported_vector(vopc, vlen, bt);
1147   }
1148   return false;
1149 }
1150 
1151 MacroLogicVNode* MacroLogicVNode::make(PhaseGVN& gvn, Node* in1, Node* in2, Node* in3,
1152                                        uint truth_table, const TypeVect* vt) {
1153   assert(truth_table <= 0xFF, "invalid");
1154   assert(in1->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
1155   assert(in2->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
1156   assert(in3->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
1157   Node* fn = gvn.intcon(truth_table);
1158   return new MacroLogicVNode(in1, in2, in3, fn, vt);
1159 }
1160 
1161 Node* VectorNode::degenerate_vector_rotate(Node* src, Node* cnt, bool is_rotate_left,
1162                                            int vlen, BasicType bt, PhaseGVN* phase) {
1163   assert(is_integral_type(bt), "sanity");
1164   const TypeVect* vt = TypeVect::make(bt, vlen);
1165 
1166   int shift_mask = (type2aelembytes(bt) * 8) - 1;
1167   int shiftLOpc = (bt == T_LONG) ? Op_LShiftL : Op_LShiftI;
1168   auto urshiftopc = [=]() {
1169     switch(bt) {
1170       case T_INT: return Op_URShiftI;
1171       case T_LONG: return Op_URShiftL;
1172       case T_BYTE: return Op_URShiftB;
1173       case T_SHORT: return Op_URShiftS;
1174       default: return (Opcodes)0;
1175     }
1176   };
1177   int shiftROpc = urshiftopc();
1178 
1179   // Compute shift values for right rotation and
1180   // later swap them in case of left rotation.
1181   Node* shiftRCnt = NULL;
1182   Node* shiftLCnt = NULL;
1183   const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
1184   bool is_binary_vector_op = false;
1185   if (cnt_type && cnt_type->is_con()) {
1186     // Constant shift.
1187     int shift = cnt_type->get_con() & shift_mask;
1188     shiftRCnt = phase->intcon(shift);
1189     shiftLCnt = phase->intcon(shift_mask + 1 - shift);
1190   } else if (VectorNode::is_invariant_vector(cnt)) {
1191     // Scalar variable shift, handle replicates generated by auto vectorizer.
1192     cnt = cnt->in(1);
1193     if (bt == T_LONG) {
1194       // Shift count vector for Rotate vector has long elements too.
1195       if (cnt->Opcode() == Op_ConvI2L) {
1196          cnt = cnt->in(1);
1197       } else {
1198          assert(cnt->bottom_type()->isa_long() &&
1199                 cnt->bottom_type()->is_long()->is_con(), "Long constant expected");
1200          cnt = phase->transform(new ConvL2INode(cnt));
1201       }
1202     }
1203     shiftRCnt = phase->transform(new AndINode(cnt, phase->intcon(shift_mask)));
1204     shiftLCnt = phase->transform(new SubINode(phase->intcon(shift_mask + 1), shiftRCnt));
1205   } else {
1206     // Variable vector rotate count.
1207     assert(Matcher::supports_vector_variable_shifts(), "");
1208 
1209     int subVopc = 0;
1210     int addVopc = 0;
1211     Node* shift_mask_node = NULL;
1212     Node* const_one_node = NULL;
1213 
1214     assert(cnt->bottom_type()->isa_vect(), "Unexpected shift");
1215     const Type* elem_ty = Type::get_const_basic_type(bt);
1216 
1217     if (bt == T_LONG) {
1218       shift_mask_node = phase->longcon(shift_mask);
1219       const_one_node = phase->longcon(1L);
1220       subVopc = VectorNode::opcode(Op_SubL, bt);
1221       addVopc = VectorNode::opcode(Op_AddL, bt);
1222     } else {
1223       shift_mask_node = phase->intcon(shift_mask);
1224       const_one_node = phase->intcon(1);
1225       subVopc = VectorNode::opcode(Op_SubI, bt);
1226       addVopc = VectorNode::opcode(Op_AddI, bt);
1227     }
1228     Node* vector_mask = phase->transform(VectorNode::scalar2vector(shift_mask_node, vlen, elem_ty));
1229     Node* vector_one = phase->transform(VectorNode::scalar2vector(const_one_node, vlen, elem_ty));
1230 
1231     shiftRCnt = cnt;
1232     shiftRCnt = phase->transform(VectorNode::make(Op_AndV, shiftRCnt, vector_mask, vt));
1233     vector_mask = phase->transform(VectorNode::make(addVopc, vector_one, vector_mask, vt));
1234     shiftLCnt = phase->transform(VectorNode::make(subVopc, vector_mask, shiftRCnt, vt));
1235     is_binary_vector_op = true;
1236   }
1237 
1238   // Swap the computed left and right shift counts.
1239   if (is_rotate_left) {
1240     swap(shiftRCnt,shiftLCnt);
1241   }
1242 
1243   if (!is_binary_vector_op) {
1244     shiftLCnt = phase->transform(new LShiftCntVNode(shiftLCnt, vt));
1245     shiftRCnt = phase->transform(new RShiftCntVNode(shiftRCnt, vt));
1246   }
1247 
1248   return new OrVNode(phase->transform(VectorNode::make(shiftLOpc, src, shiftLCnt, vlen, bt)),
1249                      phase->transform(VectorNode::make(shiftROpc, src, shiftRCnt, vlen, bt)),
1250                      vt);
1251 }
1252 
1253 Node* RotateLeftVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1254   int vlen = length();
1255   BasicType bt = vect_type()->element_basic_type();
1256   if ((!in(2)->is_Con() && !Matcher::supports_vector_variable_rotates()) ||
1257        !Matcher::match_rule_supported_vector(Op_RotateLeftV, vlen, bt)) {
1258     return VectorNode::degenerate_vector_rotate(in(1), in(2), true, vlen, bt, phase);
1259   }
1260   return NULL;
1261 }
1262 
1263 Node* RotateRightVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1264   int vlen = length();
1265   BasicType bt = vect_type()->element_basic_type();
1266   if ((!in(2)->is_Con() && !Matcher::supports_vector_variable_rotates()) ||
1267        !Matcher::match_rule_supported_vector(Op_RotateRightV, vlen, bt)) {
1268     return VectorNode::degenerate_vector_rotate(in(1), in(2), false, vlen, bt, phase);
1269   }
1270   return NULL;
1271 }
1272 
1273 #ifndef PRODUCT
1274 void VectorMaskCmpNode::dump_spec(outputStream *st) const {
1275   st->print(" %d #", _predicate); _type->dump_on(st);
1276 }
1277 #endif // PRODUCT
1278 
1279 Node* VectorReinterpretNode::Identity(PhaseGVN *phase) {
1280   Node* n = in(1);
1281   if (n->Opcode() == Op_VectorReinterpret) {
1282     // "VectorReinterpret (VectorReinterpret node) ==> node" if:
1283     //   1) Types of 'node' and 'this' are identical
1284     //   2) Truncations are not introduced by the first VectorReinterpret
1285     if (Type::cmp(bottom_type(), n->in(1)->bottom_type()) == 0 &&
1286         length_in_bytes() <= n->bottom_type()->is_vect()->length_in_bytes()) {
1287       return n->in(1);
1288     }
1289   }
1290   return this;
1291 }
1292 
1293 Node* VectorInsertNode::make(Node* vec, Node* new_val, int position) {
1294   assert(position < (int)vec->bottom_type()->is_vect()->length(), "pos in range");
1295   ConINode* pos = ConINode::make(position);
1296   return new VectorInsertNode(vec, new_val, pos, vec->bottom_type()->is_vect());
1297 }
1298 
1299 Node* VectorUnboxNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1300   Node* n = obj()->uncast();
1301   if (EnableVectorReboxing && n->Opcode() == Op_VectorBox) {
1302     if (Type::cmp(bottom_type(), n->in(VectorBoxNode::Value)->bottom_type()) == 0) {
1303       // Handled by VectorUnboxNode::Identity()
1304     } else {
1305       VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
1306       ciKlass* vbox_klass = vbox->box_type()->klass();
1307       const TypeVect* in_vt = vbox->vec_type();
1308       const TypeVect* out_vt = type()->is_vect();
1309 
1310       if (in_vt->length() == out_vt->length()) {
1311         Node* value = vbox->in(VectorBoxNode::Value);
1312 
1313         bool is_vector_mask    = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
1314         bool is_vector_shuffle = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
1315         if (is_vector_mask) {
1316           if (in_vt->length_in_bytes() == out_vt->length_in_bytes() &&
1317               Matcher::match_rule_supported_vector(Op_VectorMaskCast, out_vt->length(), out_vt->element_basic_type())) {
1318             // Apply "VectorUnbox (VectorBox vmask) ==> VectorMaskCast (vmask)"
1319             // directly. This could avoid the transformation ordering issue from
1320             // "VectorStoreMask (VectorLoadMask vmask) => vmask".
1321             return new VectorMaskCastNode(value, out_vt);
1322           }
1323           // VectorUnbox (VectorBox vmask) ==> VectorLoadMask (VectorStoreMask vmask)
1324           value = phase->transform(VectorStoreMaskNode::make(*phase, value, in_vt->element_basic_type(), in_vt->length()));
1325           return new VectorLoadMaskNode(value, out_vt);
1326         } else if (is_vector_shuffle) {
1327           if (!is_shuffle_to_vector()) {
1328             // VectorUnbox (VectorBox vshuffle) ==> VectorLoadShuffle vshuffle
1329             return new VectorLoadShuffleNode(value, out_vt);
1330           }
1331         } else {
1332           // Vector type mismatch is only supported for masks and shuffles, but sometimes it happens in pathological cases.
1333         }
1334       } else {
1335         // Vector length mismatch.
1336         // Sometimes happen in pathological cases (e.g., when unboxing happens in effectively dead code).
1337       }
1338     }
1339   }
1340   return NULL;
1341 }
1342 
1343 Node* VectorUnboxNode::Identity(PhaseGVN* phase) {
1344   Node* n = obj()->uncast();
1345   if (EnableVectorReboxing && n->Opcode() == Op_VectorBox) {
1346     if (Type::cmp(bottom_type(), n->in(VectorBoxNode::Value)->bottom_type()) == 0) {
1347       return n->in(VectorBoxNode::Value); // VectorUnbox (VectorBox v) ==> v
1348     } else {
1349       // Handled by VectorUnboxNode::Ideal().
1350     }
1351   }
1352   return this;
1353 }
1354 
1355 const TypeFunc* VectorBoxNode::vec_box_type(const TypeInstPtr* box_type) {
1356   const Type** fields = TypeTuple::fields(0);
1357   const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms, fields);
1358 
1359   fields = TypeTuple::fields(1);
1360   fields[TypeFunc::Parms+0] = box_type;
1361   const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
1362 
1363   return TypeFunc::make(domain, range);
1364 }
1365 
1366 Node* ShiftVNode::Identity(PhaseGVN* phase) {
1367   Node* in2 = in(2);
1368   // Shift by ZERO does nothing
1369   if (is_vshift_cnt(in2) && phase->find_int_type(in2->in(1)) == TypeInt::ZERO) {
1370     return in(1);
1371   }
1372   return this;
1373 }
1374 
1375 Node* VectorMaskOpNode::make(Node* mask, const Type* ty, int mopc) {
1376   switch(mopc) {
1377     case Op_VectorMaskTrueCount:
1378       return new VectorMaskTrueCountNode(mask, ty);
1379     case Op_VectorMaskLastTrue:
1380       return new VectorMaskLastTrueNode(mask, ty);
1381     case Op_VectorMaskFirstTrue:
1382       return new VectorMaskFirstTrueNode(mask, ty);
1383     default:
1384       assert(false, "Unhandled operation");
1385   }
1386   return NULL;
1387 }
1388 
1389 
1390 #ifndef PRODUCT
1391 void VectorBoxAllocateNode::dump_spec(outputStream *st) const {
1392   CallStaticJavaNode::dump_spec(st);
1393 }
1394 
1395 #endif // !PRODUCT