1 /*
   2  * Copyright (c) 2007, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 #include "precompiled.hpp"
  25 #include "memory/allocation.inline.hpp"
  26 #include "opto/connode.hpp"
  27 #include "opto/mulnode.hpp"
  28 #include "opto/subnode.hpp"
  29 #include "opto/vectornode.hpp"
  30 #include "opto/convertnode.hpp"
  31 #include "utilities/powerOfTwo.hpp"
  32 #include "utilities/globalDefinitions.hpp"
  33 
  34 //------------------------------VectorNode--------------------------------------
  35 
  36 // Return the vector operator for the specified scalar operation
  37 // and vector length.
  38 int VectorNode::opcode(int sopc, BasicType bt) {
  39   switch (sopc) {
  40   case Op_AddI:
  41     switch (bt) {
  42     case T_BOOLEAN:
  43     case T_BYTE:      return Op_AddVB;
  44     case T_CHAR:
  45     case T_SHORT:     return Op_AddVS;
  46     case T_INT:       return Op_AddVI;
  47     default:          return 0;
  48     }
  49   case Op_AddL: return (bt == T_LONG   ? Op_AddVL : 0);
  50   case Op_AddF: return (bt == T_FLOAT  ? Op_AddVF : 0);
  51   case Op_AddD: return (bt == T_DOUBLE ? Op_AddVD : 0);
  52 
  53   case Op_SubI:
  54     switch (bt) {
  55     case T_BOOLEAN:
  56     case T_BYTE:   return Op_SubVB;
  57     case T_CHAR:
  58     case T_SHORT:  return Op_SubVS;
  59     case T_INT:    return Op_SubVI;
  60     default:       return 0;
  61     }
  62   case Op_SubL: return (bt == T_LONG   ? Op_SubVL : 0);
  63   case Op_SubF: return (bt == T_FLOAT  ? Op_SubVF : 0);
  64   case Op_SubD: return (bt == T_DOUBLE ? Op_SubVD : 0);
  65 
  66   case Op_MulI:
  67     switch (bt) {
  68     case T_BOOLEAN:return 0;
  69     case T_BYTE:   return Op_MulVB;
  70     case T_CHAR:
  71     case T_SHORT:  return Op_MulVS;
  72     case T_INT:    return Op_MulVI;
  73     default:       return 0;
  74     }
  75   case Op_MulL: return (bt == T_LONG ? Op_MulVL : 0);
  76   case Op_MulF:
  77     return (bt == T_FLOAT ? Op_MulVF : 0);
  78   case Op_MulD:
  79     return (bt == T_DOUBLE ? Op_MulVD : 0);
  80   case Op_FmaD:
  81     return (bt == T_DOUBLE ? Op_FmaVD : 0);
  82   case Op_FmaF:
  83     return (bt == T_FLOAT ? Op_FmaVF : 0);
  84   case Op_CMoveF:
  85     return (bt == T_FLOAT ? Op_CMoveVF : 0);
  86   case Op_CMoveD:
  87     return (bt == T_DOUBLE ? Op_CMoveVD : 0);
  88   case Op_DivF:
  89     return (bt == T_FLOAT ? Op_DivVF : 0);
  90   case Op_DivD:
  91     return (bt == T_DOUBLE ? Op_DivVD : 0);
  92   case Op_AbsI:
  93     switch (bt) {
  94     case T_BOOLEAN:
  95     case T_CHAR:  return 0; // abs does not make sense for unsigned
  96     case T_BYTE:  return Op_AbsVB;
  97     case T_SHORT: return Op_AbsVS;
  98     case T_INT:   return Op_AbsVI;
  99     default:      return 0;
 100     }
 101   case Op_AbsL:
 102     return (bt == T_LONG ? Op_AbsVL : 0);
 103   case Op_MinI:
 104     switch (bt) {
 105     case T_BOOLEAN:
 106     case T_CHAR:   return 0;
 107     case T_BYTE:
 108     case T_SHORT:
 109     case T_INT:    return Op_MinV;
 110     default:       return 0;
 111     }
 112   case Op_MinL:
 113     return (bt == T_LONG ? Op_MinV : 0);
 114   case Op_MinF:
 115     return (bt == T_FLOAT ? Op_MinV : 0);
 116   case Op_MinD:
 117     return (bt == T_DOUBLE ? Op_MinV : 0);
 118   case Op_MaxI:
 119     switch (bt) {
 120     case T_BOOLEAN:
 121     case T_CHAR:   return 0;
 122     case T_BYTE:
 123     case T_SHORT:
 124     case T_INT:    return Op_MaxV;
 125     default:       return 0;
 126     }
 127   case Op_MaxL:
 128     return (bt == T_LONG ? Op_MaxV : 0);
 129   case Op_MaxF:
 130     return (bt == T_FLOAT ? Op_MaxV : 0);
 131   case Op_MaxD:
 132     return (bt == T_DOUBLE ? Op_MaxV : 0);
 133   case Op_AbsF:
 134     return (bt == T_FLOAT ? Op_AbsVF : 0);
 135   case Op_AbsD:
 136     return (bt == T_DOUBLE ? Op_AbsVD : 0);
 137   case Op_NegI:
 138     return (bt == T_INT ? Op_NegVI : 0);
 139   case Op_NegF:
 140     return (bt == T_FLOAT ? Op_NegVF : 0);
 141   case Op_NegD:
 142     return (bt == T_DOUBLE ? Op_NegVD : 0);
 143   case Op_RoundDoubleMode:
 144     return (bt == T_DOUBLE ? Op_RoundDoubleModeV : 0);
 145   case Op_RotateLeft:
 146     return (is_integral_type(bt) ? Op_RotateLeftV : 0);
 147   case Op_RotateRight:
 148     return (is_integral_type(bt) ? Op_RotateRightV : 0);
 149   case Op_SqrtF:
 150     return (bt == T_FLOAT ? Op_SqrtVF : 0);
 151   case Op_SqrtD:
 152     return (bt == T_DOUBLE ? Op_SqrtVD : 0);
 153   case Op_PopCountI:
 154     // Unimplemented for subword types since bit count changes
 155     // depending on size of lane (and sign bit).
 156     return (bt == T_INT ? Op_PopCountVI : 0);
 157   case Op_LShiftI:
 158     switch (bt) {
 159     case T_BOOLEAN:
 160     case T_BYTE:   return Op_LShiftVB;
 161     case T_CHAR:
 162     case T_SHORT:  return Op_LShiftVS;
 163     case T_INT:    return Op_LShiftVI;
 164     default:       return 0;
 165     }
 166   case Op_LShiftL:
 167     return (bt == T_LONG ? Op_LShiftVL : 0);
 168   case Op_RShiftI:
 169     switch (bt) {
 170     case T_BOOLEAN:return Op_URShiftVB; // boolean is unsigned value
 171     case T_CHAR:   return Op_URShiftVS; // char is unsigned value
 172     case T_BYTE:   return Op_RShiftVB;
 173     case T_SHORT:  return Op_RShiftVS;
 174     case T_INT:    return Op_RShiftVI;
 175     default:       return 0;
 176     }
 177   case Op_RShiftL:
 178     return (bt == T_LONG ? Op_RShiftVL : 0);
 179   case Op_URShiftB:
 180     return (bt == T_BYTE ? Op_URShiftVB : 0);
 181   case Op_URShiftS:
 182     return (bt == T_SHORT ? Op_URShiftVS : 0);
 183   case Op_URShiftI:
 184     switch (bt) {
 185     case T_BOOLEAN:return Op_URShiftVB;
 186     case T_CHAR:   return Op_URShiftVS;
 187     case T_BYTE:
 188     case T_SHORT:  return 0; // Vector logical right shift for signed short
 189                              // values produces incorrect Java result for
 190                              // negative data because java code should convert
 191                              // a short value into int value with sign
 192                              // extension before a shift.
 193     case T_INT:    return Op_URShiftVI;
 194     default:       return 0;
 195     }
 196   case Op_URShiftL:
 197     return (bt == T_LONG ? Op_URShiftVL : 0);
 198   case Op_AndI:
 199   case Op_AndL:
 200     return Op_AndV;
 201   case Op_OrI:
 202   case Op_OrL:
 203     return Op_OrV;
 204   case Op_XorI:
 205   case Op_XorL:
 206     return Op_XorV;
 207 
 208   case Op_LoadB:
 209   case Op_LoadUB:
 210   case Op_LoadUS:
 211   case Op_LoadS:
 212   case Op_LoadI:
 213   case Op_LoadL:
 214   case Op_LoadF:
 215   case Op_LoadD:
 216     return Op_LoadVector;
 217 
 218   case Op_StoreB:
 219   case Op_StoreC:
 220   case Op_StoreI:
 221   case Op_StoreL:
 222   case Op_StoreF:
 223   case Op_StoreD:
 224     return Op_StoreVector;
 225   case Op_MulAddS2I:
 226     return Op_MulAddVS2VI;
 227   case Op_ConvI2F:
 228     return Op_VectorCastI2X;
 229   case Op_ConvL2D:
 230     return Op_VectorCastL2X;
 231   case Op_ConvF2I:
 232     return Op_VectorCastF2X;
 233   case Op_ConvD2L:
 234     return Op_VectorCastD2X;
 235 
 236   default:
 237     return 0; // Unimplemented
 238   }
 239 }
 240 
 241 int VectorNode::replicate_opcode(BasicType bt) {
 242   switch(bt) {
 243     case T_BOOLEAN:
 244     case T_BYTE:
 245       return Op_ReplicateB;
 246     case T_SHORT:
 247     case T_CHAR:
 248       return Op_ReplicateS;
 249     case T_INT:
 250       return Op_ReplicateI;
 251     case T_LONG:
 252       return Op_ReplicateL;
 253     case T_FLOAT:
 254       return Op_ReplicateF;
 255     case T_DOUBLE:
 256       return Op_ReplicateD;
 257     default:
 258       assert(false, "wrong type: %s", type2name(bt));
 259       return 0;
 260   }
 261 }
 262 
 263 // Also used to check if the code generator
 264 // supports the vector operation.
 265 bool VectorNode::implemented(int opc, uint vlen, BasicType bt) {
 266   if (is_java_primitive(bt) &&
 267       (vlen > 1) && is_power_of_2(vlen) &&
 268       Matcher::vector_size_supported(bt, vlen)) {
 269     int vopc = VectorNode::opcode(opc, bt);
 270     // For rotate operation we will do a lazy de-generation into
 271     // OrV/LShiftV/URShiftV pattern if the target does not support
 272     // vector rotation instruction.
 273     if (VectorNode::is_vector_rotate(vopc)) {
 274       return is_vector_rotate_supported(vopc, vlen, bt);
 275     }
 276     return vopc > 0 && Matcher::match_rule_supported_vector(vopc, vlen, bt);
 277   }
 278   return false;
 279 }
 280 
 281 bool VectorNode::is_type_transition_short_to_int(Node* n) {
 282   switch (n->Opcode()) {
 283   case Op_MulAddS2I:
 284     return true;
 285   }
 286   return false;
 287 }
 288 
 289 bool VectorNode::is_type_transition_to_int(Node* n) {
 290   return is_type_transition_short_to_int(n);
 291 }
 292 
 293 bool VectorNode::is_muladds2i(Node* n) {
 294   if (n->Opcode() == Op_MulAddS2I) {
 295     return true;
 296   }
 297   return false;
 298 }
 299 
 300 bool VectorNode::is_roundopD(Node* n) {
 301   if (n->Opcode() == Op_RoundDoubleMode) {
 302     return true;
 303   }
 304   return false;
 305 }
 306 
 307 bool VectorNode::is_vector_rotate_supported(int vopc, uint vlen, BasicType bt) {
 308   assert(VectorNode::is_vector_rotate(vopc), "wrong opcode");
 309 
 310   // If target defines vector rotation patterns then no
 311   // need for degeneration.
 312   if (Matcher::match_rule_supported_vector(vopc, vlen, bt)) {
 313     return true;
 314   }
 315 
 316   // If target does not support variable shift operations then no point
 317   // in creating a rotate vector node since it will not be disintegratable.
 318   // Adding a pessimistic check to avoid complex pattern mathing which
 319   // may not be full proof.
 320   if (!Matcher::supports_vector_variable_shifts()) {
 321      return false;
 322   }
 323 
 324   // Validate existence of nodes created in case of rotate degeneration.
 325   switch (bt) {
 326     case T_INT:
 327       return Matcher::match_rule_supported_vector(Op_OrV,       vlen, bt) &&
 328              Matcher::match_rule_supported_vector(Op_LShiftVI,  vlen, bt) &&
 329              Matcher::match_rule_supported_vector(Op_URShiftVI, vlen, bt);
 330     case T_LONG:
 331       return Matcher::match_rule_supported_vector(Op_OrV,       vlen, bt) &&
 332              Matcher::match_rule_supported_vector(Op_LShiftVL,  vlen, bt) &&
 333              Matcher::match_rule_supported_vector(Op_URShiftVL, vlen, bt);
 334     default:
 335       assert(false, "not supported: %s", type2name(bt));
 336       return false;
 337   }
 338 }
 339 
 340 bool VectorNode::is_shift_opcode(int opc) {
 341   switch (opc) {
 342   case Op_LShiftI:
 343   case Op_LShiftL:
 344   case Op_RShiftI:
 345   case Op_RShiftL:
 346   case Op_URShiftB:
 347   case Op_URShiftS:
 348   case Op_URShiftI:
 349   case Op_URShiftL:
 350     return true;
 351   default:
 352     return false;
 353   }
 354 }
 355 
 356 bool VectorNode::is_shift(Node* n) {
 357   return is_shift_opcode(n->Opcode());
 358 }
 359 
 360 bool VectorNode::is_rotate_opcode(int opc) {
 361   switch (opc) {
 362   case Op_RotateRight:
 363   case Op_RotateLeft:
 364     return true;
 365   default:
 366     return false;
 367   }
 368 }
 369 
 370 bool VectorNode::is_scalar_rotate(Node* n) {
 371   if (is_rotate_opcode(n->Opcode())) {
 372     return true;
 373   }
 374   return false;
 375 }
 376 
 377 bool VectorNode::is_vshift_cnt_opcode(int opc) {
 378   switch (opc) {
 379   case Op_LShiftCntV:
 380   case Op_RShiftCntV:
 381     return true;
 382   default:
 383     return false;
 384   }
 385 }
 386 
 387 bool VectorNode::is_vshift_cnt(Node* n) {
 388   return is_vshift_cnt_opcode(n->Opcode());
 389 }
 390 
 391 // Check if input is loop invariant vector.
 392 bool VectorNode::is_invariant_vector(Node* n) {
 393   // Only Replicate vector nodes are loop invariant for now.
 394   switch (n->Opcode()) {
 395   case Op_ReplicateB:
 396   case Op_ReplicateS:
 397   case Op_ReplicateI:
 398   case Op_ReplicateL:
 399   case Op_ReplicateF:
 400   case Op_ReplicateD:
 401     return true;
 402   default:
 403     return false;
 404   }
 405 }
 406 
 407 // [Start, end) half-open range defining which operands are vectors
 408 void VectorNode::vector_operands(Node* n, uint* start, uint* end) {
 409   switch (n->Opcode()) {
 410   case Op_LoadB:   case Op_LoadUB:
 411   case Op_LoadS:   case Op_LoadUS:
 412   case Op_LoadI:   case Op_LoadL:
 413   case Op_LoadF:   case Op_LoadD:
 414   case Op_LoadP:   case Op_LoadN:
 415     *start = 0;
 416     *end   = 0; // no vector operands
 417     break;
 418   case Op_StoreB:  case Op_StoreC:
 419   case Op_StoreI:  case Op_StoreL:
 420   case Op_StoreF:  case Op_StoreD:
 421   case Op_StoreP:  case Op_StoreN:
 422     *start = MemNode::ValueIn;
 423     *end   = MemNode::ValueIn + 1; // 1 vector operand
 424     break;
 425   case Op_LShiftI:  case Op_LShiftL:
 426   case Op_RShiftI:  case Op_RShiftL:
 427   case Op_URShiftI: case Op_URShiftL:
 428     *start = 1;
 429     *end   = 2; // 1 vector operand
 430     break;
 431   case Op_AddI: case Op_AddL: case Op_AddF: case Op_AddD:
 432   case Op_SubI: case Op_SubL: case Op_SubF: case Op_SubD:
 433   case Op_MulI: case Op_MulL: case Op_MulF: case Op_MulD:
 434   case Op_DivF: case Op_DivD:
 435   case Op_AndI: case Op_AndL:
 436   case Op_OrI:  case Op_OrL:
 437   case Op_XorI: case Op_XorL:
 438   case Op_MulAddS2I:
 439     *start = 1;
 440     *end   = 3; // 2 vector operands
 441     break;
 442   case Op_CMoveI:  case Op_CMoveL:  case Op_CMoveF:  case Op_CMoveD:
 443     *start = 2;
 444     *end   = n->req();
 445     break;
 446   case Op_FmaD:
 447   case Op_FmaF:
 448     *start = 1;
 449     *end   = 4; // 3 vector operands
 450     break;
 451   default:
 452     *start = 1;
 453     *end   = n->req(); // default is all operands
 454   }
 455 }
 456 
 457 VectorNode* VectorNode::make_mask_node(int vopc, Node* n1, Node* n2, uint vlen, BasicType bt) {
 458   guarantee(vopc > 0, "vopc must be > 0");
 459   const TypeVect* vmask_type = TypeVect::makemask(bt, vlen);
 460   switch (vopc) {
 461     case Op_AndV:
 462       if (Matcher::match_rule_supported_vector_masked(Op_AndVMask, vlen, bt)) {
 463         return new AndVMaskNode(n1, n2, vmask_type);
 464       }
 465       return new AndVNode(n1, n2, vmask_type);
 466     case Op_OrV:
 467       if (Matcher::match_rule_supported_vector_masked(Op_OrVMask, vlen, bt)) {
 468         return new OrVMaskNode(n1, n2, vmask_type);
 469       }
 470       return new OrVNode(n1, n2, vmask_type);
 471     case Op_XorV:
 472       if (Matcher::match_rule_supported_vector_masked(Op_XorVMask, vlen, bt)) {
 473         return new XorVMaskNode(n1, n2, vmask_type);
 474       }
 475       return new XorVNode(n1, n2, vmask_type);
 476     default:
 477       fatal("Unsupported mask vector creation for '%s'", NodeClassNames[vopc]);
 478       return NULL;
 479   }
 480 }
 481 
 482 // Make a vector node for binary operation
 483 VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, const TypeVect* vt, bool is_mask, bool is_var_shift) {
 484   // This method should not be called for unimplemented vectors.
 485   guarantee(vopc > 0, "vopc must be > 0");
 486 
 487   if (is_mask) {
 488     return make_mask_node(vopc, n1, n2, vt->length(), vt->element_basic_type());
 489   }
 490 
 491   switch (vopc) {
 492   case Op_AddVB: return new AddVBNode(n1, n2, vt);
 493   case Op_AddVS: return new AddVSNode(n1, n2, vt);
 494   case Op_AddVI: return new AddVINode(n1, n2, vt);
 495   case Op_AddVL: return new AddVLNode(n1, n2, vt);
 496   case Op_AddVF: return new AddVFNode(n1, n2, vt);
 497   case Op_AddVD: return new AddVDNode(n1, n2, vt);
 498 
 499   case Op_SubVB: return new SubVBNode(n1, n2, vt);
 500   case Op_SubVS: return new SubVSNode(n1, n2, vt);
 501   case Op_SubVI: return new SubVINode(n1, n2, vt);
 502   case Op_SubVL: return new SubVLNode(n1, n2, vt);
 503   case Op_SubVF: return new SubVFNode(n1, n2, vt);
 504   case Op_SubVD: return new SubVDNode(n1, n2, vt);
 505 
 506   case Op_MulVB: return new MulVBNode(n1, n2, vt);
 507   case Op_MulVS: return new MulVSNode(n1, n2, vt);
 508   case Op_MulVI: return new MulVINode(n1, n2, vt);
 509   case Op_MulVL: return new MulVLNode(n1, n2, vt);
 510   case Op_MulVF: return new MulVFNode(n1, n2, vt);
 511   case Op_MulVD: return new MulVDNode(n1, n2, vt);
 512 
 513   case Op_DivVF: return new DivVFNode(n1, n2, vt);
 514   case Op_DivVD: return new DivVDNode(n1, n2, vt);
 515 
 516   case Op_MinV: return new MinVNode(n1, n2, vt);
 517   case Op_MaxV: return new MaxVNode(n1, n2, vt);
 518 
 519   case Op_AbsVF: return new AbsVFNode(n1, vt);
 520   case Op_AbsVD: return new AbsVDNode(n1, vt);
 521   case Op_AbsVB: return new AbsVBNode(n1, vt);
 522   case Op_AbsVS: return new AbsVSNode(n1, vt);
 523   case Op_AbsVI: return new AbsVINode(n1, vt);
 524   case Op_AbsVL: return new AbsVLNode(n1, vt);
 525 
 526   case Op_NegVI: return new NegVINode(n1, vt);
 527   case Op_NegVF: return new NegVFNode(n1, vt);
 528   case Op_NegVD: return new NegVDNode(n1, vt);
 529 
 530   case Op_SqrtVF: return new SqrtVFNode(n1, vt);
 531   case Op_SqrtVD: return new SqrtVDNode(n1, vt);
 532 
 533   case Op_PopCountVI: return new PopCountVINode(n1, vt);
 534   case Op_RotateLeftV: return new RotateLeftVNode(n1, n2, vt);
 535   case Op_RotateRightV: return new RotateRightVNode(n1, n2, vt);
 536 
 537   case Op_LShiftVB: return new LShiftVBNode(n1, n2, vt, is_var_shift);
 538   case Op_LShiftVS: return new LShiftVSNode(n1, n2, vt, is_var_shift);
 539   case Op_LShiftVI: return new LShiftVINode(n1, n2, vt, is_var_shift);
 540   case Op_LShiftVL: return new LShiftVLNode(n1, n2, vt, is_var_shift);
 541 
 542   case Op_RShiftVB: return new RShiftVBNode(n1, n2, vt, is_var_shift);
 543   case Op_RShiftVS: return new RShiftVSNode(n1, n2, vt, is_var_shift);
 544   case Op_RShiftVI: return new RShiftVINode(n1, n2, vt, is_var_shift);
 545   case Op_RShiftVL: return new RShiftVLNode(n1, n2, vt, is_var_shift);
 546 
 547   case Op_URShiftVB: return new URShiftVBNode(n1, n2, vt, is_var_shift);
 548   case Op_URShiftVS: return new URShiftVSNode(n1, n2, vt, is_var_shift);
 549   case Op_URShiftVI: return new URShiftVINode(n1, n2, vt, is_var_shift);
 550   case Op_URShiftVL: return new URShiftVLNode(n1, n2, vt, is_var_shift);
 551 
 552   case Op_AndV: return new AndVNode(n1, n2, vt);
 553   case Op_OrV:  return new OrVNode (n1, n2, vt);
 554   case Op_XorV: return new XorVNode(n1, n2, vt);
 555 
 556   case Op_RoundDoubleModeV: return new RoundDoubleModeVNode(n1, n2, vt);
 557 
 558   case Op_MulAddVS2VI: return new MulAddVS2VINode(n1, n2, vt);
 559   default:
 560     fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
 561     return NULL;
 562   }
 563 }
 564 
 565 // Return the vector version of a scalar binary operation node.
 566 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, uint vlen, BasicType bt, bool is_var_shift) {
 567   const TypeVect* vt = TypeVect::make(bt, vlen);
 568   int vopc = VectorNode::opcode(opc, bt);
 569   // This method should not be called for unimplemented vectors.
 570   guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]);
 571   return make(vopc, n1, n2, vt, false, is_var_shift);
 572 }
 573 
 574 // Make a vector node for ternary operation
 575 VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, Node* n3, const TypeVect* vt) {
 576   // This method should not be called for unimplemented vectors.
 577   guarantee(vopc > 0, "vopc must be > 0");
 578   switch (vopc) {
 579   case Op_FmaVD: return new FmaVDNode(n1, n2, n3, vt);
 580   case Op_FmaVF: return new FmaVFNode(n1, n2, n3, vt);
 581   default:
 582     fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
 583     return NULL;
 584   }
 585 }
 586 
 587 // Return the vector version of a scalar ternary operation node.
 588 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, Node* n3, uint vlen, BasicType bt) {
 589   const TypeVect* vt = TypeVect::make(bt, vlen);
 590   int vopc = VectorNode::opcode(opc, bt);
 591   // This method should not be called for unimplemented vectors.
 592   guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]);
 593   return make(vopc, n1, n2, n3, vt);
 594 }
 595 
 596 // Scalar promotion
 597 VectorNode* VectorNode::scalar2vector(Node* s, uint vlen, const Type* opd_t, bool is_mask) {
 598   BasicType bt = opd_t->array_element_basic_type();
 599   if (is_mask && Matcher::match_rule_supported_vector(Op_MaskAll, vlen, bt)) {
 600     const TypeVect* vt = TypeVect::make(opd_t, vlen, true);
 601     return new MaskAllNode(s, vt);
 602   }
 603 
 604   const TypeVect* vt = opd_t->singleton() ? TypeVect::make(opd_t, vlen)
 605                                           : TypeVect::make(bt, vlen);
 606   switch (bt) {
 607   case T_BOOLEAN:
 608   case T_BYTE:
 609     return new ReplicateBNode(s, vt);
 610   case T_CHAR:
 611   case T_SHORT:
 612     return new ReplicateSNode(s, vt);
 613   case T_INT:
 614     return new ReplicateINode(s, vt);
 615   case T_LONG:
 616     return new ReplicateLNode(s, vt);
 617   case T_FLOAT:
 618     return new ReplicateFNode(s, vt);
 619   case T_DOUBLE:
 620     return new ReplicateDNode(s, vt);
 621   default:
 622     fatal("Type '%s' is not supported for vectors", type2name(bt));
 623     return NULL;
 624   }
 625 }
 626 
 627 VectorNode* VectorNode::shift_count(int opc, Node* cnt, uint vlen, BasicType bt) {
 628   // Match shift count type with shift vector type.
 629   const TypeVect* vt = TypeVect::make(bt, vlen);
 630   switch (opc) {
 631   case Op_LShiftI:
 632   case Op_LShiftL:
 633     return new LShiftCntVNode(cnt, vt);
 634   case Op_RShiftI:
 635   case Op_RShiftL:
 636   case Op_URShiftB:
 637   case Op_URShiftS:
 638   case Op_URShiftI:
 639   case Op_URShiftL:
 640     return new RShiftCntVNode(cnt, vt);
 641   default:
 642     fatal("Missed vector creation for '%s'", NodeClassNames[opc]);
 643     return NULL;
 644   }
 645 }
 646 
 647 bool VectorNode::is_vector_rotate(int opc) {
 648   switch (opc) {
 649   case Op_RotateLeftV:
 650   case Op_RotateRightV:
 651     return true;
 652   default:
 653     return false;
 654   }
 655 }
 656 
 657 bool VectorNode::is_vector_shift(int opc) {
 658   assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode");
 659   switch (opc) {
 660   case Op_LShiftVB:
 661   case Op_LShiftVS:
 662   case Op_LShiftVI:
 663   case Op_LShiftVL:
 664   case Op_RShiftVB:
 665   case Op_RShiftVS:
 666   case Op_RShiftVI:
 667   case Op_RShiftVL:
 668   case Op_URShiftVB:
 669   case Op_URShiftVS:
 670   case Op_URShiftVI:
 671   case Op_URShiftVL:
 672     return true;
 673   default:
 674     return false;
 675   }
 676 }
 677 
 678 bool VectorNode::is_vector_shift_count(int opc) {
 679   assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode");
 680   switch (opc) {
 681   case Op_RShiftCntV:
 682   case Op_LShiftCntV:
 683     return true;
 684   default:
 685     return false;
 686   }
 687 }
 688 
 689 static bool is_con_M1(Node* n) {
 690   if (n->is_Con()) {
 691     const Type* t = n->bottom_type();
 692     if (t->isa_int() && t->is_int()->get_con() == -1) {
 693       return true;
 694     }
 695     if (t->isa_long() && t->is_long()->get_con() == -1) {
 696       return true;
 697     }
 698   }
 699   return false;
 700 }
 701 
 702 bool VectorNode::is_all_ones_vector(Node* n) {
 703   switch (n->Opcode()) {
 704   case Op_ReplicateB:
 705   case Op_ReplicateS:
 706   case Op_ReplicateI:
 707   case Op_ReplicateL:
 708     return is_con_M1(n->in(1));
 709   default:
 710     return false;
 711   }
 712 }
 713 
 714 bool VectorNode::is_vector_bitwise_not_pattern(Node* n) {
 715   if (n->Opcode() == Op_XorV) {
 716     return is_all_ones_vector(n->in(1)) ||
 717            is_all_ones_vector(n->in(2));
 718   }
 719   return false;
 720 }
 721 
 722 // Return initial Pack node. Additional operands added with add_opd() calls.
 723 PackNode* PackNode::make(Node* s, uint vlen, BasicType bt) {
 724   const TypeVect* vt = TypeVect::make(bt, vlen);
 725   switch (bt) {
 726   case T_BOOLEAN:
 727   case T_BYTE:
 728     return new PackBNode(s, vt);
 729   case T_CHAR:
 730   case T_SHORT:
 731     return new PackSNode(s, vt);
 732   case T_INT:
 733     return new PackINode(s, vt);
 734   case T_LONG:
 735     return new PackLNode(s, vt);
 736   case T_FLOAT:
 737     return new PackFNode(s, vt);
 738   case T_DOUBLE:
 739     return new PackDNode(s, vt);
 740   default:
 741     fatal("Type '%s' is not supported for vectors", type2name(bt));
 742     return NULL;
 743   }
 744 }
 745 
 746 // Create a binary tree form for Packs. [lo, hi) (half-open) range
 747 PackNode* PackNode::binary_tree_pack(int lo, int hi) {
 748   int ct = hi - lo;
 749   assert(is_power_of_2(ct), "power of 2");
 750   if (ct == 2) {
 751     PackNode* pk = PackNode::make(in(lo), 2, vect_type()->element_basic_type());
 752     pk->add_opd(in(lo+1));
 753     return pk;
 754   } else {
 755     int mid = lo + ct/2;
 756     PackNode* n1 = binary_tree_pack(lo,  mid);
 757     PackNode* n2 = binary_tree_pack(mid, hi );
 758 
 759     BasicType bt = n1->vect_type()->element_basic_type();
 760     assert(bt == n2->vect_type()->element_basic_type(), "should be the same");
 761     switch (bt) {
 762     case T_BOOLEAN:
 763     case T_BYTE:
 764       return new PackSNode(n1, n2, TypeVect::make(T_SHORT, 2));
 765     case T_CHAR:
 766     case T_SHORT:
 767       return new PackINode(n1, n2, TypeVect::make(T_INT, 2));
 768     case T_INT:
 769       return new PackLNode(n1, n2, TypeVect::make(T_LONG, 2));
 770     case T_LONG:
 771       return new Pack2LNode(n1, n2, TypeVect::make(T_LONG, 2));
 772     case T_FLOAT:
 773       return new PackDNode(n1, n2, TypeVect::make(T_DOUBLE, 2));
 774     case T_DOUBLE:
 775       return new Pack2DNode(n1, n2, TypeVect::make(T_DOUBLE, 2));
 776     default:
 777       fatal("Type '%s' is not supported for vectors", type2name(bt));
 778       return NULL;
 779     }
 780   }
 781 }
 782 
 783 // Return the vector version of a scalar load node.
 784 LoadVectorNode* LoadVectorNode::make(int opc, Node* ctl, Node* mem,
 785                                      Node* adr, const TypePtr* atyp,
 786                                      uint vlen, BasicType bt,
 787                                      ControlDependency control_dependency) {
 788   const TypeVect* vt = TypeVect::make(bt, vlen);
 789   return new LoadVectorNode(ctl, mem, adr, atyp, vt, control_dependency);
 790 }
 791 
 792 // Return the vector version of a scalar store node.
 793 StoreVectorNode* StoreVectorNode::make(int opc, Node* ctl, Node* mem,
 794                                        Node* adr, const TypePtr* atyp, Node* val,
 795                                        uint vlen) {
 796   return new StoreVectorNode(ctl, mem, adr, atyp, val);
 797 }
 798 
 799 Node* LoadVectorMaskedNode::Ideal(PhaseGVN* phase, bool can_reshape) {
 800   if (!in(3)->is_top() && in(3)->Opcode() == Op_VectorMaskGen) {
 801     Node* mask_len = in(3)->in(1);
 802     const TypeLong* ty = phase->type(mask_len)->isa_long();
 803     if (ty && ty->is_con()) {
 804       BasicType mask_bt = Matcher::vector_element_basic_type(in(3));
 805       int load_sz = type2aelembytes(mask_bt) * ty->get_con();
 806       assert(load_sz <= MaxVectorSize, "Unexpected load size");
 807       if (load_sz == MaxVectorSize) {
 808         Node* ctr = in(MemNode::Control);
 809         Node* mem = in(MemNode::Memory);
 810         Node* adr = in(MemNode::Address);
 811         return phase->transform(new LoadVectorNode(ctr, mem, adr, adr_type(), vect_type()));
 812       }
 813     }
 814   }
 815   return NULL;
 816 }
 817 
 818 Node* StoreVectorMaskedNode::Ideal(PhaseGVN* phase, bool can_reshape) {
 819   if (!in(4)->is_top() && in(4)->Opcode() == Op_VectorMaskGen) {
 820     Node* mask_len = in(4)->in(1);
 821     const TypeLong* ty = phase->type(mask_len)->isa_long();
 822     if (ty && ty->is_con()) {
 823       BasicType mask_bt = Matcher::vector_element_basic_type(in(4));
 824       int load_sz = type2aelembytes(mask_bt) * ty->get_con();
 825       assert(load_sz <= MaxVectorSize, "Unexpected store size");
 826       if (load_sz == MaxVectorSize) {
 827         Node* ctr = in(MemNode::Control);
 828         Node* mem = in(MemNode::Memory);
 829         Node* adr = in(MemNode::Address);
 830         Node* val = in(MemNode::ValueIn);
 831         return phase->transform(new StoreVectorNode(ctr, mem, adr, adr_type(), val));
 832       }
 833     }
 834   }
 835   return NULL;
 836 }
 837 
 838 int ExtractNode::opcode(BasicType bt) {
 839   switch (bt) {
 840     case T_BOOLEAN: return Op_ExtractUB;
 841     case T_BYTE:    return Op_ExtractB;
 842     case T_CHAR:    return Op_ExtractC;
 843     case T_SHORT:   return Op_ExtractS;
 844     case T_INT:     return Op_ExtractI;
 845     case T_LONG:    return Op_ExtractL;
 846     case T_FLOAT:   return Op_ExtractF;
 847     case T_DOUBLE:  return Op_ExtractD;
 848     default:
 849       assert(false, "wrong type: %s", type2name(bt));
 850       return 0;
 851   }
 852 }
 853 
 854 // Extract a scalar element of vector.
 855 Node* ExtractNode::make(Node* v, uint position, BasicType bt) {
 856   assert((int)position < Matcher::max_vector_size(bt), "pos in range");
 857   ConINode* pos = ConINode::make((int)position);
 858   switch (bt) {
 859   case T_BOOLEAN: return new ExtractUBNode(v, pos);
 860   case T_BYTE:    return new ExtractBNode(v, pos);
 861   case T_CHAR:    return new ExtractCNode(v, pos);
 862   case T_SHORT:   return new ExtractSNode(v, pos);
 863   case T_INT:     return new ExtractINode(v, pos);
 864   case T_LONG:    return new ExtractLNode(v, pos);
 865   case T_FLOAT:   return new ExtractFNode(v, pos);
 866   case T_DOUBLE:  return new ExtractDNode(v, pos);
 867   default:
 868     assert(false, "wrong type: %s", type2name(bt));
 869     return NULL;
 870   }
 871 }
 872 
 873 int ReductionNode::opcode(int opc, BasicType bt) {
 874   int vopc = opc;
 875   switch (opc) {
 876     case Op_AddI:
 877       switch (bt) {
 878         case T_BOOLEAN:
 879         case T_CHAR: return 0;
 880         case T_BYTE:
 881         case T_SHORT:
 882         case T_INT:
 883           vopc = Op_AddReductionVI;
 884           break;
 885         default: ShouldNotReachHere(); return 0;
 886       }
 887       break;
 888     case Op_AddL:
 889       assert(bt == T_LONG, "must be");
 890       vopc = Op_AddReductionVL;
 891       break;
 892     case Op_AddF:
 893       assert(bt == T_FLOAT, "must be");
 894       vopc = Op_AddReductionVF;
 895       break;
 896     case Op_AddD:
 897       assert(bt == T_DOUBLE, "must be");
 898       vopc = Op_AddReductionVD;
 899       break;
 900     case Op_MulI:
 901       switch (bt) {
 902         case T_BOOLEAN:
 903         case T_CHAR: return 0;
 904         case T_BYTE:
 905         case T_SHORT:
 906         case T_INT:
 907           vopc = Op_MulReductionVI;
 908           break;
 909         default: ShouldNotReachHere(); return 0;
 910       }
 911       break;
 912     case Op_MulL:
 913       assert(bt == T_LONG, "must be");
 914       vopc = Op_MulReductionVL;
 915       break;
 916     case Op_MulF:
 917       assert(bt == T_FLOAT, "must be");
 918       vopc = Op_MulReductionVF;
 919       break;
 920     case Op_MulD:
 921       assert(bt == T_DOUBLE, "must be");
 922       vopc = Op_MulReductionVD;
 923       break;
 924     case Op_MinI:
 925       switch (bt) {
 926         case T_BOOLEAN:
 927         case T_CHAR: return 0;
 928         case T_BYTE:
 929         case T_SHORT:
 930         case T_INT:
 931           vopc = Op_MinReductionV;
 932           break;
 933         default: ShouldNotReachHere(); return 0;
 934       }
 935       break;
 936     case Op_MinL:
 937       assert(bt == T_LONG, "must be");
 938       vopc = Op_MinReductionV;
 939       break;
 940     case Op_MinF:
 941       assert(bt == T_FLOAT, "must be");
 942       vopc = Op_MinReductionV;
 943       break;
 944     case Op_MinD:
 945       assert(bt == T_DOUBLE, "must be");
 946       vopc = Op_MinReductionV;
 947       break;
 948     case Op_MaxI:
 949       switch (bt) {
 950         case T_BOOLEAN:
 951         case T_CHAR: return 0;
 952         case T_BYTE:
 953         case T_SHORT:
 954         case T_INT:
 955           vopc = Op_MaxReductionV;
 956           break;
 957         default: ShouldNotReachHere(); return 0;
 958       }
 959       break;
 960     case Op_MaxL:
 961       assert(bt == T_LONG, "must be");
 962       vopc = Op_MaxReductionV;
 963       break;
 964     case Op_MaxF:
 965       assert(bt == T_FLOAT, "must be");
 966       vopc = Op_MaxReductionV;
 967       break;
 968     case Op_MaxD:
 969       assert(bt == T_DOUBLE, "must be");
 970       vopc = Op_MaxReductionV;
 971       break;
 972     case Op_AndI:
 973       switch (bt) {
 974       case T_BOOLEAN:
 975       case T_CHAR: return 0;
 976       case T_BYTE:
 977       case T_SHORT:
 978       case T_INT:
 979         vopc = Op_AndReductionV;
 980         break;
 981       default: ShouldNotReachHere(); return 0;
 982       }
 983       break;
 984     case Op_AndL:
 985       assert(bt == T_LONG, "must be");
 986       vopc = Op_AndReductionV;
 987       break;
 988     case Op_OrI:
 989       switch(bt) {
 990       case T_BOOLEAN:
 991       case T_CHAR: return 0;
 992       case T_BYTE:
 993       case T_SHORT:
 994       case T_INT:
 995         vopc = Op_OrReductionV;
 996         break;
 997       default: ShouldNotReachHere(); return 0;
 998       }
 999       break;
1000     case Op_OrL:
1001       assert(bt == T_LONG, "must be");
1002       vopc = Op_OrReductionV;
1003       break;
1004     case Op_XorI:
1005       switch(bt) {
1006       case T_BOOLEAN:
1007       case T_CHAR: return 0;
1008       case T_BYTE:
1009       case T_SHORT:
1010       case T_INT:
1011         vopc = Op_XorReductionV;
1012         break;
1013       default: ShouldNotReachHere(); return 0;
1014       }
1015       break;
1016     case Op_XorL:
1017       assert(bt == T_LONG, "must be");
1018       vopc = Op_XorReductionV;
1019       break;
1020     default:
1021       break;
1022   }
1023   return vopc;
1024 }
1025 
1026 // Return the appropriate reduction node.
1027 ReductionNode* ReductionNode::make(int opc, Node *ctrl, Node* n1, Node* n2, BasicType bt) {
1028 
1029   int vopc = opcode(opc, bt);
1030 
1031   // This method should not be called for unimplemented vectors.
1032   guarantee(vopc != opc, "Vector for '%s' is not implemented", NodeClassNames[opc]);
1033 
1034   switch (vopc) {
1035   case Op_AddReductionVI: return new AddReductionVINode(ctrl, n1, n2);
1036   case Op_AddReductionVL: return new AddReductionVLNode(ctrl, n1, n2);
1037   case Op_AddReductionVF: return new AddReductionVFNode(ctrl, n1, n2);
1038   case Op_AddReductionVD: return new AddReductionVDNode(ctrl, n1, n2);
1039   case Op_MulReductionVI: return new MulReductionVINode(ctrl, n1, n2);
1040   case Op_MulReductionVL: return new MulReductionVLNode(ctrl, n1, n2);
1041   case Op_MulReductionVF: return new MulReductionVFNode(ctrl, n1, n2);
1042   case Op_MulReductionVD: return new MulReductionVDNode(ctrl, n1, n2);
1043   case Op_MinReductionV:  return new MinReductionVNode(ctrl, n1, n2);
1044   case Op_MaxReductionV:  return new MaxReductionVNode(ctrl, n1, n2);
1045   case Op_AndReductionV:  return new AndReductionVNode(ctrl, n1, n2);
1046   case Op_OrReductionV:   return new OrReductionVNode(ctrl, n1, n2);
1047   case Op_XorReductionV:  return new XorReductionVNode(ctrl, n1, n2);
1048   default:
1049     assert(false, "unknown node: %s", NodeClassNames[vopc]);
1050     return NULL;
1051   }
1052 }
1053 
1054 Node* VectorLoadMaskNode::Identity(PhaseGVN* phase) {
1055   BasicType out_bt = type()->is_vect()->element_basic_type();
1056   if (!Matcher::has_predicated_vectors() && out_bt == T_BOOLEAN) {
1057     return in(1); // redundant conversion
1058   }
1059 
1060   return this;
1061 }
1062 
1063 Node* VectorStoreMaskNode::Identity(PhaseGVN* phase) {
1064   // Identity transformation on boolean vectors.
1065   //   VectorStoreMask (VectorLoadMask bv) elem_size ==> bv
1066   //   vector[n]{bool} => vector[n]{t} => vector[n]{bool}
1067   if (in(1)->Opcode() == Op_VectorLoadMask) {
1068     return in(1)->in(1);
1069   }
1070   return this;
1071 }
1072 
1073 VectorStoreMaskNode* VectorStoreMaskNode::make(PhaseGVN& gvn, Node* in, BasicType in_type, uint num_elem) {
1074   assert(in->bottom_type()->isa_vect(), "sanity");
1075   const TypeVect* vt = TypeVect::make(T_BOOLEAN, num_elem);
1076   int elem_size = type2aelembytes(in_type);
1077   return new VectorStoreMaskNode(in, gvn.intcon(elem_size), vt);
1078 }
1079 
1080 VectorCastNode* VectorCastNode::make(int vopc, Node* n1, BasicType bt, uint vlen) {
1081   const TypeVect* vt = TypeVect::make(bt, vlen);
1082   switch (vopc) {
1083     case Op_VectorCastB2X: return new VectorCastB2XNode(n1, vt);
1084     case Op_VectorCastS2X: return new VectorCastS2XNode(n1, vt);
1085     case Op_VectorCastI2X: return new VectorCastI2XNode(n1, vt);
1086     case Op_VectorCastL2X: return new VectorCastL2XNode(n1, vt);
1087     case Op_VectorCastF2X: return new VectorCastF2XNode(n1, vt);
1088     case Op_VectorCastD2X: return new VectorCastD2XNode(n1, vt);
1089     default:
1090       assert(false, "unknown node: %s", NodeClassNames[vopc]);
1091       return NULL;
1092   }
1093 }
1094 
1095 int VectorCastNode::opcode(BasicType bt) {
1096   switch (bt) {
1097     case T_BYTE:   return Op_VectorCastB2X;
1098     case T_SHORT:  return Op_VectorCastS2X;
1099     case T_INT:    return Op_VectorCastI2X;
1100     case T_LONG:   return Op_VectorCastL2X;
1101     case T_FLOAT:  return Op_VectorCastF2X;
1102     case T_DOUBLE: return Op_VectorCastD2X;
1103     default:
1104       assert(false, "unknown type: %s", type2name(bt));
1105       return 0;
1106   }
1107 }
1108 
1109 Node* VectorCastNode::Identity(PhaseGVN* phase) {
1110   if (!in(1)->is_top()) {
1111     BasicType  in_bt = in(1)->bottom_type()->is_vect()->element_basic_type();
1112     BasicType out_bt = vect_type()->element_basic_type();
1113     if (in_bt == out_bt) {
1114       return in(1); // redundant cast
1115     }
1116   }
1117   return this;
1118 }
1119 
1120 Node* ReductionNode::make_reduction_input(PhaseGVN& gvn, int opc, BasicType bt) {
1121   int vopc = opcode(opc, bt);
1122   guarantee(vopc != opc, "Vector reduction for '%s' is not implemented", NodeClassNames[opc]);
1123 
1124   switch (vopc) {
1125     case Op_AndReductionV:
1126       switch (bt) {
1127         case T_BYTE:
1128         case T_SHORT:
1129         case T_INT:
1130           return gvn.makecon(TypeInt::MINUS_1);
1131         case T_LONG:
1132           return gvn.makecon(TypeLong::MINUS_1);
1133         default:
1134           fatal("Missed vector creation for '%s' as the basic type is not correct.", NodeClassNames[vopc]);
1135           return NULL;
1136       }
1137       break;
1138     case Op_AddReductionVI: // fallthrough
1139     case Op_AddReductionVL: // fallthrough
1140     case Op_AddReductionVF: // fallthrough
1141     case Op_AddReductionVD:
1142     case Op_OrReductionV:
1143     case Op_XorReductionV:
1144       return gvn.zerocon(bt);
1145     case Op_MulReductionVI:
1146       return gvn.makecon(TypeInt::ONE);
1147     case Op_MulReductionVL:
1148       return gvn.makecon(TypeLong::ONE);
1149     case Op_MulReductionVF:
1150       return gvn.makecon(TypeF::ONE);
1151     case Op_MulReductionVD:
1152       return gvn.makecon(TypeD::ONE);
1153     case Op_MinReductionV:
1154       switch (bt) {
1155         case T_BYTE:
1156           return gvn.makecon(TypeInt::make(max_jbyte));
1157         case T_SHORT:
1158           return gvn.makecon(TypeInt::make(max_jshort));
1159         case T_INT:
1160           return gvn.makecon(TypeInt::MAX);
1161         case T_LONG:
1162           return gvn.makecon(TypeLong::MAX);
1163         case T_FLOAT:
1164           return gvn.makecon(TypeF::POS_INF);
1165         case T_DOUBLE:
1166           return gvn.makecon(TypeD::POS_INF);
1167           default: Unimplemented(); return NULL;
1168       }
1169       break;
1170     case Op_MaxReductionV:
1171       switch (bt) {
1172         case T_BYTE:
1173           return gvn.makecon(TypeInt::make(min_jbyte));
1174         case T_SHORT:
1175           return gvn.makecon(TypeInt::make(min_jshort));
1176         case T_INT:
1177           return gvn.makecon(TypeInt::MIN);
1178         case T_LONG:
1179           return gvn.makecon(TypeLong::MIN);
1180         case T_FLOAT:
1181           return gvn.makecon(TypeF::NEG_INF);
1182         case T_DOUBLE:
1183           return gvn.makecon(TypeD::NEG_INF);
1184           default: Unimplemented(); return NULL;
1185       }
1186       break;
1187     default:
1188       fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
1189       return NULL;
1190   }
1191 }
1192 
1193 bool ReductionNode::implemented(int opc, uint vlen, BasicType bt) {
1194   if (is_java_primitive(bt) &&
1195       (vlen > 1) && is_power_of_2(vlen) &&
1196       Matcher::vector_size_supported(bt, vlen)) {
1197     int vopc = ReductionNode::opcode(opc, bt);
1198     return vopc != opc && Matcher::match_rule_supported_vector(vopc, vlen, bt);
1199   }
1200   return false;
1201 }
1202 
1203 MacroLogicVNode* MacroLogicVNode::make(PhaseGVN& gvn, Node* in1, Node* in2, Node* in3,
1204                                        uint truth_table, const TypeVect* vt) {
1205   assert(truth_table <= 0xFF, "invalid");
1206   assert(in1->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
1207   assert(in2->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
1208   assert(in3->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
1209   Node* fn = gvn.intcon(truth_table);
1210   return new MacroLogicVNode(in1, in2, in3, fn, vt);
1211 }
1212 
1213 Node* VectorNode::degenerate_vector_rotate(Node* src, Node* cnt, bool is_rotate_left,
1214                                            int vlen, BasicType bt, PhaseGVN* phase) {
1215   assert(is_integral_type(bt), "sanity");
1216   const TypeVect* vt = TypeVect::make(bt, vlen);
1217 
1218   int shift_mask = (type2aelembytes(bt) * 8) - 1;
1219   int shiftLOpc = (bt == T_LONG) ? Op_LShiftL : Op_LShiftI;
1220   auto urshiftopc = [=]() {
1221     switch(bt) {
1222       case T_INT: return Op_URShiftI;
1223       case T_LONG: return Op_URShiftL;
1224       case T_BYTE: return Op_URShiftB;
1225       case T_SHORT: return Op_URShiftS;
1226       default: return (Opcodes)0;
1227     }
1228   };
1229   int shiftROpc = urshiftopc();
1230 
1231   // Compute shift values for right rotation and
1232   // later swap them in case of left rotation.
1233   Node* shiftRCnt = NULL;
1234   Node* shiftLCnt = NULL;
1235   const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
1236   bool is_binary_vector_op = false;
1237   if (cnt_type && cnt_type->is_con()) {
1238     // Constant shift.
1239     int shift = cnt_type->get_con() & shift_mask;
1240     shiftRCnt = phase->intcon(shift);
1241     shiftLCnt = phase->intcon(shift_mask + 1 - shift);
1242   } else if (VectorNode::is_invariant_vector(cnt)) {
1243     // Scalar variable shift, handle replicates generated by auto vectorizer.
1244     cnt = cnt->in(1);
1245     if (bt == T_LONG) {
1246       // Shift count vector for Rotate vector has long elements too.
1247       if (cnt->Opcode() == Op_ConvI2L) {
1248          cnt = cnt->in(1);
1249       } else {
1250          assert(cnt->bottom_type()->isa_long() &&
1251                 cnt->bottom_type()->is_long()->is_con(), "Long constant expected");
1252          cnt = phase->transform(new ConvL2INode(cnt));
1253       }
1254     }
1255     shiftRCnt = phase->transform(new AndINode(cnt, phase->intcon(shift_mask)));
1256     shiftLCnt = phase->transform(new SubINode(phase->intcon(shift_mask + 1), shiftRCnt));
1257   } else {
1258     // Variable vector rotate count.
1259     assert(Matcher::supports_vector_variable_shifts(), "");
1260 
1261     int subVopc = 0;
1262     int addVopc = 0;
1263     Node* shift_mask_node = NULL;
1264     Node* const_one_node = NULL;
1265 
1266     assert(cnt->bottom_type()->isa_vect(), "Unexpected shift");
1267     const Type* elem_ty = Type::get_const_basic_type(bt);
1268 
1269     if (bt == T_LONG) {
1270       shift_mask_node = phase->longcon(shift_mask);
1271       const_one_node = phase->longcon(1L);
1272       subVopc = VectorNode::opcode(Op_SubL, bt);
1273       addVopc = VectorNode::opcode(Op_AddL, bt);
1274     } else {
1275       shift_mask_node = phase->intcon(shift_mask);
1276       const_one_node = phase->intcon(1);
1277       subVopc = VectorNode::opcode(Op_SubI, bt);
1278       addVopc = VectorNode::opcode(Op_AddI, bt);
1279     }
1280     Node* vector_mask = phase->transform(VectorNode::scalar2vector(shift_mask_node, vlen, elem_ty));
1281     Node* vector_one = phase->transform(VectorNode::scalar2vector(const_one_node, vlen, elem_ty));
1282 
1283     shiftRCnt = cnt;
1284     shiftRCnt = phase->transform(VectorNode::make(Op_AndV, shiftRCnt, vector_mask, vt));
1285     vector_mask = phase->transform(VectorNode::make(addVopc, vector_one, vector_mask, vt));
1286     shiftLCnt = phase->transform(VectorNode::make(subVopc, vector_mask, shiftRCnt, vt));
1287     is_binary_vector_op = true;
1288   }
1289 
1290   // Swap the computed left and right shift counts.
1291   if (is_rotate_left) {
1292     swap(shiftRCnt,shiftLCnt);
1293   }
1294 
1295   if (!is_binary_vector_op) {
1296     shiftLCnt = phase->transform(new LShiftCntVNode(shiftLCnt, vt));
1297     shiftRCnt = phase->transform(new RShiftCntVNode(shiftRCnt, vt));
1298   }
1299 
1300   return new OrVNode(phase->transform(VectorNode::make(shiftLOpc, src, shiftLCnt, vlen, bt, is_binary_vector_op)),
1301                      phase->transform(VectorNode::make(shiftROpc, src, shiftRCnt, vlen, bt, is_binary_vector_op)),
1302                      vt);
1303 }
1304 
1305 Node* RotateLeftVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1306   int vlen = length();
1307   BasicType bt = vect_type()->element_basic_type();
1308   if ((!in(2)->is_Con() && !Matcher::supports_vector_variable_rotates()) ||
1309        !Matcher::match_rule_supported_vector(Op_RotateLeftV, vlen, bt)) {
1310     return VectorNode::degenerate_vector_rotate(in(1), in(2), true, vlen, bt, phase);
1311   }
1312   return NULL;
1313 }
1314 
1315 Node* RotateRightVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1316   int vlen = length();
1317   BasicType bt = vect_type()->element_basic_type();
1318   if ((!in(2)->is_Con() && !Matcher::supports_vector_variable_rotates()) ||
1319        !Matcher::match_rule_supported_vector(Op_RotateRightV, vlen, bt)) {
1320     return VectorNode::degenerate_vector_rotate(in(1), in(2), false, vlen, bt, phase);
1321   }
1322   return NULL;
1323 }
1324 
1325 #ifndef PRODUCT
1326 void VectorMaskCmpNode::dump_spec(outputStream *st) const {
1327   st->print(" %d #", _predicate); _type->dump_on(st);
1328 }
1329 #endif // PRODUCT
1330 
1331 Node* VectorReinterpretNode::Identity(PhaseGVN *phase) {
1332   Node* n = in(1);
1333   if (n->Opcode() == Op_VectorReinterpret) {
1334     // "VectorReinterpret (VectorReinterpret node) ==> node" if:
1335     //   1) Types of 'node' and 'this' are identical
1336     //   2) Truncations are not introduced by the first VectorReinterpret
1337     if (Type::cmp(bottom_type(), n->in(1)->bottom_type()) == 0 &&
1338         length_in_bytes() <= n->bottom_type()->is_vect()->length_in_bytes()) {
1339       return n->in(1);
1340     }
1341   }
1342   return this;
1343 }
1344 
1345 Node* VectorInsertNode::make(Node* vec, Node* new_val, int position) {
1346   assert(position < (int)vec->bottom_type()->is_vect()->length(), "pos in range");
1347   ConINode* pos = ConINode::make(position);
1348   return new VectorInsertNode(vec, new_val, pos, vec->bottom_type()->is_vect());
1349 }
1350 
1351 Node* VectorUnboxNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1352   Node* n = obj()->uncast();
1353   if (EnableVectorReboxing && n->Opcode() == Op_VectorBox) {
1354     if (Type::cmp(bottom_type(), n->in(VectorBoxNode::Value)->bottom_type()) == 0) {
1355       // Handled by VectorUnboxNode::Identity()
1356     } else {
1357       VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
1358       ciKlass* vbox_klass = vbox->box_type()->klass();
1359       const TypeVect* in_vt = vbox->vec_type();
1360       const TypeVect* out_vt = type()->is_vect();
1361 
1362       if (in_vt->length() == out_vt->length()) {
1363         Node* value = vbox->in(VectorBoxNode::Value);
1364 
1365         bool is_vector_mask    = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
1366         bool is_vector_shuffle = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
1367         if (is_vector_mask) {
1368           const TypeVect* vmask_type = TypeVect::makemask(out_vt->element_basic_type(), out_vt->length());
1369           if (in_vt->length_in_bytes() == out_vt->length_in_bytes() &&
1370               Matcher::match_rule_supported_vector(Op_VectorMaskCast, out_vt->length(), out_vt->element_basic_type())) {
1371             // Apply "VectorUnbox (VectorBox vmask) ==> VectorMaskCast (vmask)"
1372             // directly. This could avoid the transformation ordering issue from
1373             // "VectorStoreMask (VectorLoadMask vmask) => vmask".
1374             return new VectorMaskCastNode(value, vmask_type);
1375           }
1376           // VectorUnbox (VectorBox vmask) ==> VectorLoadMask (VectorStoreMask vmask)
1377           value = phase->transform(VectorStoreMaskNode::make(*phase, value, in_vt->element_basic_type(), in_vt->length()));
1378           return new VectorLoadMaskNode(value, vmask_type);
1379         } else if (is_vector_shuffle) {
1380           if (!is_shuffle_to_vector()) {
1381             // VectorUnbox (VectorBox vshuffle) ==> VectorLoadShuffle vshuffle
1382             return new VectorLoadShuffleNode(value, out_vt);
1383           }
1384         } else {
1385           // Vector type mismatch is only supported for masks and shuffles, but sometimes it happens in pathological cases.
1386         }
1387       } else {
1388         // Vector length mismatch.
1389         // Sometimes happen in pathological cases (e.g., when unboxing happens in effectively dead code).
1390       }
1391     }
1392   }
1393   return NULL;
1394 }
1395 
1396 Node* VectorUnboxNode::Identity(PhaseGVN* phase) {
1397   Node* n = obj()->uncast();
1398   if (EnableVectorReboxing && n->Opcode() == Op_VectorBox) {
1399     if (Type::cmp(bottom_type(), n->in(VectorBoxNode::Value)->bottom_type()) == 0) {
1400       return n->in(VectorBoxNode::Value); // VectorUnbox (VectorBox v) ==> v
1401     } else {
1402       // Handled by VectorUnboxNode::Ideal().
1403     }
1404   }
1405   return this;
1406 }
1407 
1408 const TypeFunc* VectorBoxNode::vec_box_type(const TypeInstPtr* box_type) {
1409   const Type** fields = TypeTuple::fields(0);
1410   const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms, fields);
1411 
1412   fields = TypeTuple::fields(1);
1413   fields[TypeFunc::Parms+0] = box_type;
1414   const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
1415 
1416   return TypeFunc::make(domain, range);
1417 }
1418 
1419 Node* ShiftVNode::Identity(PhaseGVN* phase) {
1420   Node* in2 = in(2);
1421   // Shift by ZERO does nothing
1422   if (is_vshift_cnt(in2) && phase->find_int_type(in2->in(1)) == TypeInt::ZERO) {
1423     return in(1);
1424   }
1425   return this;
1426 }
1427 
1428 Node* VectorMaskGenNode::make(Node* length, BasicType mask_bt) {
1429   int max_vector = Matcher::max_vector_size(mask_bt);
1430   const TypeVectMask* t_vmask = TypeVectMask::make(mask_bt, max_vector);
1431   return new VectorMaskGenNode(length, t_vmask);
1432 }
1433 
1434 Node* VectorMaskOpNode::make(Node* mask, const Type* ty, int mopc) {
1435   switch(mopc) {
1436     case Op_VectorMaskTrueCount:
1437       return new VectorMaskTrueCountNode(mask, ty);
1438     case Op_VectorMaskLastTrue:
1439       return new VectorMaskLastTrueNode(mask, ty);
1440     case Op_VectorMaskFirstTrue:
1441       return new VectorMaskFirstTrueNode(mask, ty);
1442     case Op_VectorMaskToLong:
1443       return new VectorMaskToLongNode(mask, ty);
1444     default:
1445       assert(false, "Unhandled operation");
1446   }
1447   return NULL;
1448 }
1449 
1450 Node* VectorMaskToLongNode::Identity(PhaseGVN* phase) {
1451   if (in(1)->Opcode() == Op_VectorLongToMask) {
1452     return in(1)->in(1);
1453   }
1454   return this;
1455 }
1456 
1457 
1458 Node* VectorMaskCastNode::makeCastNode(PhaseGVN* phase, Node* src, const TypeVect* dst_type) {
1459   const TypeVect* src_type = src->bottom_type()->is_vect();
1460   assert(src_type->length() == dst_type->length(), "");
1461 
1462   int num_elem = src_type->length();
1463   BasicType elem_bt_from = src_type->element_basic_type();
1464   BasicType elem_bt_to = dst_type->element_basic_type();
1465 
1466   if (dst_type->isa_vectmask() == NULL && src_type->isa_vectmask() == NULL &&
1467       type2aelembytes(elem_bt_from) != type2aelembytes(elem_bt_to)) {
1468 
1469     Node* op = src;
1470     BasicType new_elem_bt_from = elem_bt_from;
1471     BasicType new_elem_bt_to = elem_bt_to;
1472     if (is_floating_point_type(elem_bt_from)) {
1473       new_elem_bt_from =  elem_bt_from == T_FLOAT ? T_INT : T_LONG;
1474     }
1475     if (is_floating_point_type(elem_bt_to)) {
1476       new_elem_bt_to = elem_bt_to == T_FLOAT ? T_INT : T_LONG;
1477     }
1478 
1479     // Special handling for casting operation involving floating point types.
1480     // Case A) F -> X :=  F -> VectorMaskCast (F->I/L [NOP]) -> VectorCast[I/L]2X
1481     // Case B) X -> F :=  X -> VectorCastX2[I/L] -> VectorMaskCast ([I/L]->F [NOP])
1482     // Case C) F -> F :=  VectorMaskCast (F->I/L [NOP]) -> VectorCast[I/L]2[L/I] -> VectotMaskCast (L/I->F [NOP])
1483 
1484     if (new_elem_bt_from != elem_bt_from) {
1485       const TypeVect* new_src_type = TypeVect::makemask(new_elem_bt_from, num_elem);
1486       op = phase->transform(new VectorMaskCastNode(op, new_src_type));
1487     }
1488 
1489     op = phase->transform(VectorCastNode::make(VectorCastNode::opcode(new_elem_bt_from), op, new_elem_bt_to, num_elem));
1490 
1491     if (new_elem_bt_to != elem_bt_to) {
1492       op = phase->transform(new VectorMaskCastNode(op, dst_type));
1493     }
1494     return op;
1495   } else {
1496     return new VectorMaskCastNode(src, dst_type);
1497   }
1498 }
1499 
1500 Node* VectorLongToMaskNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1501   const TypeVect* dst_type = bottom_type()->is_vect();
1502   if (in(1)->Opcode() == Op_AndL &&
1503       in(1)->in(1)->Opcode() == Op_VectorMaskToLong &&
1504       in(1)->in(2)->bottom_type()->isa_long() &&
1505       in(1)->in(2)->bottom_type()->is_long()->is_con() &&
1506       in(1)->in(2)->bottom_type()->is_long()->get_con() == ((1L << dst_type->length()) - 1)) {
1507       // Different src/dst mask length represents a re-interpretation operation,
1508       // we can however generate a mask casting operation if length matches.
1509      Node* src = in(1)->in(1)->in(1);
1510      if (dst_type->isa_vectmask() == NULL) {
1511        if (src->Opcode() != Op_VectorStoreMask) {
1512          return NULL;
1513        }
1514        src = src->in(1);
1515      }
1516      const TypeVect* src_type = src->bottom_type()->is_vect();
1517      if (src_type->length() == dst_type->length() &&
1518          ((src_type->isa_vectmask() == NULL && dst_type->isa_vectmask() == NULL) ||
1519           (src_type->isa_vectmask() && dst_type->isa_vectmask()))) {
1520        return VectorMaskCastNode::makeCastNode(phase, src, dst_type);
1521      }
1522   }
1523   return NULL;
1524 }
1525 
1526 #ifndef PRODUCT
1527 void VectorBoxAllocateNode::dump_spec(outputStream *st) const {
1528   CallStaticJavaNode::dump_spec(st);
1529 }
1530 
1531 #endif // !PRODUCT