1 /*
   2  * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "ci/ciSymbols.hpp"
  27 #include "classfile/vmSymbols.hpp"
  28 #include "opto/library_call.hpp"
  29 #include "opto/runtime.hpp"
  30 #include "opto/vectornode.hpp"
  31 #include "prims/vectorSupport.hpp"
  32 #include "runtime/stubRoutines.hpp"
  33 
  34 #ifdef ASSERT
  35 static bool is_vector(ciKlass* klass) {
  36   return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
  37 }
  38 
  39 static bool check_vbox(const TypeInstPtr* vbox_type) {
  40   assert(vbox_type->klass_is_exact(), "");
  41 
  42   ciInstanceKlass* ik = vbox_type->instance_klass();
  43   assert(is_vector(ik), "not a vector");
  44 
  45   ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
  46   assert(fd1 != nullptr, "element type info is missing");
  47 
  48   ciConstant val1 = fd1->constant_value();
  49   BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
  50   assert(is_java_primitive(elem_bt), "element type info is missing");
  51 
  52   ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
  53   assert(fd2 != nullptr, "vector length info is missing");
  54 
  55   ciConstant val2 = fd2->constant_value();
  56   assert(val2.as_int() > 0, "vector length info is missing");
  57 
  58   return true;
  59 }
  60 #endif
  61 
  62 static bool is_vector_mask(ciKlass* klass) {
  63   return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
  64 }
  65 
  66 static bool is_vector_shuffle(ciKlass* klass) {
  67   return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
  68 }
  69 
  70 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
  71                                                  VectorMaskUseType mask_use_type, bool has_scalar_args) {
  72   bool is_supported = true;
  73 
  74   // has_scalar_args flag is true only for non-constant scalar shift count,
  75   // since in this case shift needs to be broadcasted.
  76   if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
  77        (has_scalar_args && !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
  78     is_supported = false;
  79   }
  80 
  81   if (is_supported) {
  82     // Check if mask unboxing is supported, this is a two step process which first loads the contents
  83     // of boolean array into vector followed by either lane expansion to match the lane size of masked
  84     // vector operation or populate the predicate register.
  85     if ((mask_use_type & VecMaskUseLoad) != 0) {
  86       if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt) ||
  87           !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
  88       #ifndef PRODUCT
  89         if (C->print_intrinsics()) {
  90           tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
  91                         NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
  92         }
  93       #endif
  94         return false;
  95       }
  96     }
  97 
  98     if ((mask_use_type & VecMaskUsePred) != 0) {
  99       if (!Matcher::has_predicated_vectors() ||
 100           !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
 101       #ifndef PRODUCT
 102         if (C->print_intrinsics()) {
 103           tty->print_cr("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
 104                         NodeClassNames[opc], type2name(elem_bt), num_elem);
 105         }
 106       #endif
 107         return false;
 108       }
 109     }
 110   }
 111 
 112   int lshiftopc, rshiftopc;
 113   switch(elem_bt) {
 114     case T_BYTE:
 115       lshiftopc = Op_LShiftI;
 116       rshiftopc = Op_URShiftB;
 117       break;
 118     case T_SHORT:
 119       lshiftopc = Op_LShiftI;
 120       rshiftopc = Op_URShiftS;
 121       break;
 122     case T_INT:
 123       lshiftopc = Op_LShiftI;
 124       rshiftopc = Op_URShiftI;
 125       break;
 126     case T_LONG:
 127       lshiftopc = Op_LShiftL;
 128       rshiftopc = Op_URShiftL;
 129       break;
 130     default: fatal("Unexpected type: %s", type2name(elem_bt));
 131   }
 132   int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
 133   int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
 134   if (!is_supported &&
 135       arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
 136       arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
 137       arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
 138     is_supported = true;
 139   }
 140   return is_supported;
 141 }
 142 
 143 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
 144   assert(EnableVectorSupport, "");
 145 
 146   PreserveReexecuteState preexecs(this);
 147   jvms()->set_should_reexecute(true);
 148 
 149   VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
 150   set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
 151   make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
 152   set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
 153   set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
 154   Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
 155 
 156   assert(check_vbox(vbox_type), "");
 157   const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
 158   VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
 159   return gvn().transform(vbox);
 160 }
 161 
 162 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) {
 163   assert(EnableVectorSupport, "");
 164   const TypeInstPtr* vbox_type_v = gvn().type(v)->is_instptr();
 165   if (vbox_type->instance_klass() != vbox_type_v->instance_klass()) {
 166     return nullptr; // arguments don't agree on vector shapes
 167   }
 168   if (vbox_type_v->maybe_null()) {
 169     return nullptr; // no nulls are allowed
 170   }
 171   assert(check_vbox(vbox_type), "");
 172   const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
 173   Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector));
 174   return unbox;
 175 }
 176 
 177 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
 178   assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
 179   juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
 180   Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
 181   Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
 182   return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
 183 }
 184 
 185 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
 186   // Check that the operation is valid.
 187   if (sopc <= 0) {
 188 #ifndef PRODUCT
 189     if (C->print_intrinsics()) {
 190       tty->print_cr("  ** Rejected intrinsification because no valid vector op could be extracted");
 191     }
 192 #endif
 193     return false;
 194   }
 195 
 196   if (VectorNode::is_vector_rotate(sopc)) {
 197     if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
 198 #ifndef PRODUCT
 199       if (C->print_intrinsics()) {
 200         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
 201                       NodeClassNames[sopc], type2name(type), num_elem);
 202       }
 203 #endif
 204       return false;
 205     }
 206   } else if (VectorNode::is_vector_integral_negate(sopc)) {
 207     if (!VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, false)) {
 208 #ifndef PRODUCT
 209       if (C->print_intrinsics()) {
 210         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support integral vector negate",
 211                       NodeClassNames[sopc], type2name(type), num_elem);
 212       }
 213 #endif
 214       return false;
 215     }
 216   } else {
 217     // Check that architecture supports this op-size-type combination.
 218     if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
 219 #ifndef PRODUCT
 220       if (C->print_intrinsics()) {
 221         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support it",
 222                       NodeClassNames[sopc], type2name(type), num_elem);
 223       }
 224 #endif
 225       return false;
 226     } else {
 227       assert(Matcher::match_rule_supported(sopc), "must be supported");
 228     }
 229   }
 230 
 231   if (num_elem == 1) {
 232     if (mask_use_type != VecMaskNotUsed) {
 233 #ifndef PRODUCT
 234       if (C->print_intrinsics()) {
 235         tty->print_cr("  ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
 236                       NodeClassNames[sopc], type2name(type), num_elem);
 237       }
 238 #endif
 239       return false;
 240     }
 241 
 242     if (sopc != 0) {
 243       if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
 244 #ifndef PRODUCT
 245         if (C->print_intrinsics()) {
 246           tty->print_cr("  ** Not a svml call or load/store vector op (%s,%s,%d)",
 247                         NodeClassNames[sopc], type2name(type), num_elem);
 248         }
 249 #endif
 250         return false;
 251       }
 252     }
 253   }
 254 
 255   if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
 256       Matcher::supports_vector_variable_shifts() == false) {
 257     if (C->print_intrinsics()) {
 258       tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
 259                     NodeClassNames[sopc], type2name(type), num_elem);
 260     }
 261     return false;
 262   }
 263 
 264   // Check if mask unboxing is supported, this is a two step process which first loads the contents
 265   // of boolean array into vector followed by either lane expansion to match the lane size of masked
 266   // vector operation or populate the predicate register.
 267   if ((mask_use_type & VecMaskUseLoad) != 0) {
 268     if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type) ||
 269         !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
 270     #ifndef PRODUCT
 271       if (C->print_intrinsics()) {
 272         tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
 273                       NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
 274       }
 275     #endif
 276       return false;
 277     }
 278   }
 279 
 280   // Check if mask boxing is supported, this is a two step process which first stores the contents
 281   // of mask vector / predicate register into a boolean vector followed by vector store operation to
 282   // transfer the contents to underlined storage of mask boxes which is a boolean array.
 283   if ((mask_use_type & VecMaskUseStore) != 0) {
 284     if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type) ||
 285         !Matcher::match_rule_supported_vector(Op_StoreVector, num_elem, T_BOOLEAN)) {
 286     #ifndef PRODUCT
 287       if (C->print_intrinsics()) {
 288         tty->print_cr("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
 289                       NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
 290       }
 291     #endif
 292       return false;
 293     }
 294   }
 295 
 296   if ((mask_use_type & VecMaskUsePred) != 0) {
 297     bool is_supported = false;
 298     if (Matcher::has_predicated_vectors()) {
 299       if (VectorNode::is_vector_integral_negate(sopc)) {
 300         is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
 301       } else {
 302         is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
 303       }
 304     }
 305 
 306     if (!is_supported) {
 307     #ifndef PRODUCT
 308       if (C->print_intrinsics()) {
 309         tty->print_cr("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
 310                       NodeClassNames[sopc], type2name(type), num_elem);
 311       }
 312     #endif
 313       return false;
 314     }
 315   }
 316 
 317   return true;
 318 }
 319 
 320 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
 321   if (vec_klass->const_oop() == nullptr) {
 322     return false; // uninitialized or some kind of unsafe access
 323   }
 324   assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != nullptr, "klass instance expected");
 325   ciInstanceKlass* klass =  vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
 326   return klass->is_initialized();
 327 }
 328 
 329 // public static
 330 // <V extends Vector<E>,
 331 //  M extends VectorMask<E>,
 332 //  E>
 333 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 334 //           int length, V v, M m,
 335 //           UnaryOperation<V, M> defaultImpl)
 336 //
 337 // public static
 338 // <V,
 339 //  M extends VectorMask<E>,
 340 //  E>
 341 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 342 //            int length, V v1, V v2, M m,
 343 //            BinaryOperation<V, M> defaultImpl)
 344 //
 345 // public static
 346 // <V extends Vector<E>,
 347 //  M extends VectorMask<E>,
 348 //  E>
 349 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 350 //             int length, V v1, V v2, V v3, M m,
 351 //             TernaryOperation<V, M> defaultImpl)
 352 //
 353 bool LibraryCallKit::inline_vector_nary_operation(int n) {
 354   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
 355   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
 356   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
 357   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
 358   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
 359 
 360   if (opr          == nullptr || !opr->is_con() ||
 361       vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
 362       elem_klass   == nullptr || elem_klass->const_oop()   == nullptr ||
 363       vlen         == nullptr || !vlen->is_con()) {
 364     if (C->print_intrinsics()) {
 365       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
 366                     NodeClassNames[argument(0)->Opcode()],
 367                     NodeClassNames[argument(1)->Opcode()],
 368                     NodeClassNames[argument(3)->Opcode()],
 369                     NodeClassNames[argument(4)->Opcode()]);
 370     }
 371     return false; // not enough info for intrinsification
 372   }
 373 
 374   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 375   if (!elem_type->is_primitive_type()) {
 376     if (C->print_intrinsics()) {
 377       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 378     }
 379     return false; // should be primitive type
 380   }
 381   if (!is_klass_initialized(vector_klass)) {
 382     if (C->print_intrinsics()) {
 383       tty->print_cr("  ** klass argument not initialized");
 384     }
 385     return false;
 386   }
 387 
 388   // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
 389   // is used to control this operation.
 390   const Type* vmask_type = gvn().type(argument(n + 5));
 391   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
 392   if (is_masked_op) {
 393     if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
 394       if (C->print_intrinsics()) {
 395         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
 396       }
 397       return false; // not enough info for intrinsification
 398     }
 399 
 400     if (!is_klass_initialized(mask_klass)) {
 401       if (C->print_intrinsics()) {
 402         tty->print_cr("  ** mask klass argument not initialized");
 403       }
 404       return false;
 405     }
 406 
 407     if (vmask_type->maybe_null()) {
 408       if (C->print_intrinsics()) {
 409         tty->print_cr("  ** null mask values are not allowed for masked op");
 410       }
 411       return false;
 412     }
 413   }
 414 
 415   BasicType elem_bt = elem_type->basic_type();
 416   int num_elem = vlen->get_con();
 417   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
 418   int sopc = VectorNode::opcode(opc, elem_bt);
 419   if ((opc != Op_CallLeafVector) && (sopc == 0)) {
 420     if (C->print_intrinsics()) {
 421       tty->print_cr("  ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
 422     }
 423     return false; // operation not supported
 424   }
 425   if (num_elem == 1) {
 426     if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
 427       if (C->print_intrinsics()) {
 428         tty->print_cr("  ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
 429                       n, opc, num_elem, type2name(elem_bt));
 430       }
 431       return false;
 432     }
 433   }
 434   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 435   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 436 
 437   if (is_vector_mask(vbox_klass)) {
 438     assert(!is_masked_op, "mask operations do not need mask to control");
 439   }
 440 
 441   if (opc == Op_CallLeafVector) {
 442     if (!UseVectorStubs) {
 443       if (C->print_intrinsics()) {
 444         tty->print_cr("  ** vector stubs support is disabled");
 445       }
 446       return false;
 447     }
 448     if (!Matcher::supports_vector_calling_convention()) {
 449       if (C->print_intrinsics()) {
 450         tty->print_cr("  ** no vector calling conventions supported");
 451       }
 452       return false;
 453     }
 454     if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
 455       if (C->print_intrinsics()) {
 456         tty->print_cr("  ** vector size (vlen=%d, etype=%s) is not supported",
 457                       num_elem, type2name(elem_bt));
 458       }
 459       return false;
 460     }
 461   }
 462 
 463   // When using mask, mask use type needs to be VecMaskUseLoad.
 464   VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
 465                                       : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
 466   if ((sopc != 0) && !arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
 467     if (C->print_intrinsics()) {
 468       tty->print_cr("  ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
 469                     n, sopc, num_elem, type2name(elem_bt),
 470                     is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
 471     }
 472     return false; // not supported
 473   }
 474 
 475   // Return true if current platform has implemented the masked operation with predicate feature.
 476   bool use_predicate = is_masked_op && sopc != 0 && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
 477   if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
 478     if (C->print_intrinsics()) {
 479       tty->print_cr("  ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
 480                     n, sopc, num_elem, type2name(elem_bt));
 481     }
 482     return false;
 483   }
 484 
 485   Node* opd1 = nullptr; Node* opd2 = nullptr; Node* opd3 = nullptr;
 486   switch (n) {
 487     case 3: {
 488       opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
 489       if (opd3 == nullptr) {
 490         if (C->print_intrinsics()) {
 491           tty->print_cr("  ** unbox failed v3=%s",
 492                         NodeClassNames[argument(7)->Opcode()]);
 493         }
 494         return false;
 495       }
 496       // fall-through
 497     }
 498     case 2: {
 499       opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
 500       if (opd2 == nullptr) {
 501         if (C->print_intrinsics()) {
 502           tty->print_cr("  ** unbox failed v2=%s",
 503                         NodeClassNames[argument(6)->Opcode()]);
 504         }
 505         return false;
 506       }
 507       // fall-through
 508     }
 509     case 1: {
 510       opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
 511       if (opd1 == nullptr) {
 512         if (C->print_intrinsics()) {
 513           tty->print_cr("  ** unbox failed v1=%s",
 514                         NodeClassNames[argument(5)->Opcode()]);
 515         }
 516         return false;
 517       }
 518       break;
 519     }
 520     default: fatal("unsupported arity: %d", n);
 521   }
 522 
 523   Node* mask = nullptr;
 524   if (is_masked_op) {
 525     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
 526     assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
 527     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
 528     mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
 529     if (mask == nullptr) {
 530       if (C->print_intrinsics()) {
 531         tty->print_cr("  ** unbox failed mask=%s",
 532                       NodeClassNames[argument(n + 5)->Opcode()]);
 533       }
 534       return false;
 535     }
 536   }
 537 
 538   Node* operation = nullptr;
 539   if (opc == Op_CallLeafVector) {
 540     assert(UseVectorStubs, "sanity");
 541     operation = gen_call_to_svml(opr->get_con(), elem_bt, num_elem, opd1, opd2);
 542     if (operation == nullptr) {
 543       if (C->print_intrinsics()) {
 544         tty->print_cr("  ** svml call failed for %s_%s_%d",
 545                          (elem_bt == T_FLOAT)?"float":"double",
 546                          VectorSupport::svmlname[opr->get_con() - VectorSupport::VECTOR_OP_SVML_START],
 547                          num_elem * type2aelembytes(elem_bt));
 548       }
 549       return false;
 550      }
 551   } else {
 552     const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
 553     switch (n) {
 554       case 1:
 555       case 2: {
 556         operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc));
 557         break;
 558       }
 559       case 3: {
 560         operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
 561         break;
 562       }
 563       default: fatal("unsupported arity: %d", n);
 564     }
 565   }
 566 
 567   if (is_masked_op && mask != nullptr) {
 568     if (use_predicate) {
 569       operation->add_req(mask);
 570       operation->add_flag(Node::Flag_is_predicated_vector);
 571     } else {
 572       operation->add_flag(Node::Flag_is_predicated_using_blend);
 573       operation = gvn().transform(operation);
 574       operation = new VectorBlendNode(opd1, operation, mask);
 575     }
 576   }
 577   operation = gvn().transform(operation);
 578 
 579   // Wrap it up in VectorBox to keep object type information.
 580   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
 581   set_result(vbox);
 582   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 583   return true;
 584 }
 585 
 586 // <Sh extends VectorShuffle<E>,  E>
 587 //  Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
 588 //                  int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
 589 bool LibraryCallKit::inline_vector_shuffle_iota() {
 590   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
 591   const TypeInt*     vlen          = gvn().type(argument(3))->isa_int();
 592   const TypeInt*     start_val     = gvn().type(argument(4))->isa_int();
 593   const TypeInt*     step_val      = gvn().type(argument(5))->isa_int();
 594   const TypeInt*     wrap          = gvn().type(argument(6))->isa_int();
 595 
 596   if (shuffle_klass == nullptr || shuffle_klass->const_oop() == nullptr ||
 597       vlen          == nullptr || !vlen->is_con() ||
 598       start_val     == nullptr ||
 599       step_val      == nullptr ||
 600       wrap          == nullptr || !wrap->is_con()) {
 601     return false; // not enough info for intrinsification
 602   }
 603 
 604   if (!is_klass_initialized(shuffle_klass)) {
 605     if (C->print_intrinsics()) {
 606       tty->print_cr("  ** klass argument not initialized");
 607     }
 608     return false;
 609   }
 610 
 611   int do_wrap = wrap->get_con();
 612   int num_elem = vlen->get_con();
 613   BasicType elem_bt = T_BYTE;
 614 
 615   bool effective_indices_in_range = false;
 616   if (start_val->is_con() && step_val->is_con()) {
 617     int effective_min_index = start_val->get_con();
 618     int effective_max_index = start_val->get_con() + step_val->get_con() * (num_elem - 1);
 619     effective_indices_in_range = effective_max_index >= effective_min_index && effective_min_index >= -128 && effective_max_index <= 127;
 620   }
 621 
 622   if (!do_wrap && !effective_indices_in_range) {
 623     // Disable instrinsification for unwrapped shuffle iota if start/step
 624     // values are non-constant OR if intermediate result overflows byte value range.
 625     return false;
 626   }
 627 
 628   if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed)           ||
 629       !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)            ||
 630       !arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
 631       !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed)) {
 632     return false;
 633   }
 634 
 635   if (!do_wrap &&
 636       (!arch_supports_vector(Op_SubVB, num_elem, elem_bt, VecMaskNotUsed)       ||
 637       !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskNotUsed)  ||
 638       !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskNotUsed))) {
 639     return false;
 640   }
 641 
 642   bool step_multiply = !step_val->is_con() || !is_power_of_2(step_val->get_con());
 643   if ((step_multiply && !arch_supports_vector(Op_MulVB, num_elem, elem_bt, VecMaskNotUsed)) ||
 644       (!step_multiply && !arch_supports_vector(Op_LShiftVB, num_elem, elem_bt, VecMaskNotUsed))) {
 645     return false;
 646   }
 647 
 648   const Type * type_bt = Type::get_const_basic_type(elem_bt);
 649   const TypeVect * vt  = TypeVect::make(type_bt, num_elem);
 650 
 651   Node* res = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
 652 
 653   Node* start = argument(4);
 654   Node* step  = argument(5);
 655 
 656   if (step_multiply) {
 657     Node* bcast_step     = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt));
 658     res = gvn().transform(VectorNode::make(Op_MulVB, res, bcast_step, vt));
 659   } else if (step_val->get_con() > 1) {
 660     Node* cnt = gvn().makecon(TypeInt::make(log2i_exact(step_val->get_con())));
 661     Node* shift_cnt = vector_shift_count(cnt, Op_LShiftI, elem_bt, num_elem);
 662     res = gvn().transform(VectorNode::make(Op_LShiftVB, res, shift_cnt, vt));
 663   }
 664 
 665   if (!start_val->is_con() || start_val->get_con() != 0) {
 666     Node* bcast_start    = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt));
 667     res = gvn().transform(VectorNode::make(Op_AddVB, res, bcast_start, vt));
 668   }
 669 
 670   Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1));
 671   Node * bcast_mod  = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt));
 672 
 673   if (do_wrap)  {
 674     // Wrap the indices greater than lane count.
 675      res = gvn().transform(VectorNode::make(Op_AndV, res, bcast_mod, vt));
 676   } else {
 677     ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::ugt));
 678     Node * lane_cnt  = gvn().makecon(TypeInt::make(num_elem));
 679     Node * bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt));
 680     const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
 681     Node* mask = gvn().transform(new VectorMaskCmpNode(BoolTest::ugt, bcast_lane_cnt, res, pred_node, vmask_type));
 682 
 683     // Make the indices greater than lane count as -ve values to match the java side implementation.
 684     res = gvn().transform(VectorNode::make(Op_AndV, res, bcast_mod, vt));
 685     Node * biased_val = gvn().transform(VectorNode::make(Op_SubVB, res, bcast_lane_cnt, vt));
 686     res = gvn().transform(new VectorBlendNode(biased_val, res, mask));
 687   }
 688 
 689   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 690   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 691 
 692   // Wrap it up in VectorBox to keep object type information.
 693   res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
 694   set_result(res);
 695   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 696   return true;
 697 }
 698 
 699 // <E, M>
 700 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
 701 //                          int length, M m, VectorMaskOp<M> defaultImpl)
 702 bool LibraryCallKit::inline_vector_mask_operation() {
 703   const TypeInt*     oper       = gvn().type(argument(0))->isa_int();
 704   const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
 705   const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
 706   const TypeInt*     vlen       = gvn().type(argument(3))->isa_int();
 707   Node*              mask       = argument(4);
 708 
 709   if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
 710       elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
 711       vlen       == nullptr || !vlen->is_con() ||
 712       oper       == nullptr || !oper->is_con() ||
 713       mask->is_top()) {
 714     return false; // dead code
 715   }
 716 
 717   if (!is_klass_initialized(mask_klass)) {
 718     if (C->print_intrinsics()) {
 719       tty->print_cr("  ** klass argument not initialized");
 720     }
 721     return false;
 722   }
 723 
 724   int num_elem = vlen->get_con();
 725   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 726   BasicType elem_bt = elem_type->basic_type();
 727 
 728   int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
 729   if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
 730     if (C->print_intrinsics()) {
 731       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 732                     mopc, num_elem, type2name(elem_bt));
 733     }
 734     return false; // not supported
 735   }
 736 
 737   const Type* elem_ty = Type::get_const_basic_type(elem_bt);
 738   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
 739   const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
 740   Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
 741   if (mask_vec == nullptr) {
 742     if (C->print_intrinsics()) {
 743         tty->print_cr("  ** unbox failed mask=%s",
 744                       NodeClassNames[argument(4)->Opcode()]);
 745     }
 746     return false;
 747   }
 748 
 749   if (mask_vec->bottom_type()->isa_vectmask() == nullptr) {
 750     mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
 751   }
 752   const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
 753   Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
 754   if (mopc != Op_VectorMaskToLong) {
 755     maskoper = ConvI2L(maskoper);
 756   }
 757   set_result(maskoper);
 758 
 759   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 760   return true;
 761 }
 762 
 763 // public static
 764 // <V,
 765 //  Sh extends VectorShuffle<E>,
 766 //  E>
 767 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
 768 //                   Class<? extends Sh> shuffleClass, Sh s, int length,
 769 //                   ShuffleToVectorOperation<V, Sh, E> defaultImpl)
 770 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
 771   const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
 772   const TypeInstPtr* elem_klass    = gvn().type(argument(1))->isa_instptr();
 773   const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
 774   Node*              shuffle       = argument(3);
 775   const TypeInt*     vlen          = gvn().type(argument(4))->isa_int();
 776 
 777   if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) {
 778     return false; // dead code
 779   }
 780   if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr || !vlen->is_con()) {
 781     return false; // not enough info for intrinsification
 782   }
 783   if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
 784     if (C->print_intrinsics()) {
 785       tty->print_cr("  ** klass argument not initialized");
 786     }
 787     return false;
 788   }
 789 
 790   int num_elem = vlen->get_con();
 791   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 792   BasicType elem_bt = elem_type->basic_type();
 793 
 794   if (num_elem < 4) {
 795     return false;
 796   }
 797 
 798   int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE
 799   // Make sure that cast is implemented to particular type/size combination.
 800   if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
 801     if (C->print_intrinsics()) {
 802       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 803         cast_vopc, num_elem, type2name(elem_bt));
 804     }
 805     return false;
 806   }
 807 
 808   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 809   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 810 
 811   // Unbox shuffle with true flag to indicate its load shuffle to vector
 812   // shuffle is a byte array
 813   Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, T_BYTE, num_elem, true);
 814 
 815   // cast byte to target element type
 816   shuffle_vec = gvn().transform(VectorCastNode::make(cast_vopc, shuffle_vec, elem_bt, num_elem));
 817 
 818   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 819   const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 820 
 821   // Box vector
 822   Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem);
 823   set_result(res);
 824   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 825   return true;
 826 }
 827 
 828 // public static
 829 // <M,
 830 //  S extends VectorSpecies<E>,
 831 //  E>
 832 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
 833 //                    long bits, int mode, S s,
 834 //                    BroadcastOperation<M, E, S> defaultImpl)
 835 bool LibraryCallKit::inline_vector_frombits_coerced() {
 836   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 837   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 838   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 839   const TypeLong*    bits_type    = gvn().type(argument(3))->isa_long();
 840   // Mode argument determines the mode of operation it can take following values:-
 841   // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
 842   // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
 843   const TypeInt*     mode         = gvn().type(argument(5))->isa_int();
 844 
 845   if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
 846       elem_klass   == nullptr || elem_klass->const_oop()   == nullptr ||
 847       vlen         == nullptr || !vlen->is_con() ||
 848       bits_type    == nullptr ||
 849       mode         == nullptr || !mode->is_con()) {
 850     if (C->print_intrinsics()) {
 851       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
 852                     NodeClassNames[argument(0)->Opcode()],
 853                     NodeClassNames[argument(1)->Opcode()],
 854                     NodeClassNames[argument(2)->Opcode()],
 855                     NodeClassNames[argument(5)->Opcode()]);
 856     }
 857     return false; // not enough info for intrinsification
 858   }
 859 
 860   if (!is_klass_initialized(vector_klass)) {
 861     if (C->print_intrinsics()) {
 862       tty->print_cr("  ** klass argument not initialized");
 863     }
 864     return false;
 865   }
 866   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 867   if (!elem_type->is_primitive_type()) {
 868     if (C->print_intrinsics()) {
 869       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 870     }
 871     return false; // should be primitive type
 872   }
 873   BasicType elem_bt = elem_type->basic_type();
 874   int num_elem = vlen->get_con();
 875   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 876   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 877 
 878   bool is_mask = is_vector_mask(vbox_klass);
 879   int  bcast_mode = mode->get_con();
 880   VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
 881   int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : Op_Replicate;
 882 
 883   if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
 884     if (C->print_intrinsics()) {
 885       tty->print_cr("  ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
 886                     num_elem, type2name(elem_bt),
 887                     is_mask ? 1 : 0,
 888                     bcast_mode);
 889     }
 890     return false; // not supported
 891   }
 892 
 893   Node* broadcast = nullptr;
 894   Node* bits = argument(3);
 895   Node* elem = bits;
 896 
 897   if (opc == Op_VectorLongToMask) {
 898     const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
 899     if (vt->isa_vectmask()) {
 900       broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
 901     } else {
 902       const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
 903       broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
 904       broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
 905     }
 906   } else {
 907     switch (elem_bt) {
 908       case T_BOOLEAN: // fall-through
 909       case T_BYTE:    // fall-through
 910       case T_SHORT:   // fall-through
 911       case T_CHAR:    // fall-through
 912       case T_INT: {
 913         elem = gvn().transform(new ConvL2INode(bits));
 914         break;
 915       }
 916       case T_DOUBLE: {
 917         elem = gvn().transform(new MoveL2DNode(bits));
 918         break;
 919       }
 920       case T_FLOAT: {
 921         bits = gvn().transform(new ConvL2INode(bits));
 922         elem = gvn().transform(new MoveI2FNode(bits));
 923         break;
 924       }
 925       case T_LONG: {
 926         // no conversion needed
 927         break;
 928       }
 929       default: fatal("%s", type2name(elem_bt));
 930     }
 931     broadcast = VectorNode::scalar2vector(elem, num_elem, Type::get_const_basic_type(elem_bt), is_mask);
 932     broadcast = gvn().transform(broadcast);
 933   }
 934 
 935   Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
 936   set_result(box);
 937   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 938   return true;
 939 }
 940 
 941 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type, bool mismatched_ms) {
 942   assert(arr_type != nullptr, "unexpected");
 943   BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
 944   if (elem_bt == arr_elem_bt) {
 945     return true;
 946   } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
 947     // Load/store of short vector from/to char[] is supported
 948     return true;
 949   } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
 950     // Load/store of byte vector from/to boolean[] is supported
 951     return true;
 952   } else {
 953     return mismatched_ms;
 954   }
 955 }
 956 
 957 //  public static
 958 //  <C,
 959 //   VM extends VectorPayload,
 960 //   E,
 961 //   S extends VectorSpecies<E>>
 962 //  VM load(Class<? extends VM> vmClass, Class<E> eClass,
 963 //          int length,
 964 //          Object base, long offset,            // Unsafe addressing
 965 //          boolean fromSegment,
 966 //          C container, long index, S s,        // Arguments for default implementation
 967 //          LoadOperation<C, VM, S> defaultImpl) {
 968 //  public static
 969 //  <C,
 970 //   V extends VectorPayload>
 971 //  void store(Class<?> vClass, Class<?> eClass,
 972 //             int length,
 973 //             Object base, long offset,        // Unsafe addressing
 974 //             boolean fromSegment,
 975 //             V v, C container, long index,    // Arguments for default implementation
 976 //             StoreVectorOperation<C, V> defaultImpl) {
 977 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
 978   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 979   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 980   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 981   const TypeInt*     from_ms      = gvn().type(argument(6))->isa_int();
 982 
 983   if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
 984       elem_klass   == nullptr || elem_klass->const_oop()   == nullptr ||
 985       vlen         == nullptr || !vlen->is_con() ||
 986       from_ms      == nullptr || !from_ms->is_con()) {
 987     if (C->print_intrinsics()) {
 988       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
 989                     NodeClassNames[argument(0)->Opcode()],
 990                     NodeClassNames[argument(1)->Opcode()],
 991                     NodeClassNames[argument(2)->Opcode()],
 992                     NodeClassNames[argument(6)->Opcode()]);
 993     }
 994     return false; // not enough info for intrinsification
 995   }
 996   if (!is_klass_initialized(vector_klass)) {
 997     if (C->print_intrinsics()) {
 998       tty->print_cr("  ** klass argument not initialized");
 999     }
1000     return false;
1001   }
1002 
1003   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1004   if (!elem_type->is_primitive_type()) {
1005     if (C->print_intrinsics()) {
1006       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1007     }
1008     return false; // should be primitive type
1009   }
1010   BasicType elem_bt = elem_type->basic_type();
1011   int num_elem = vlen->get_con();
1012 
1013   // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
1014   if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
1015     if (C->print_intrinsics()) {
1016       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
1017                     is_store, is_store ? "store" : "load",
1018                     num_elem, type2name(elem_bt));
1019     }
1020     return false; // not supported
1021   }
1022 
1023   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1024   bool is_mask = is_vector_mask(vbox_klass);
1025 
1026   Node* base = argument(3);
1027   Node* offset = ConvL2X(argument(4));
1028 
1029   // Save state and restore on bailout
1030   uint old_sp = sp();
1031   SafePointNode* old_map = clone_map();
1032 
1033   Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
1034 
1035   // The memory barrier checks are based on ones for unsafe access.
1036   // This is not 1-1 implementation.
1037   const Type *const base_type = gvn().type(base);
1038 
1039   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1040   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1041 
1042   const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
1043   const bool in_heap   = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
1044 
1045   const bool is_mixed_access = !in_heap && !in_native;
1046 
1047   const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == nullptr);
1048 
1049   const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
1050 
1051   bool mismatched_ms = from_ms->get_con() && !is_mask && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
1052   BasicType mem_elem_bt = mismatched_ms ? arr_type->elem()->array_element_basic_type() : elem_bt;
1053   int mem_num_elem = mismatched_ms ? (num_elem * type2aelembytes(elem_bt)) / type2aelembytes(mem_elem_bt) : num_elem;
1054   if (arr_type != nullptr && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
1055     if (C->print_intrinsics()) {
1056       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1057                     is_store, is_store ? "store" : "load",
1058                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1059     }
1060     set_map(old_map);
1061     set_sp(old_sp);
1062     return false;
1063   }
1064 
1065   // In case of mismatched memory segment accesses, we need to double check that the source type memory operations are supported by backend.
1066   if (mismatched_ms) {
1067     if (is_store) {
1068       if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskNotUsed)
1069           || !arch_supports_vector(Op_VectorReinterpret, mem_num_elem, mem_elem_bt, VecMaskNotUsed)) {
1070         if (C->print_intrinsics()) {
1071           tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
1072                         is_store, "store",
1073                         num_elem, type2name(elem_bt));
1074         }
1075         set_map(old_map);
1076         set_sp(old_sp);
1077         return false; // not supported
1078       }
1079     } else {
1080       if (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed)
1081           || !arch_supports_vector(Op_VectorReinterpret, num_elem, elem_bt, VecMaskNotUsed)) {
1082         if (C->print_intrinsics()) {
1083           tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
1084                         is_store, "load",
1085                         mem_num_elem, type2name(mem_elem_bt));
1086         }
1087         set_map(old_map);
1088         set_sp(old_sp);
1089         return false; // not supported
1090       }
1091     }
1092   }
1093   if (is_mask) {
1094     if (!is_store) {
1095       if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1096         set_map(old_map);
1097         set_sp(old_sp);
1098         return false; // not supported
1099       }
1100     } else {
1101       if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
1102         set_map(old_map);
1103         set_sp(old_sp);
1104         return false; // not supported
1105       }
1106     }
1107   }
1108 
1109   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1110 
1111   if (needs_cpu_membar) {
1112     insert_mem_bar(Op_MemBarCPUOrder);
1113   }
1114 
1115   if (is_store) {
1116     Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
1117     if (val == nullptr) {
1118       set_map(old_map);
1119       set_sp(old_sp);
1120       return false; // operand unboxing failed
1121     }
1122     set_all_memory(reset_memory());
1123 
1124     // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
1125     int store_num_elem = num_elem;
1126     if (mismatched_ms) {
1127       store_num_elem = mem_num_elem;
1128       const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, store_num_elem);
1129       val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1130     }
1131     if (is_mask) {
1132       val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
1133     }
1134     Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
1135     set_memory(vstore, addr_type);
1136   } else {
1137     // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
1138     Node* vload = nullptr;
1139     if (mismatched_ms) {
1140       vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1141       const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1142       vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1143     } else {
1144       // Special handle for masks
1145       if (is_mask) {
1146         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
1147         vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
1148       } else {
1149         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
1150       }
1151     }
1152     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1153     set_result(box);
1154   }
1155 
1156   destruct_map_clone(old_map);
1157 
1158   if (needs_cpu_membar) {
1159     insert_mem_bar(Op_MemBarCPUOrder);
1160   }
1161 
1162   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1163   return true;
1164 }
1165 
1166 //  public static
1167 //  <C,
1168 //   V extends Vector<?>,
1169 //   E,
1170 //   S extends VectorSpecies<E>,
1171 //   M extends VectorMask<E>>
1172 //  V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1173 //               int length, Object base, long offset,          // Unsafe addressing
1174 //               boolean fromSegment,
1175 //               M m, int offsetInRange,
1176 //               C container, long index, S s,                  // Arguments for default implementation
1177 //               LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1178 //  public static
1179 //  <C,
1180 //   V extends Vector<E>,
1181 //   M extends VectorMask<E>,
1182 //   E>
1183 //  void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1184 //                   int length,
1185 //                   Object base, long offset,                  // Unsafe addressing
1186 //                   boolean fromSegment,
1187 //                   V v, M m, C container, long index,         // Arguments for default implementation
1188 //                   StoreVectorMaskedOperation<C, V, M> defaultImpl) {
1189 
1190 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1191   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1192   const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1193   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1194   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1195   const TypeInt*     from_ms      = gvn().type(argument(7))->isa_int();
1196 
1197   if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1198       mask_klass   == nullptr || mask_klass->const_oop()   == nullptr ||
1199       elem_klass   == nullptr || elem_klass->const_oop()   == nullptr ||
1200       vlen         == nullptr || !vlen->is_con() ||
1201       from_ms      == nullptr || !from_ms->is_con()) {
1202     if (C->print_intrinsics()) {
1203       tty->print_cr("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1204                     NodeClassNames[argument(0)->Opcode()],
1205                     NodeClassNames[argument(1)->Opcode()],
1206                     NodeClassNames[argument(2)->Opcode()],
1207                     NodeClassNames[argument(3)->Opcode()],
1208                     NodeClassNames[argument(7)->Opcode()]);
1209     }
1210     return false; // not enough info for intrinsification
1211   }
1212   if (!is_klass_initialized(vector_klass)) {
1213     if (C->print_intrinsics()) {
1214       tty->print_cr("  ** klass argument not initialized");
1215     }
1216     return false;
1217   }
1218 
1219   if (!is_klass_initialized(mask_klass)) {
1220     if (C->print_intrinsics()) {
1221       tty->print_cr("  ** mask klass argument not initialized");
1222     }
1223     return false;
1224   }
1225 
1226   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1227   if (!elem_type->is_primitive_type()) {
1228     if (C->print_intrinsics()) {
1229       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1230     }
1231     return false; // should be primitive type
1232   }
1233 
1234   BasicType elem_bt = elem_type->basic_type();
1235   int num_elem = vlen->get_con();
1236 
1237   Node* base = argument(4);
1238   Node* offset = ConvL2X(argument(5));
1239 
1240   // Save state and restore on bailout
1241   uint old_sp = sp();
1242   SafePointNode* old_map = clone_map();
1243 
1244   Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1245   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1246   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1247 
1248   bool mismatched_ms = from_ms->get_con() && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
1249   BIG_ENDIAN_ONLY(if (mismatched_ms) return false;)
1250   // If there is no consistency between array and vector element types, it must be special byte array case
1251   if (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
1252     if (C->print_intrinsics()) {
1253       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
1254                     is_store, is_store ? "storeMasked" : "loadMasked",
1255                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1256     }
1257     set_map(old_map);
1258     set_sp(old_sp);
1259     return false;
1260   }
1261 
1262   int mem_num_elem = mismatched_ms ? num_elem * type2aelembytes(elem_bt) : num_elem;
1263   BasicType mem_elem_bt = mismatched_ms ? T_BYTE : elem_bt;
1264   bool supports_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
1265                                                 mem_num_elem, mem_elem_bt, VecMaskUseLoad);
1266 
1267   // If current arch does not support the predicated operations, we have to bail
1268   // out when current case uses the predicate feature.
1269   if (!supports_predicate) {
1270     bool needs_predicate = false;
1271     if (is_store) {
1272       // Masked vector store always uses the predicated store.
1273       needs_predicate = true;
1274     } else {
1275       // Masked vector load with IOOBE always uses the predicated load.
1276       const TypeInt* offset_in_range = gvn().type(argument(9))->isa_int();
1277       if (!offset_in_range->is_con()) {
1278         if (C->print_intrinsics()) {
1279           tty->print_cr("  ** missing constant: offsetInRange=%s",
1280                         NodeClassNames[argument(8)->Opcode()]);
1281         }
1282         set_map(old_map);
1283         set_sp(old_sp);
1284         return false;
1285       }
1286       needs_predicate = (offset_in_range->get_con() == 0);
1287     }
1288 
1289     if (needs_predicate) {
1290       if (C->print_intrinsics()) {
1291         tty->print_cr("  ** not supported: op=%s vlen=%d etype=%s mismatched_ms=%d",
1292                       is_store ? "storeMasked" : "loadMasked",
1293                       num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1294       }
1295       set_map(old_map);
1296       set_sp(old_sp);
1297       return false;
1298     }
1299   }
1300 
1301   // This only happens for masked vector load. If predicate is not supported, then check whether
1302   // the normal vector load and blend operations are supported by backend.
1303   if (!supports_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1304       !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1305     if (C->print_intrinsics()) {
1306       tty->print_cr("  ** not supported: op=loadMasked vlen=%d etype=%s mismatched_ms=%d",
1307                     num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1308     }
1309     set_map(old_map);
1310     set_sp(old_sp);
1311     return false;
1312   }
1313 
1314   // Since we are using byte array, we need to double check that the vector reinterpret operation
1315   // with byte type is supported by backend.
1316   if (mismatched_ms) {
1317     if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1318       if (C->print_intrinsics()) {
1319         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s mismatched_ms=1",
1320                       is_store, is_store ? "storeMasked" : "loadMasked",
1321                       num_elem, type2name(elem_bt));
1322       }
1323       set_map(old_map);
1324       set_sp(old_sp);
1325       return false;
1326     }
1327   }
1328 
1329   // Since it needs to unbox the mask, we need to double check that the related load operations
1330   // for mask are supported by backend.
1331   if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1332     if (C->print_intrinsics()) {
1333       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s",
1334                       is_store, is_store ? "storeMasked" : "loadMasked",
1335                       num_elem, type2name(elem_bt));
1336     }
1337     set_map(old_map);
1338     set_sp(old_sp);
1339     return false;
1340   }
1341 
1342   // Can base be null? Otherwise, always on-heap access.
1343   bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1344   if (can_access_non_heap) {
1345     insert_mem_bar(Op_MemBarCPUOrder);
1346   }
1347 
1348   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1349   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1350   assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1351   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1352   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1353 
1354   Node* mask = unbox_vector(is_store ? argument(9) : argument(8), mbox_type, elem_bt, num_elem);
1355   if (mask == nullptr) {
1356     if (C->print_intrinsics()) {
1357       tty->print_cr("  ** unbox failed mask=%s",
1358                     is_store ? NodeClassNames[argument(9)->Opcode()]
1359                              : NodeClassNames[argument(8)->Opcode()]);
1360     }
1361     set_map(old_map);
1362     set_sp(old_sp);
1363     return false;
1364   }
1365 
1366   if (is_store) {
1367     Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
1368     if (val == nullptr) {
1369       if (C->print_intrinsics()) {
1370         tty->print_cr("  ** unbox failed vector=%s",
1371                       NodeClassNames[argument(8)->Opcode()]);
1372       }
1373       set_map(old_map);
1374       set_sp(old_sp);
1375       return false; // operand unboxing failed
1376     }
1377     set_all_memory(reset_memory());
1378 
1379     if (mismatched_ms) {
1380       // Reinterpret the incoming vector to byte vector.
1381       const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1382       val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1383       // Reinterpret the vector mask to byte type.
1384       const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1385       const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1386       mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1387     }
1388     Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1389     set_memory(vstore, addr_type);
1390   } else {
1391     Node* vload = nullptr;
1392 
1393     if (mismatched_ms) {
1394       // Reinterpret the vector mask to byte type.
1395       const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1396       const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1397       mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1398     }
1399 
1400     if (supports_predicate) {
1401       // Generate masked load vector node if predicate feature is supported.
1402       const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1403       vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1404     } else {
1405       // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1406       Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1407       zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, Type::get_const_basic_type(mem_elem_bt)));
1408       vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1409       vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1410     }
1411 
1412     if (mismatched_ms) {
1413       const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1414       vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1415     }
1416 
1417     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1418     set_result(box);
1419   }
1420 
1421   destruct_map_clone(old_map);
1422 
1423   if (can_access_non_heap) {
1424     insert_mem_bar(Op_MemBarCPUOrder);
1425   }
1426 
1427   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1428   return true;
1429 }
1430 
1431 // <C,
1432 //  V extends Vector<?>,
1433 //  W extends Vector<Integer>,
1434 //  S extends VectorSpecies<E>,
1435 //  M extends VectorMask<E>,
1436 //  E>
1437 // V loadWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int length,
1438 //               Class<? extends Vector<Integer>> vectorIndexClass,
1439 //               Object base, long offset, // Unsafe addressing
1440 //               W index_vector, M m,
1441 //               C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
1442 //               LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1443 //
1444 //  <C,
1445 //   V extends Vector<E>,
1446 //   W extends Vector<Integer>,
1447 //   M extends VectorMask<E>,
1448 //   E>
1449 //  void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1450 //                    int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset,    // Unsafe addressing
1451 //                    W index_vector, V v, M m,
1452 //                    C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1453 //                    StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1454 //
1455 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1456   const TypeInstPtr* vector_klass     = gvn().type(argument(0))->isa_instptr();
1457   const TypeInstPtr* mask_klass       = gvn().type(argument(1))->isa_instptr();
1458   const TypeInstPtr* elem_klass       = gvn().type(argument(2))->isa_instptr();
1459   const TypeInt*     vlen             = gvn().type(argument(3))->isa_int();
1460   const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1461 
1462   if (vector_klass     == nullptr || vector_klass->const_oop()     == nullptr ||
1463 //      mask_klass       == nullptr || mask_klass->const_oop()       == nullptr ||
1464       elem_klass       == nullptr || elem_klass->const_oop()       == nullptr ||
1465       vlen             == nullptr || !vlen->is_con() ||
1466       vector_idx_klass == nullptr || vector_idx_klass->const_oop() == nullptr) {
1467     if (C->print_intrinsics()) {
1468       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1469                     NodeClassNames[argument(0)->Opcode()],
1470                     NodeClassNames[argument(2)->Opcode()],
1471                     NodeClassNames[argument(3)->Opcode()],
1472                     NodeClassNames[argument(4)->Opcode()]);
1473     }
1474     return false; // not enough info for intrinsification
1475   }
1476 
1477   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1478     if (C->print_intrinsics()) {
1479       tty->print_cr("  ** klass argument not initialized");
1480     }
1481     return false;
1482   }
1483 
1484   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1485   if (!elem_type->is_primitive_type()) {
1486     if (C->print_intrinsics()) {
1487       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1488     }
1489     return false; // should be primitive type
1490   }
1491 
1492   BasicType elem_bt = elem_type->basic_type();
1493   int num_elem = vlen->get_con();
1494 
1495   const Type* vmask_type = gvn().type(is_scatter ? argument(10) : argument(9));
1496   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1497   if (is_masked_op) {
1498     if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1499       if (C->print_intrinsics()) {
1500         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1501       }
1502       return false; // not enough info for intrinsification
1503     }
1504 
1505     if (!is_klass_initialized(mask_klass)) {
1506       if (C->print_intrinsics()) {
1507         tty->print_cr("  ** mask klass argument not initialized");
1508       }
1509       return false;
1510     }
1511 
1512     if (vmask_type->maybe_null()) {
1513       if (C->print_intrinsics()) {
1514         tty->print_cr("  ** null mask values are not allowed for masked op");
1515       }
1516       return false;
1517     }
1518 
1519     // Check whether the predicated gather/scatter node is supported by architecture.
1520     if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt,
1521                               (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred))) {
1522       if (C->print_intrinsics()) {
1523         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1524                       is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1525                       num_elem, type2name(elem_bt));
1526       }
1527       return false; // not supported
1528     }
1529   } else {
1530     // Check whether the normal gather/scatter node is supported for non-masked operation.
1531     if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1532       if (C->print_intrinsics()) {
1533         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1534                       is_scatter, is_scatter ? "scatter" : "gather",
1535                       num_elem, type2name(elem_bt));
1536       }
1537       return false; // not supported
1538     }
1539   }
1540 
1541   // Check that the vector holding indices is supported by architecture
1542   if (!arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
1543       if (C->print_intrinsics()) {
1544         tty->print_cr("  ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1545                       is_scatter, is_scatter ? "scatter" : "gather",
1546                       num_elem, is_masked_op ? 1 : 0);
1547       }
1548       return false; // not supported
1549   }
1550 
1551   Node* base = argument(5);
1552   Node* offset = ConvL2X(argument(6));
1553 
1554   // Save state and restore on bailout
1555   uint old_sp = sp();
1556   SafePointNode* old_map = clone_map();
1557 
1558   Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1559 
1560   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1561   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1562 
1563   // The array must be consistent with vector type
1564   if (arr_type == nullptr || (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, false))) {
1565     if (C->print_intrinsics()) {
1566       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1567                     is_scatter, is_scatter ? "scatter" : "gather",
1568                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1569     }
1570     set_map(old_map);
1571     set_sp(old_sp);
1572     return false;
1573   }
1574 
1575   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1576   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1577   ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1578   if (vbox_idx_klass == nullptr) {
1579     set_map(old_map);
1580     set_sp(old_sp);
1581     return false;
1582   }
1583 
1584   const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1585   Node* index_vect = unbox_vector(argument(8), vbox_idx_type, T_INT, num_elem);
1586   if (index_vect == nullptr) {
1587     set_map(old_map);
1588     set_sp(old_sp);
1589     return false;
1590   }
1591 
1592   Node* mask = nullptr;
1593   if (is_masked_op) {
1594     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1595     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1596     mask = unbox_vector(is_scatter ? argument(10) : argument(9), mbox_type, elem_bt, num_elem);
1597     if (mask == nullptr) {
1598       if (C->print_intrinsics()) {
1599         tty->print_cr("  ** unbox failed mask=%s",
1600                     is_scatter ? NodeClassNames[argument(10)->Opcode()]
1601                                : NodeClassNames[argument(9)->Opcode()]);
1602       }
1603       set_map(old_map);
1604       set_sp(old_sp);
1605       return false;
1606     }
1607   }
1608 
1609   const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1610   if (is_scatter) {
1611     Node* val = unbox_vector(argument(9), vbox_type, elem_bt, num_elem);
1612     if (val == nullptr) {
1613       set_map(old_map);
1614       set_sp(old_sp);
1615       return false; // operand unboxing failed
1616     }
1617     set_all_memory(reset_memory());
1618 
1619     Node* vstore = nullptr;
1620     if (mask != nullptr) {
1621       vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, index_vect, mask));
1622     } else {
1623       vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1624     }
1625     set_memory(vstore, addr_type);
1626   } else {
1627     Node* vload = nullptr;
1628     if (mask != nullptr) {
1629       vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_vect, mask));
1630     } else {
1631       vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1632     }
1633     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1634     set_result(box);
1635   }
1636 
1637   destruct_map_clone(old_map);
1638 
1639   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1640   return true;
1641 }
1642 
1643 // public static
1644 // <V extends Vector<E>,
1645 //  M extends VectorMask<E>,
1646 //  E>
1647 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1648 //                       Class<E> elementType, int length, V v, M m,
1649 //                       ReductionOperation<V, M> defaultImpl)
1650 bool LibraryCallKit::inline_vector_reduction() {
1651   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
1652   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1653   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
1654   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
1655   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
1656 
1657   if (opr          == nullptr || !opr->is_con() ||
1658       vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1659 //      mask_klass   == nullptr || mask_klass->const_oop()   == nullptr ||
1660       elem_klass   == nullptr || elem_klass->const_oop()   == nullptr ||
1661       vlen         == nullptr || !vlen->is_con()) {
1662     if (C->print_intrinsics()) {
1663       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1664                     NodeClassNames[argument(0)->Opcode()],
1665                     NodeClassNames[argument(1)->Opcode()],
1666                     NodeClassNames[argument(3)->Opcode()],
1667                     NodeClassNames[argument(4)->Opcode()]);
1668     }
1669     return false; // not enough info for intrinsification
1670   }
1671   if (!is_klass_initialized(vector_klass)) {
1672     if (C->print_intrinsics()) {
1673       tty->print_cr("  ** klass argument not initialized");
1674     }
1675     return false;
1676   }
1677   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1678   if (!elem_type->is_primitive_type()) {
1679     if (C->print_intrinsics()) {
1680       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1681     }
1682     return false; // should be primitive type
1683   }
1684 
1685   const Type* vmask_type = gvn().type(argument(6));
1686   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1687   if (is_masked_op) {
1688     if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1689       if (C->print_intrinsics()) {
1690         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1691       }
1692       return false; // not enough info for intrinsification
1693     }
1694 
1695     if (!is_klass_initialized(mask_klass)) {
1696       if (C->print_intrinsics()) {
1697         tty->print_cr("  ** mask klass argument not initialized");
1698       }
1699       return false;
1700     }
1701 
1702     if (vmask_type->maybe_null()) {
1703       if (C->print_intrinsics()) {
1704         tty->print_cr("  ** null mask values are not allowed for masked op");
1705       }
1706       return false;
1707     }
1708   }
1709 
1710   BasicType elem_bt = elem_type->basic_type();
1711   int num_elem = vlen->get_con();
1712   int opc  = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1713   int sopc = ReductionNode::opcode(opc, elem_bt);
1714 
1715   // When using mask, mask use type needs to be VecMaskUseLoad.
1716   if (!arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1717     if (C->print_intrinsics()) {
1718       tty->print_cr("  ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1719                     sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1720     }
1721     return false;
1722   }
1723 
1724   // Return true if current platform has implemented the masked operation with predicate feature.
1725   bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1726   if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1727     if (C->print_intrinsics()) {
1728       tty->print_cr("  ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1729                     sopc, num_elem, type2name(elem_bt));
1730     }
1731     return false;
1732   }
1733 
1734   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1735   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1736 
1737   Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1738   if (opd == nullptr) {
1739     return false; // operand unboxing failed
1740   }
1741 
1742   Node* mask = nullptr;
1743   if (is_masked_op) {
1744     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1745     assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1746     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1747     mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1748     if (mask == nullptr) {
1749       if (C->print_intrinsics()) {
1750         tty->print_cr("  ** unbox failed mask=%s",
1751                       NodeClassNames[argument(6)->Opcode()]);
1752       }
1753       return false;
1754     }
1755   }
1756 
1757   Node* init = ReductionNode::make_identity_con_scalar(gvn(), opc, elem_bt);
1758   Node* value = nullptr;
1759   if (mask == nullptr) {
1760     assert(!is_masked_op, "Masked op needs the mask value never null");
1761     value = ReductionNode::make(opc, nullptr, init, opd, elem_bt);
1762   } else {
1763     if (use_predicate) {
1764       value = ReductionNode::make(opc, nullptr, init, opd, elem_bt);
1765       value->add_req(mask);
1766       value->add_flag(Node::Flag_is_predicated_vector);
1767     } else {
1768       Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, Type::get_const_basic_type(elem_bt)));
1769       value = gvn().transform(new VectorBlendNode(reduce_identity, opd, mask));
1770       value = ReductionNode::make(opc, nullptr, init, value, elem_bt);
1771     }
1772   }
1773   value = gvn().transform(value);
1774 
1775   Node* bits = nullptr;
1776   switch (elem_bt) {
1777     case T_BYTE:
1778     case T_SHORT:
1779     case T_INT: {
1780       bits = gvn().transform(new ConvI2LNode(value));
1781       break;
1782     }
1783     case T_FLOAT: {
1784       value = gvn().transform(new MoveF2INode(value));
1785       bits  = gvn().transform(new ConvI2LNode(value));
1786       break;
1787     }
1788     case T_DOUBLE: {
1789       bits = gvn().transform(new MoveD2LNode(value));
1790       break;
1791     }
1792     case T_LONG: {
1793       bits = value; // no conversion needed
1794       break;
1795     }
1796     default: fatal("%s", type2name(elem_bt));
1797   }
1798   set_result(bits);
1799   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1800   return true;
1801 }
1802 
1803 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1804 //                                V v1, V v2,
1805 //                                BiFunction<V, V, Boolean> defaultImpl)
1806 //
1807 bool LibraryCallKit::inline_vector_test() {
1808   const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1809   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1810   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1811   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1812 
1813   if (cond         == nullptr || !cond->is_con() ||
1814       vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1815       elem_klass   == nullptr || elem_klass->const_oop()   == nullptr ||
1816       vlen         == nullptr || !vlen->is_con()) {
1817     if (C->print_intrinsics()) {
1818       tty->print_cr("  ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1819                     NodeClassNames[argument(0)->Opcode()],
1820                     NodeClassNames[argument(1)->Opcode()],
1821                     NodeClassNames[argument(2)->Opcode()],
1822                     NodeClassNames[argument(3)->Opcode()]);
1823     }
1824     return false; // not enough info for intrinsification
1825   }
1826   if (!is_klass_initialized(vector_klass)) {
1827     if (C->print_intrinsics()) {
1828       tty->print_cr("  ** klass argument not initialized");
1829     }
1830     return false;
1831   }
1832   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1833   if (!elem_type->is_primitive_type()) {
1834     if (C->print_intrinsics()) {
1835       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1836     }
1837     return false; // should be primitive type
1838   }
1839   BasicType elem_bt = elem_type->basic_type();
1840   int num_elem = vlen->get_con();
1841   BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1842   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1843   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1844 
1845   if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1846     if (C->print_intrinsics()) {
1847       tty->print_cr("  ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1848                     cond->get_con(), num_elem, type2name(elem_bt),
1849                     is_vector_mask(vbox_klass));
1850     }
1851     return false;
1852   }
1853 
1854   Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1855   Node* opd2;
1856   if (Matcher::vectortest_needs_second_argument(booltest == BoolTest::overflow,
1857                                                 opd1->bottom_type()->isa_vectmask())) {
1858     opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1859   } else {
1860     opd2 = opd1;
1861   }
1862   if (opd1 == nullptr || opd2 == nullptr) {
1863     return false; // operand unboxing failed
1864   }
1865 
1866   Node* cmp = gvn().transform(new VectorTestNode(opd1, opd2, booltest));
1867   BoolTest::mask test = Matcher::vectortest_mask(booltest == BoolTest::overflow,
1868                                                  opd1->bottom_type()->isa_vectmask(), num_elem);
1869   Node* bol = gvn().transform(new BoolNode(cmp, test));
1870   Node* res = gvn().transform(new CMoveINode(bol, gvn().intcon(0), gvn().intcon(1), TypeInt::BOOL));
1871 
1872   set_result(res);
1873   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1874   return true;
1875 }
1876 
1877 // public static
1878 // <V extends Vector<E>,
1879 //  M extends VectorMask<E>,
1880 //  E>
1881 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1882 //         V v1, V v2, M m,
1883 //         VectorBlendOp<V, M, E> defaultImpl)
1884 bool LibraryCallKit::inline_vector_blend() {
1885   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1886   const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1887   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1888   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1889 
1890   if (mask_klass == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1891     return false; // dead code
1892   }
1893   if (mask_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr ||
1894       elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1895     if (C->print_intrinsics()) {
1896       tty->print_cr("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1897                     NodeClassNames[argument(0)->Opcode()],
1898                     NodeClassNames[argument(1)->Opcode()],
1899                     NodeClassNames[argument(2)->Opcode()],
1900                     NodeClassNames[argument(3)->Opcode()]);
1901     }
1902     return false; // not enough info for intrinsification
1903   }
1904   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1905     if (C->print_intrinsics()) {
1906       tty->print_cr("  ** klass argument not initialized");
1907     }
1908     return false;
1909   }
1910   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1911   if (!elem_type->is_primitive_type()) {
1912     if (C->print_intrinsics()) {
1913       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1914     }
1915     return false; // should be primitive type
1916   }
1917   BasicType elem_bt = elem_type->basic_type();
1918   BasicType mask_bt = elem_bt;
1919   int num_elem = vlen->get_con();
1920 
1921   if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1922     if (C->print_intrinsics()) {
1923       tty->print_cr("  ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1924                     num_elem, type2name(elem_bt));
1925     }
1926     return false; // not supported
1927   }
1928   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1929   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1930 
1931   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1932   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1933 
1934   Node* v1   = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1935   Node* v2   = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1936   Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1937 
1938   if (v1 == nullptr || v2 == nullptr || mask == nullptr) {
1939     return false; // operand unboxing failed
1940   }
1941 
1942   Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1943 
1944   Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1945   set_result(box);
1946   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1947   return true;
1948 }
1949 
1950 //  public static
1951 //  <V extends Vector<E>,
1952 //   M extends VectorMask<E>,
1953 //   E>
1954 //  M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1955 //            V v1, V v2, M m,
1956 //            VectorCompareOp<V,M> defaultImpl)
1957 bool LibraryCallKit::inline_vector_compare() {
1958   const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1959   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1960   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
1961   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
1962   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
1963 
1964   if (cond == nullptr || vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1965     return false; // dead code
1966   }
1967   if (!cond->is_con() || vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
1968       elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1969     if (C->print_intrinsics()) {
1970       tty->print_cr("  ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1971                     NodeClassNames[argument(0)->Opcode()],
1972                     NodeClassNames[argument(1)->Opcode()],
1973                     NodeClassNames[argument(2)->Opcode()],
1974                     NodeClassNames[argument(3)->Opcode()],
1975                     NodeClassNames[argument(4)->Opcode()]);
1976     }
1977     return false; // not enough info for intrinsification
1978   }
1979   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1980     if (C->print_intrinsics()) {
1981       tty->print_cr("  ** klass argument not initialized");
1982     }
1983     return false;
1984   }
1985   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1986   if (!elem_type->is_primitive_type()) {
1987     if (C->print_intrinsics()) {
1988       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1989     }
1990     return false; // should be primitive type
1991   }
1992 
1993   int num_elem = vlen->get_con();
1994   BasicType elem_bt = elem_type->basic_type();
1995   BasicType mask_bt = elem_bt;
1996 
1997   if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1998     if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1999       if (C->print_intrinsics()) {
2000         tty->print_cr("  ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
2001                       cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
2002       }
2003       return false;
2004     }
2005   }
2006 
2007   if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
2008     if (C->print_intrinsics()) {
2009       tty->print_cr("  ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
2010                     cond->get_con(), num_elem, type2name(elem_bt));
2011     }
2012     return false;
2013   }
2014 
2015   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2016   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2017 
2018   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2019   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2020 
2021   Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2022   Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
2023 
2024   bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
2025   Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : nullptr;
2026   if (is_masked_op && mask == nullptr) {
2027     if (C->print_intrinsics()) {
2028       tty->print_cr("  ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
2029                     cond->get_con(), num_elem, type2name(elem_bt));
2030     }
2031     return false;
2032   }
2033 
2034   bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
2035   if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
2036     if (C->print_intrinsics()) {
2037       tty->print_cr("  ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
2038                     cond->get_con(), num_elem, type2name(elem_bt));
2039     }
2040     return false;
2041   }
2042 
2043   if (v1 == nullptr || v2 == nullptr) {
2044     return false; // operand unboxing failed
2045   }
2046   BoolTest::mask pred = (BoolTest::mask)cond->get_con();
2047   ConINode* pred_node = (ConINode*)gvn().makecon(cond);
2048 
2049   const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
2050   Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
2051 
2052   if (is_masked_op) {
2053     if (use_predicate) {
2054       operation->add_req(mask);
2055       operation->add_flag(Node::Flag_is_predicated_vector);
2056     } else {
2057       operation = gvn().transform(operation);
2058       operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
2059     }
2060   }
2061 
2062   operation = gvn().transform(operation);
2063 
2064   Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
2065   set_result(box);
2066   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2067   return true;
2068 }
2069 
2070 // public static
2071 // <V extends Vector<E>,
2072 //  Sh extends VectorShuffle<E>,
2073 //  M extends VectorMask<E>,
2074 //  E>
2075 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
2076 //               V v1, Sh sh, M m,
2077 //               VectorRearrangeOp<V, Sh, M, E> defaultImpl)
2078 bool LibraryCallKit::inline_vector_rearrange() {
2079   const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
2080   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
2081   const TypeInstPtr* mask_klass    = gvn().type(argument(2))->isa_instptr();
2082   const TypeInstPtr* elem_klass    = gvn().type(argument(3))->isa_instptr();
2083   const TypeInt*     vlen          = gvn().type(argument(4))->isa_int();
2084 
2085   if (vector_klass == nullptr  || shuffle_klass == nullptr ||  elem_klass == nullptr || vlen == nullptr) {
2086     return false; // dead code
2087   }
2088   if (shuffle_klass->const_oop() == nullptr ||
2089       vector_klass->const_oop()  == nullptr ||
2090       elem_klass->const_oop()    == nullptr ||
2091       !vlen->is_con()) {
2092     if (C->print_intrinsics()) {
2093       tty->print_cr("  ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
2094                     NodeClassNames[argument(0)->Opcode()],
2095                     NodeClassNames[argument(1)->Opcode()],
2096                     NodeClassNames[argument(3)->Opcode()],
2097                     NodeClassNames[argument(4)->Opcode()]);
2098     }
2099     return false; // not enough info for intrinsification
2100   }
2101   if (!is_klass_initialized(vector_klass)  ||
2102       !is_klass_initialized(shuffle_klass)) {
2103     if (C->print_intrinsics()) {
2104       tty->print_cr("  ** klass argument not initialized");
2105     }
2106     return false;
2107   }
2108   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2109   if (!elem_type->is_primitive_type()) {
2110     if (C->print_intrinsics()) {
2111       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2112     }
2113     return false; // should be primitive type
2114   }
2115   BasicType elem_bt = elem_type->basic_type();
2116   BasicType shuffle_bt = elem_bt;
2117   int num_elem = vlen->get_con();
2118 
2119   if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
2120     if (C->print_intrinsics()) {
2121       tty->print_cr("  ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
2122                     num_elem, type2name(elem_bt));
2123     }
2124     return false; // not supported
2125   }
2126 
2127   bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
2128   bool use_predicate = is_masked_op;
2129   if (is_masked_op &&
2130       (mask_klass == nullptr ||
2131        mask_klass->const_oop() == nullptr ||
2132        !is_klass_initialized(mask_klass))) {
2133     if (C->print_intrinsics()) {
2134       tty->print_cr("  ** mask_klass argument not initialized");
2135     }
2136   }
2137   VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2138   if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
2139     use_predicate = false;
2140     if(!is_masked_op ||
2141        (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2142         !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)     ||
2143         !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
2144       if (C->print_intrinsics()) {
2145         tty->print_cr("  ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
2146                       num_elem, type2name(elem_bt));
2147       }
2148       return false; // not supported
2149     }
2150   }
2151   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2152   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2153 
2154   ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
2155   const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
2156 
2157   Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2158   Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
2159 
2160   if (v1 == nullptr || shuffle == nullptr) {
2161     return false; // operand unboxing failed
2162   }
2163 
2164   Node* mask = nullptr;
2165   if (is_masked_op) {
2166     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2167     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2168     mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2169     if (mask == nullptr) {
2170       if (C->print_intrinsics()) {
2171         tty->print_cr("  ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
2172                       num_elem, type2name(elem_bt));
2173       }
2174       return false;
2175     }
2176   }
2177 
2178   Node* rearrange = new VectorRearrangeNode(v1, shuffle);
2179   if (is_masked_op) {
2180     if (use_predicate) {
2181       rearrange->add_req(mask);
2182       rearrange->add_flag(Node::Flag_is_predicated_vector);
2183     } else {
2184       const TypeVect* vt = v1->bottom_type()->is_vect();
2185       rearrange = gvn().transform(rearrange);
2186       Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2187       Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, Type::get_const_basic_type(elem_bt)));
2188       rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2189     }
2190   }
2191   rearrange = gvn().transform(rearrange);
2192 
2193   Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2194   set_result(box);
2195   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2196   return true;
2197 }
2198 
2199 static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
2200   address addr = nullptr;
2201   assert(UseVectorStubs, "sanity");
2202   assert(name_ptr != nullptr, "unexpected");
2203   assert((vop >= VectorSupport::VECTOR_OP_SVML_START) && (vop <= VectorSupport::VECTOR_OP_SVML_END), "unexpected");
2204   int op = vop - VectorSupport::VECTOR_OP_SVML_START;
2205 
2206   switch(bits) {
2207     case 64:  //fallthough
2208     case 128: //fallthough
2209     case 256: //fallthough
2210     case 512:
2211       if (bt == T_FLOAT) {
2212         snprintf(name_ptr, name_len, "vector_%s_float%d", VectorSupport::svmlname[op], bits);
2213         addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op];
2214       } else {
2215         assert(bt == T_DOUBLE, "must be FP type only");
2216         snprintf(name_ptr, name_len, "vector_%s_double%d", VectorSupport::svmlname[op], bits);
2217         addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op];
2218       }
2219       break;
2220     default:
2221       snprintf(name_ptr, name_len, "invalid");
2222       addr = nullptr;
2223       Unimplemented();
2224       break;
2225   }
2226 
2227   return addr;
2228 }
2229 
2230 Node* LibraryCallKit::gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
2231   assert(UseVectorStubs, "sanity");
2232   assert(vector_api_op_id >= VectorSupport::VECTOR_OP_SVML_START && vector_api_op_id <= VectorSupport::VECTOR_OP_SVML_END, "need valid op id");
2233   assert(opd1 != nullptr, "must not be null");
2234   const TypeVect* vt = TypeVect::make(bt, num_elem);
2235   const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != nullptr ? 2 : 1, vt, vt);
2236   char name[100] = "";
2237 
2238   // Get address for svml method.
2239   address addr = get_svml_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
2240 
2241   if (addr == nullptr) {
2242     return nullptr;
2243   }
2244 
2245   assert(name[0] != '\0', "name must not be null");
2246   Node* operation = make_runtime_call(RC_VECTOR,
2247                                       call_type,
2248                                       addr,
2249                                       name,
2250                                       TypePtr::BOTTOM,
2251                                       opd1,
2252                                       opd2);
2253   return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
2254 }
2255 
2256 //  public static
2257 //  <V extends Vector<E>,
2258 //   M extends VectorMask<E>,
2259 //   E>
2260 //  V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2261 //                 Class<E> elementType, int length,
2262 //                 V v, int n, M m,
2263 //                 VectorBroadcastIntOp<V, M> defaultImpl)
2264 bool LibraryCallKit::inline_vector_broadcast_int() {
2265   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
2266   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2267   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
2268   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
2269   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
2270 
2271   if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
2272     return false; // dead code
2273   }
2274   if (!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2275     if (C->print_intrinsics()) {
2276       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2277                     NodeClassNames[argument(0)->Opcode()],
2278                     NodeClassNames[argument(1)->Opcode()],
2279                     NodeClassNames[argument(3)->Opcode()],
2280                     NodeClassNames[argument(4)->Opcode()]);
2281     }
2282     return false; // not enough info for intrinsification
2283   }
2284   if (!is_klass_initialized(vector_klass)) {
2285     if (C->print_intrinsics()) {
2286       tty->print_cr("  ** klass argument not initialized");
2287     }
2288     return false;
2289   }
2290 
2291   const Type* vmask_type = gvn().type(argument(7));
2292   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2293   if (is_masked_op) {
2294     if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
2295       if (C->print_intrinsics()) {
2296         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2297       }
2298       return false; // not enough info for intrinsification
2299     }
2300 
2301     if (!is_klass_initialized(mask_klass)) {
2302       if (C->print_intrinsics()) {
2303         tty->print_cr("  ** mask klass argument not initialized");
2304       }
2305       return false;
2306     }
2307 
2308     if (vmask_type->maybe_null()) {
2309       if (C->print_intrinsics()) {
2310         tty->print_cr("  ** null mask values are not allowed for masked op");
2311       }
2312       return false;
2313     }
2314   }
2315 
2316   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2317   if (!elem_type->is_primitive_type()) {
2318     if (C->print_intrinsics()) {
2319       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2320     }
2321     return false; // should be primitive type
2322   }
2323 
2324   int num_elem = vlen->get_con();
2325   BasicType elem_bt = elem_type->basic_type();
2326   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2327 
2328   bool is_shift  = VectorNode::is_shift_opcode(opc);
2329   bool is_rotate = VectorNode::is_rotate_opcode(opc);
2330 
2331   if (opc == 0 || (!is_shift && !is_rotate)) {
2332     if (C->print_intrinsics()) {
2333       tty->print_cr("  ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2334     }
2335     return false; // operation not supported
2336   }
2337 
2338   int sopc = VectorNode::opcode(opc, elem_bt);
2339   if (sopc == 0) {
2340     if (C->print_intrinsics()) {
2341       tty->print_cr("  ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2342     }
2343     return false; // operation not supported
2344   }
2345 
2346   Node* cnt  = argument(6);
2347   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2348   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2349   const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2350 
2351   // If CPU supports vector constant rotate instructions pass it directly
2352   bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2353                          Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2354   bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2355 
2356   VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2357   bool use_predicate = is_masked_op;
2358 
2359   if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2360     use_predicate = false;
2361     if (!is_masked_op ||
2362         (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2363          !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2364 
2365       if (C->print_intrinsics()) {
2366         tty->print_cr("  ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2367                       sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2368       }
2369       return false; // not supported
2370     }
2371   }
2372 
2373   Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2374   Node* opd2 = nullptr;
2375   if (is_shift) {
2376     opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2377   } else {
2378     assert(is_rotate, "unexpected operation");
2379     if (!is_const_rotate) {
2380       const Type * type_bt = Type::get_const_basic_type(elem_bt);
2381       cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2382       opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, type_bt));
2383     } else {
2384       // Constant shift value.
2385       opd2 = cnt;
2386     }
2387   }
2388 
2389   if (opd1 == nullptr || opd2 == nullptr) {
2390     return false;
2391   }
2392 
2393   Node* mask = nullptr;
2394   if (is_masked_op) {
2395     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2396     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2397     mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2398     if (mask == nullptr) {
2399       if (C->print_intrinsics()) {
2400         tty->print_cr("  ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2401       }
2402       return false;
2403     }
2404   }
2405 
2406   Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2407   if (is_masked_op && mask != nullptr) {
2408     if (use_predicate) {
2409       operation->add_req(mask);
2410       operation->add_flag(Node::Flag_is_predicated_vector);
2411     } else {
2412       operation = gvn().transform(operation);
2413       operation = new VectorBlendNode(opd1, operation, mask);
2414     }
2415   }
2416   operation = gvn().transform(operation);
2417   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2418   set_result(vbox);
2419   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2420   return true;
2421 }
2422 
2423 // public static <VOUT extends VectorPayload,
2424 //                 VIN extends VectorPayload,
2425 //                   S extends VectorSpecies>
2426 // VOUT convert(int oprId,
2427 //           Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2428 //           Class<?>   toVectorClass, Class<?>   toElementType, int   toVLen,
2429 //           VIN v, S s,
2430 //           VectorConvertOp<VOUT, VIN, S> defaultImpl)
2431 //
2432 bool LibraryCallKit::inline_vector_convert() {
2433   const TypeInt*     opr               = gvn().type(argument(0))->isa_int();
2434 
2435   const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2436   const TypeInstPtr* elem_klass_from   = gvn().type(argument(2))->isa_instptr();
2437   const TypeInt*     vlen_from         = gvn().type(argument(3))->isa_int();
2438 
2439   const TypeInstPtr* vector_klass_to   = gvn().type(argument(4))->isa_instptr();
2440   const TypeInstPtr* elem_klass_to     = gvn().type(argument(5))->isa_instptr();
2441   const TypeInt*     vlen_to           = gvn().type(argument(6))->isa_int();
2442 
2443   if (opr == nullptr ||
2444       vector_klass_from == nullptr || elem_klass_from == nullptr || vlen_from == nullptr ||
2445       vector_klass_to   == nullptr || elem_klass_to   == nullptr || vlen_to   == nullptr) {
2446     return false; // dead code
2447   }
2448   if (!opr->is_con() ||
2449       vector_klass_from->const_oop() == nullptr || elem_klass_from->const_oop() == nullptr || !vlen_from->is_con() ||
2450       vector_klass_to->const_oop() == nullptr || elem_klass_to->const_oop() == nullptr || !vlen_to->is_con()) {
2451     if (C->print_intrinsics()) {
2452       tty->print_cr("  ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2453                     NodeClassNames[argument(0)->Opcode()],
2454                     NodeClassNames[argument(1)->Opcode()],
2455                     NodeClassNames[argument(2)->Opcode()],
2456                     NodeClassNames[argument(3)->Opcode()],
2457                     NodeClassNames[argument(4)->Opcode()],
2458                     NodeClassNames[argument(5)->Opcode()],
2459                     NodeClassNames[argument(6)->Opcode()]);
2460     }
2461     return false; // not enough info for intrinsification
2462   }
2463   if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2464     if (C->print_intrinsics()) {
2465       tty->print_cr("  ** klass argument not initialized");
2466     }
2467     return false;
2468   }
2469 
2470   assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST  ||
2471          opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
2472          opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2473   bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2474   bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2475 
2476   ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2477   ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2478   if (is_vector_shuffle(vbox_klass_from)) {
2479     return false; // vector shuffles aren't supported
2480   }
2481   bool is_mask = is_vector_mask(vbox_klass_from);
2482 
2483   ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2484   if (!elem_type_from->is_primitive_type()) {
2485     return false; // should be primitive type
2486   }
2487   BasicType elem_bt_from = elem_type_from->basic_type();
2488   ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2489   if (!elem_type_to->is_primitive_type()) {
2490     return false; // should be primitive type
2491   }
2492   BasicType elem_bt_to = elem_type_to->basic_type();
2493 
2494   int num_elem_from = vlen_from->get_con();
2495   int num_elem_to = vlen_to->get_con();
2496 
2497   // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2498   // since we may need to change size.
2499   if (!arch_supports_vector(Op_VectorReinterpret,
2500                             num_elem_from,
2501                             elem_bt_from,
2502                             is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2503     if (C->print_intrinsics()) {
2504       tty->print_cr("  ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2505                     is_cast ? "cast" : "reinterpret",
2506                     num_elem_from, type2name(elem_bt_from), is_mask);
2507     }
2508     return false;
2509   }
2510 
2511   // Check whether we can support resizing/reinterpreting to the new size.
2512   if (!arch_supports_vector(Op_VectorReinterpret,
2513                             num_elem_to,
2514                             elem_bt_to,
2515                             is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2516     if (C->print_intrinsics()) {
2517       tty->print_cr("  ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2518                     is_cast ? "cast" : "reinterpret",
2519                     num_elem_to, type2name(elem_bt_to), is_mask);
2520     }
2521     return false;
2522   }
2523 
2524   // At this point, we know that both input and output vector registers are supported
2525   // by the architecture. Next check if the casted type is simply to same type - which means
2526   // that it is actually a resize and not a cast.
2527   if (is_cast && elem_bt_from == elem_bt_to) {
2528     is_cast = false;
2529   }
2530 
2531   const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2532 
2533   Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2534   if (opd1 == nullptr) {
2535     return false;
2536   }
2537 
2538   const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2539   const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2540 
2541   // Safety check to prevent casting if source mask is of type vector
2542   // and destination mask of type predicate vector and vice-versa.
2543   // From X86 standpoint, this case will only arise over KNL target,
2544   // where certain masks (depending on the species) are either propagated
2545   // through a vector or predicate register.
2546   if (is_mask &&
2547       ((src_type->isa_vectmask() == nullptr && dst_type->isa_vectmask()) ||
2548        (dst_type->isa_vectmask() == nullptr && src_type->isa_vectmask()))) {
2549     return false;
2550   }
2551 
2552   Node* op = opd1;
2553   if (is_cast) {
2554     assert(!is_mask || num_elem_from == num_elem_to, "vector mask cast needs the same elem num");
2555     int cast_vopc = VectorCastNode::opcode(-1, elem_bt_from, !is_ucast);
2556 
2557     // Make sure that vector cast is implemented to particular type/size combination if it is
2558     // not a mask casting.
2559     if (!is_mask && !arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2560       if (C->print_intrinsics()) {
2561         tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2562                       cast_vopc, num_elem_to, type2name(elem_bt_to), is_mask);
2563       }
2564       return false;
2565     }
2566 
2567     if (num_elem_from < num_elem_to) {
2568       // Since input and output number of elements are not consistent, we need to make sure we
2569       // properly size. Thus, first make a cast that retains the number of elements from source.
2570       int num_elem_for_cast = num_elem_from;
2571 
2572       // It is possible that arch does not support this intermediate vector size
2573       // TODO More complex logic required here to handle this corner case for the sizes.
2574       if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2575         if (C->print_intrinsics()) {
2576           tty->print_cr("  ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2577                         cast_vopc,
2578                         num_elem_for_cast, type2name(elem_bt_to), is_mask);
2579         }
2580         return false;
2581       }
2582 
2583       op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2584       // Now ensure that the destination gets properly resized to needed size.
2585       op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2586     } else if (num_elem_from > num_elem_to) {
2587       // Since number of elements from input is larger than output, simply reduce size of input
2588       // (we are supposed to drop top elements anyway).
2589       int num_elem_for_resize = num_elem_to;
2590 
2591       // It is possible that arch does not support this intermediate vector size
2592       // TODO More complex logic required here to handle this corner case for the sizes.
2593       if (!arch_supports_vector(Op_VectorReinterpret,
2594                                 num_elem_for_resize,
2595                                 elem_bt_from,
2596                                 VecMaskNotUsed)) {
2597         if (C->print_intrinsics()) {
2598           tty->print_cr("  ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2599                         num_elem_for_resize, type2name(elem_bt_from), is_mask);
2600         }
2601         return false;
2602       }
2603 
2604       const TypeVect* resize_type = TypeVect::make(elem_bt_from, num_elem_for_resize);
2605       op = gvn().transform(new VectorReinterpretNode(op, src_type, resize_type));
2606       op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2607     } else { // num_elem_from == num_elem_to
2608       if (is_mask) {
2609         // Make sure that cast for vector mask is implemented to particular type/size combination.
2610         if (!arch_supports_vector(Op_VectorMaskCast, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2611           if (C->print_intrinsics()) {
2612             tty->print_cr("  ** not supported: arity=1 op=maskcast vlen2=%d etype2=%s ismask=%d",
2613                           num_elem_to, type2name(elem_bt_to), is_mask);
2614           }
2615           return false;
2616         }
2617         op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2618       } else {
2619         // Since input and output number of elements match, and since we know this vector size is
2620         // supported, simply do a cast with no resize needed.
2621         op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2622       }
2623     }
2624   } else if (Type::cmp(src_type, dst_type) != 0) {
2625     assert(!is_cast, "must be reinterpret");
2626     op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2627   }
2628 
2629   const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2630   Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2631   set_result(vbox);
2632   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2633   return true;
2634 }
2635 
2636 //  public static
2637 //  <V extends Vector<E>,
2638 //   E>
2639 //  V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2640 //           V vec, int ix, long val,
2641 //           VecInsertOp<V> defaultImpl)
2642 bool LibraryCallKit::inline_vector_insert() {
2643   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2644   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2645   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2646   const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
2647 
2648   if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2649     return false; // dead code
2650   }
2651   if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2652     if (C->print_intrinsics()) {
2653       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2654                     NodeClassNames[argument(0)->Opcode()],
2655                     NodeClassNames[argument(1)->Opcode()],
2656                     NodeClassNames[argument(2)->Opcode()],
2657                     NodeClassNames[argument(4)->Opcode()]);
2658     }
2659     return false; // not enough info for intrinsification
2660   }
2661   if (!is_klass_initialized(vector_klass)) {
2662     if (C->print_intrinsics()) {
2663       tty->print_cr("  ** klass argument not initialized");
2664     }
2665     return false;
2666   }
2667   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2668   if (!elem_type->is_primitive_type()) {
2669     if (C->print_intrinsics()) {
2670       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2671     }
2672     return false; // should be primitive type
2673   }
2674   BasicType elem_bt = elem_type->basic_type();
2675   int num_elem = vlen->get_con();
2676   if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2677     if (C->print_intrinsics()) {
2678       tty->print_cr("  ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2679                     num_elem, type2name(elem_bt));
2680     }
2681     return false; // not supported
2682   }
2683 
2684   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2685   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2686 
2687   Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2688   if (opd == nullptr) {
2689     return false;
2690   }
2691 
2692   Node* insert_val = argument(5);
2693   assert(gvn().type(insert_val)->isa_long() != nullptr, "expected to be long");
2694 
2695   // Convert insert value back to its appropriate type.
2696   switch (elem_bt) {
2697     case T_BYTE:
2698       insert_val = gvn().transform(new ConvL2INode(insert_val));
2699       insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::BYTE));
2700       break;
2701     case T_SHORT:
2702       insert_val = gvn().transform(new ConvL2INode(insert_val));
2703       insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::SHORT));
2704       break;
2705     case T_INT:
2706       insert_val = gvn().transform(new ConvL2INode(insert_val));
2707       break;
2708     case T_FLOAT:
2709       insert_val = gvn().transform(new ConvL2INode(insert_val));
2710       insert_val = gvn().transform(new MoveI2FNode(insert_val));
2711       break;
2712     case T_DOUBLE:
2713       insert_val = gvn().transform(new MoveL2DNode(insert_val));
2714       break;
2715     case T_LONG:
2716       // no conversion needed
2717       break;
2718     default: fatal("%s", type2name(elem_bt)); break;
2719   }
2720 
2721   Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con()));
2722 
2723   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2724   set_result(vbox);
2725   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2726   return true;
2727 }
2728 
2729 //  public static
2730 //  <VM extends VectorPayload,
2731 //   E>
2732 //  long extract(Class<? extends VM> vClass, Class<E> eClass,
2733 //               int length,
2734 //               VM vm, int i,
2735 //               VecExtractOp<VM> defaultImpl)
2736 bool LibraryCallKit::inline_vector_extract() {
2737   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2738   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2739   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2740   const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
2741 
2742   if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2743     return false; // dead code
2744   }
2745   if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2746     if (C->print_intrinsics()) {
2747       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
2748                     NodeClassNames[argument(0)->Opcode()],
2749                     NodeClassNames[argument(1)->Opcode()],
2750                     NodeClassNames[argument(2)->Opcode()]);
2751     }
2752     return false; // not enough info for intrinsification
2753   }
2754   if (!is_klass_initialized(vector_klass)) {
2755     if (C->print_intrinsics()) {
2756       tty->print_cr("  ** klass argument not initialized");
2757     }
2758     return false;
2759   }
2760   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2761   if (!elem_type->is_primitive_type()) {
2762     if (C->print_intrinsics()) {
2763       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2764     }
2765     return false; // should be primitive type
2766   }
2767   BasicType elem_bt = elem_type->basic_type();
2768   int num_elem = vlen->get_con();
2769 
2770   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2771   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2772 
2773   Node* opd = nullptr;
2774 
2775   if (is_vector_mask(vbox_klass)) {
2776     // vbox_klass is mask. This is used for VectorMask.laneIsSet(int).
2777 
2778     Node* pos = argument(4); // can be variable
2779     if (arch_supports_vector(Op_ExtractUB, num_elem, elem_bt, VecMaskUseAll)) {
2780       // Transform mask to vector with type of boolean and utilize ExtractUB node.
2781       opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2782       if (opd == nullptr) {
2783         return false;
2784       }
2785       opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2786       opd = gvn().transform(new ExtractUBNode(opd, pos));
2787       opd = gvn().transform(new ConvI2LNode(opd));
2788     } else if (arch_supports_vector(Op_VectorMaskToLong, num_elem, elem_bt, VecMaskUseLoad)) {
2789       opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2790       if (opd == nullptr) {
2791         return false;
2792       }
2793       // VectorMaskToLongNode requires the input is either a mask or a vector with BOOLEAN type.
2794       if (opd->bottom_type()->isa_vectmask() == nullptr) {
2795         opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2796       }
2797       // ((toLong() >>> pos) & 1L
2798       opd = gvn().transform(new VectorMaskToLongNode(opd, TypeLong::LONG));
2799       opd = gvn().transform(new URShiftLNode(opd, pos));
2800       opd = gvn().transform(new AndLNode(opd, gvn().makecon(TypeLong::ONE)));
2801     } else {
2802       if (C->print_intrinsics()) {
2803         tty->print_cr("  ** Rejected mask extraction because architecture does not support it");
2804       }
2805       return false; // not supported
2806     }
2807   } else {
2808     // vbox_klass is vector. This is used for Vector.lane(int).
2809     if (!idx->is_con()) {
2810       if (C->print_intrinsics()) {
2811         tty->print_cr("  ** missing constant: idx=%s", NodeClassNames[argument(4)->Opcode()]);
2812       }
2813       return false; // not enough info for intrinsification
2814     }
2815 
2816     int vopc = ExtractNode::opcode(elem_bt);
2817     if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2818       if (C->print_intrinsics()) {
2819         tty->print_cr("  ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2820                       num_elem, type2name(elem_bt));
2821       }
2822       return false; // not supported
2823     }
2824 
2825     opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2826     if (opd == nullptr) {
2827       return false;
2828     }
2829     ConINode* idx_con = gvn().intcon(idx->get_con())->as_ConI();
2830 
2831     opd = gvn().transform(ExtractNode::make(opd, idx_con, elem_bt));
2832     switch (elem_bt) {
2833       case T_BYTE:
2834       case T_SHORT:
2835       case T_INT: {
2836         opd = gvn().transform(new ConvI2LNode(opd));
2837         break;
2838       }
2839       case T_FLOAT: {
2840         opd = gvn().transform(new MoveF2INode(opd));
2841         opd = gvn().transform(new ConvI2LNode(opd));
2842         break;
2843       }
2844       case T_DOUBLE: {
2845         opd = gvn().transform(new MoveD2LNode(opd));
2846         break;
2847       }
2848       case T_LONG: {
2849         // no conversion needed
2850         break;
2851       }
2852       default: fatal("%s", type2name(elem_bt));
2853     }
2854   }
2855   set_result(opd);
2856   return true;
2857 }
2858 
2859 // public static
2860 // <V extends Vector<E>,
2861 //  M extends VectorMask<E>,
2862 //  E>
2863 //  V compressExpandOp(int opr,
2864 //                    Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2865 //                    int length, V v, M m,
2866 //                    CompressExpandOperation<V, M> defaultImpl)
2867 bool LibraryCallKit::inline_vector_compress_expand() {
2868   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
2869   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2870   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
2871   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
2872   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
2873 
2874   if (opr          == nullptr || !opr->is_con() ||
2875       vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2876       mask_klass   == nullptr || mask_klass->const_oop()   == nullptr ||
2877       elem_klass   == nullptr || elem_klass->const_oop()   == nullptr ||
2878       vlen         == nullptr || !vlen->is_con()) {
2879     if (C->print_intrinsics()) {
2880       tty->print_cr("  ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2881                     NodeClassNames[argument(0)->Opcode()],
2882                     NodeClassNames[argument(1)->Opcode()],
2883                     NodeClassNames[argument(2)->Opcode()],
2884                     NodeClassNames[argument(3)->Opcode()],
2885                     NodeClassNames[argument(4)->Opcode()]);
2886     }
2887     return false; // not enough info for intrinsification
2888   }
2889 
2890   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2891     if (C->print_intrinsics()) {
2892       tty->print_cr("  ** klass argument not initialized");
2893     }
2894     return false;
2895   }
2896 
2897   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2898   if (!elem_type->is_primitive_type()) {
2899     if (C->print_intrinsics()) {
2900       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2901     }
2902     return false; // should be primitive type
2903   }
2904 
2905   int num_elem = vlen->get_con();
2906   BasicType elem_bt = elem_type->basic_type();
2907   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2908 
2909   if (!arch_supports_vector(opc, num_elem, elem_bt, VecMaskUseLoad)) {
2910     if (C->print_intrinsics()) {
2911       tty->print_cr("  ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2912                     opc, num_elem, type2name(elem_bt));
2913     }
2914     return false; // not supported
2915   }
2916 
2917   Node* opd1 = nullptr;
2918   const TypeInstPtr* vbox_type = nullptr;
2919   if (opc != Op_CompressM) {
2920     ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2921     vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2922     opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2923     if (opd1 == nullptr) {
2924       if (C->print_intrinsics()) {
2925         tty->print_cr("  ** unbox failed vector=%s",
2926                       NodeClassNames[argument(5)->Opcode()]);
2927       }
2928       return false;
2929     }
2930   }
2931 
2932   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2933   assert(is_vector_mask(mbox_klass), "argument(6) should be a mask class");
2934   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2935 
2936   Node* mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2937   if (mask == nullptr) {
2938     if (C->print_intrinsics()) {
2939       tty->print_cr("  ** unbox failed mask=%s",
2940                     NodeClassNames[argument(6)->Opcode()]);
2941     }
2942     return false;
2943   }
2944 
2945   const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
2946   Node* operation = gvn().transform(VectorNode::make(opc, opd1, mask, vt));
2947 
2948   // Wrap it up in VectorBox to keep object type information.
2949   const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2950   Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2951   set_result(vbox);
2952   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2953   return true;
2954 }
2955 
2956 // public static
2957 // <V extends Vector<E>,
2958 //  E,
2959 //  S extends VectorSpecies<E>>
2960 //  V indexVector(Class<? extends V> vClass, Class<E> eClass,
2961 //                int length,
2962 //                V v, int step, S s,
2963 //                IndexOperation<V, S> defaultImpl)
2964 bool LibraryCallKit::inline_index_vector() {
2965   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2966   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2967   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2968 
2969   if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2970       elem_klass   == nullptr || elem_klass->const_oop()   == nullptr ||
2971       vlen         == nullptr || !vlen->is_con() ) {
2972     if (C->print_intrinsics()) {
2973       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
2974                     NodeClassNames[argument(0)->Opcode()],
2975                     NodeClassNames[argument(1)->Opcode()],
2976                     NodeClassNames[argument(2)->Opcode()]);
2977     }
2978     return false; // not enough info for intrinsification
2979   }
2980 
2981   if (!is_klass_initialized(vector_klass)) {
2982     if (C->print_intrinsics()) {
2983       tty->print_cr("  ** klass argument not initialized");
2984     }
2985     return false;
2986   }
2987 
2988   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2989   if (!elem_type->is_primitive_type()) {
2990     if (C->print_intrinsics()) {
2991       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2992     }
2993     return false; // should be primitive type
2994   }
2995 
2996   int num_elem = vlen->get_con();
2997   BasicType elem_bt = elem_type->basic_type();
2998 
2999   // Check whether the iota index generation op is supported by the current hardware
3000   if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
3001     if (C->print_intrinsics()) {
3002       tty->print_cr("  ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3003     }
3004     return false; // not supported
3005   }
3006 
3007   int mul_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_MUL, elem_bt);
3008   int vmul_op = VectorNode::opcode(mul_op, elem_bt);
3009   bool needs_mul = true;
3010   Node* scale = argument(4);
3011   const TypeInt* scale_type = gvn().type(scale)->isa_int();
3012   // Multiply is not needed if the scale is a constant "1".
3013   if (scale_type && scale_type->is_con() && scale_type->get_con() == 1) {
3014     needs_mul = false;
3015   } else {
3016     // Check whether the vector multiply op is supported by the current hardware
3017     if (!arch_supports_vector(vmul_op, num_elem, elem_bt, VecMaskNotUsed)) {
3018       if (C->print_intrinsics()) {
3019         tty->print_cr("  ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3020       }
3021       return false; // not supported
3022     }
3023 
3024     // Check whether the scalar cast op is supported by the current hardware
3025     if (is_floating_point_type(elem_bt) || elem_bt == T_LONG) {
3026       int cast_op = elem_bt == T_LONG ? Op_ConvI2L :
3027                     elem_bt == T_FLOAT? Op_ConvI2F : Op_ConvI2D;
3028       if (!Matcher::match_rule_supported(cast_op)) {
3029         if (C->print_intrinsics()) {
3030           tty->print_cr("  ** Rejected op (%s) because architecture does not support it",
3031                         NodeClassNames[cast_op]);
3032         }
3033         return false; // not supported
3034       }
3035     }
3036   }
3037 
3038   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
3039   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
3040   Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
3041   if (opd == nullptr) {
3042     if (C->print_intrinsics()) {
3043       tty->print_cr("  ** unbox failed vector=%s",
3044                     NodeClassNames[argument(3)->Opcode()]);
3045     }
3046     return false;
3047   }
3048 
3049   int add_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_ADD, elem_bt);
3050   int vadd_op = VectorNode::opcode(add_op, elem_bt);
3051   bool needs_add = true;
3052   // The addition is not needed if all the element values of "opd" are zero
3053   if (VectorNode::is_all_zeros_vector(opd)) {
3054     needs_add = false;
3055   } else {
3056     // Check whether the vector addition op is supported by the current hardware
3057     if (!arch_supports_vector(vadd_op, num_elem, elem_bt, VecMaskNotUsed)) {
3058       if (C->print_intrinsics()) {
3059         tty->print_cr("  ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3060       }
3061       return false; // not supported
3062     }
3063   }
3064 
3065   // Compute the iota indice vector
3066   const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
3067   Node* index = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
3068 
3069   // Broadcast the "scale" to a vector, and multiply the "scale" with iota indice vector.
3070   if (needs_mul) {
3071     switch (elem_bt) {
3072       case T_BOOLEAN: // fall-through
3073       case T_BYTE:    // fall-through
3074       case T_SHORT:   // fall-through
3075       case T_CHAR:    // fall-through
3076       case T_INT: {
3077         // no conversion needed
3078         break;
3079       }
3080       case T_LONG: {
3081         scale = gvn().transform(new ConvI2LNode(scale));
3082         break;
3083       }
3084       case T_FLOAT: {
3085         scale = gvn().transform(new ConvI2FNode(scale));
3086         break;
3087       }
3088       case T_DOUBLE: {
3089         scale = gvn().transform(new ConvI2DNode(scale));
3090         break;
3091       }
3092       default: fatal("%s", type2name(elem_bt));
3093     }
3094     scale = gvn().transform(VectorNode::scalar2vector(scale, num_elem, Type::get_const_basic_type(elem_bt)));
3095     index = gvn().transform(VectorNode::make(vmul_op, index, scale, vt));
3096   }
3097 
3098   // Add "opd" if addition is needed.
3099   if (needs_add) {
3100     index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3101   }
3102   Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3103   set_result(vbox);
3104   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3105   return true;
3106 }
3107 
3108 // public static
3109 // <E,
3110 //  M extends VectorMask<E>>
3111 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3112 //                              long offset, long limit,
3113 //                              IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3114 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3115   const TypeInstPtr* mask_klass   = gvn().type(argument(0))->isa_instptr();
3116   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
3117   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
3118 
3119   if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
3120       elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
3121       vlen       == nullptr || !vlen->is_con()) {
3122     if (C->print_intrinsics()) {
3123       tty->print_cr("  ** missing constant: mclass=%s etype=%s vlen=%s",
3124                     NodeClassNames[argument(0)->Opcode()],
3125                     NodeClassNames[argument(1)->Opcode()],
3126                     NodeClassNames[argument(2)->Opcode()]);
3127     }
3128     return false; // not enough info for intrinsification
3129   }
3130 
3131   if (!is_klass_initialized(mask_klass)) {
3132     if (C->print_intrinsics()) {
3133       tty->print_cr("  ** klass argument not initialized");
3134     }
3135     return false;
3136   }
3137 
3138   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3139   if (!elem_type->is_primitive_type()) {
3140     if (C->print_intrinsics()) {
3141       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
3142     }
3143     return false; // should be primitive type
3144   }
3145 
3146   int num_elem = vlen->get_con();
3147   BasicType elem_bt = elem_type->basic_type();
3148 
3149   // Check whether the necessary ops are supported by current hardware.
3150   bool supports_mask_gen = arch_supports_vector(Op_VectorMaskGen, num_elem, elem_bt, VecMaskUseStore);
3151   if (!supports_mask_gen) {
3152     if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
3153         !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed) ||
3154         !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
3155       if (C->print_intrinsics()) {
3156         tty->print_cr("  ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3157       }
3158       return false; // not supported
3159     }
3160 
3161     // Check whether the scalar cast operation is supported by current hardware.
3162     if (elem_bt != T_LONG) {
3163       int cast_op = is_integral_type(elem_bt) ? Op_ConvL2I
3164                                               : (elem_bt == T_FLOAT ? Op_ConvL2F : Op_ConvL2D);
3165       if (!Matcher::match_rule_supported(cast_op)) {
3166         if (C->print_intrinsics()) {
3167           tty->print_cr("  ** Rejected op (%s) because architecture does not support it",
3168                         NodeClassNames[cast_op]);
3169         }
3170         return false; // not supported
3171       }
3172     }
3173   }
3174 
3175   Node* offset = argument(3);
3176   Node* limit = argument(5);
3177   if (offset == nullptr || limit == nullptr) {
3178     if (C->print_intrinsics()) {
3179       tty->print_cr("  ** offset or limit argument is null");
3180     }
3181     return false; // not supported
3182   }
3183 
3184   ciKlass* box_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
3185   assert(is_vector_mask(box_klass), "argument(0) should be a mask class");
3186   const TypeInstPtr* box_type = TypeInstPtr::make_exact(TypePtr::NotNull, box_klass);
3187 
3188   // We assume "offset > 0 && limit >= offset && limit - offset < num_elem".
3189   // So directly get indexLimit with "indexLimit = limit - offset".
3190   Node* indexLimit = gvn().transform(new SubLNode(limit, offset));
3191   Node* mask = nullptr;
3192   if (supports_mask_gen) {
3193     mask = gvn().transform(VectorMaskGenNode::make(indexLimit, elem_bt, num_elem));
3194   } else {
3195     // Generate the vector mask based on "mask = iota < indexLimit".
3196     // Broadcast "indexLimit" to a vector.
3197     switch (elem_bt) {
3198       case T_BOOLEAN: // fall-through
3199       case T_BYTE:    // fall-through
3200       case T_SHORT:   // fall-through
3201       case T_CHAR:    // fall-through
3202       case T_INT: {
3203         indexLimit = gvn().transform(new ConvL2INode(indexLimit));
3204         break;
3205       }
3206       case T_DOUBLE: {
3207         indexLimit = gvn().transform(new ConvL2DNode(indexLimit));
3208         break;
3209       }
3210       case T_FLOAT: {
3211         indexLimit = gvn().transform(new ConvL2FNode(indexLimit));
3212         break;
3213       }
3214       case T_LONG: {
3215         // no conversion needed
3216         break;
3217       }
3218       default: fatal("%s", type2name(elem_bt));
3219     }
3220     indexLimit = gvn().transform(VectorNode::scalar2vector(indexLimit, num_elem, Type::get_const_basic_type(elem_bt)));
3221 
3222     // Load the "iota" vector.
3223     const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
3224     Node* iota = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
3225 
3226     // Compute the vector mask with "mask = iota < indexLimit".
3227     ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::lt));
3228     const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
3229     mask = gvn().transform(new VectorMaskCmpNode(BoolTest::lt, iota, indexLimit, pred_node, vmask_type));
3230   }
3231   Node* vbox = box_vector(mask, box_type, elem_bt, num_elem);
3232   set_result(vbox);
3233   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3234   return true;
3235 }