1 /*
   2  * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "ci/ciSymbols.hpp"
  27 #include "classfile/vmSymbols.hpp"
  28 #include "opto/library_call.hpp"
  29 #include "opto/runtime.hpp"
  30 #include "opto/vectornode.hpp"
  31 #include "prims/vectorSupport.hpp"
  32 #include "runtime/stubRoutines.hpp"
  33 
  34 #ifdef ASSERT
  35 static bool is_vector(ciKlass* klass) {
  36   return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
  37 }
  38 
  39 static bool check_vbox(const TypeInstPtr* vbox_type) {
  40   assert(vbox_type->klass_is_exact(), "");
  41 
  42   ciInstanceKlass* ik = vbox_type->instance_klass();
  43   assert(is_vector(ik), "not a vector");
  44 
  45   ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
  46   assert(fd1 != nullptr, "element type info is missing");
  47 
  48   ciConstant val1 = fd1->constant_value();
  49   BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
  50   assert(is_java_primitive(elem_bt), "element type info is missing");
  51 
  52   ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
  53   assert(fd2 != nullptr, "vector length info is missing");
  54 
  55   ciConstant val2 = fd2->constant_value();
  56   assert(val2.as_int() > 0, "vector length info is missing");
  57 
  58   return true;
  59 }
  60 #endif
  61 
  62 static bool is_vector_mask(ciKlass* klass) {
  63   return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
  64 }
  65 
  66 static bool is_vector_shuffle(ciKlass* klass) {
  67   return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
  68 }
  69 
  70 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
  71                                                  VectorMaskUseType mask_use_type, bool has_scalar_args) {
  72   bool is_supported = true;
  73 
  74   // has_scalar_args flag is true only for non-constant scalar shift count,
  75   // since in this case shift needs to be broadcasted.
  76   if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
  77        (has_scalar_args && !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
  78     is_supported = false;
  79   }
  80 
  81   if (is_supported) {
  82     // Check if mask unboxing is supported, this is a two step process which first loads the contents
  83     // of boolean array into vector followed by either lane expansion to match the lane size of masked
  84     // vector operation or populate the predicate register.
  85     if ((mask_use_type & VecMaskUseLoad) != 0) {
  86       if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt) ||
  87           !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
  88       #ifndef PRODUCT
  89         if (C->print_intrinsics()) {
  90           tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
  91                         NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
  92         }
  93       #endif
  94         return false;
  95       }
  96     }
  97 
  98     if ((mask_use_type & VecMaskUsePred) != 0) {
  99       if (!Matcher::has_predicated_vectors() ||
 100           !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
 101       #ifndef PRODUCT
 102         if (C->print_intrinsics()) {
 103           tty->print_cr("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
 104                         NodeClassNames[opc], type2name(elem_bt), num_elem);
 105         }
 106       #endif
 107         return false;
 108       }
 109     }
 110   }
 111 
 112   int lshiftopc, rshiftopc;
 113   switch(elem_bt) {
 114     case T_BYTE:
 115       lshiftopc = Op_LShiftI;
 116       rshiftopc = Op_URShiftB;
 117       break;
 118     case T_SHORT:
 119       lshiftopc = Op_LShiftI;
 120       rshiftopc = Op_URShiftS;
 121       break;
 122     case T_INT:
 123       lshiftopc = Op_LShiftI;
 124       rshiftopc = Op_URShiftI;
 125       break;
 126     case T_LONG:
 127       lshiftopc = Op_LShiftL;
 128       rshiftopc = Op_URShiftL;
 129       break;
 130     default: fatal("Unexpected type: %s", type2name(elem_bt));
 131   }
 132   int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
 133   int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
 134   if (!is_supported &&
 135       arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
 136       arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
 137       arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
 138     is_supported = true;
 139   }
 140   return is_supported;
 141 }
 142 
 143 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
 144   assert(EnableVectorSupport, "");
 145 
 146   PreserveReexecuteState preexecs(this);
 147   jvms()->set_should_reexecute(true);
 148 
 149   VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
 150   set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
 151   make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
 152   set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
 153   set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
 154   Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
 155 
 156   assert(check_vbox(vbox_type), "");
 157   const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
 158   VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
 159   return gvn().transform(vbox);
 160 }
 161 
 162 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) {
 163   assert(EnableVectorSupport, "");
 164   const TypeInstPtr* vbox_type_v = gvn().type(v)->is_instptr();
 165   if (vbox_type->instance_klass() != vbox_type_v->instance_klass()) {
 166     return nullptr; // arguments don't agree on vector shapes
 167   }
 168   if (vbox_type_v->maybe_null()) {
 169     return nullptr; // no nulls are allowed
 170   }
 171   assert(check_vbox(vbox_type), "");
 172   const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
 173   Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector));
 174   return unbox;
 175 }
 176 
 177 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
 178   assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
 179   juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
 180   Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
 181   Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
 182   return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
 183 }
 184 
 185 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
 186   // Check that the operation is valid.
 187   if (sopc <= 0) {
 188 #ifndef PRODUCT
 189     if (C->print_intrinsics()) {
 190       tty->print_cr("  ** Rejected intrinsification because no valid vector op could be extracted");
 191     }
 192 #endif
 193     return false;
 194   }
 195 
 196   if (VectorNode::is_vector_rotate(sopc)) {
 197     if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
 198 #ifndef PRODUCT
 199       if (C->print_intrinsics()) {
 200         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
 201                       NodeClassNames[sopc], type2name(type), num_elem);
 202       }
 203 #endif
 204       return false;
 205     }
 206   } else if (VectorNode::is_vector_integral_negate(sopc)) {
 207     if (!VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, false)) {
 208 #ifndef PRODUCT
 209       if (C->print_intrinsics()) {
 210         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support integral vector negate",
 211                       NodeClassNames[sopc], type2name(type), num_elem);
 212       }
 213 #endif
 214       return false;
 215     }
 216   } else {
 217     // Check that architecture supports this op-size-type combination.
 218     if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
 219 #ifndef PRODUCT
 220       if (C->print_intrinsics()) {
 221         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support it",
 222                       NodeClassNames[sopc], type2name(type), num_elem);
 223       }
 224 #endif
 225       return false;
 226     } else {
 227       assert(Matcher::match_rule_supported(sopc), "must be supported");
 228     }
 229   }
 230 
 231   if (num_elem == 1) {
 232     if (mask_use_type != VecMaskNotUsed) {
 233 #ifndef PRODUCT
 234       if (C->print_intrinsics()) {
 235         tty->print_cr("  ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
 236                       NodeClassNames[sopc], type2name(type), num_elem);
 237       }
 238 #endif
 239       return false;
 240     }
 241 
 242     if (sopc != 0) {
 243       if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
 244 #ifndef PRODUCT
 245         if (C->print_intrinsics()) {
 246           tty->print_cr("  ** Not a svml call or load/store vector op (%s,%s,%d)",
 247                         NodeClassNames[sopc], type2name(type), num_elem);
 248         }
 249 #endif
 250         return false;
 251       }
 252     }
 253   }
 254 
 255   if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
 256       Matcher::supports_vector_variable_shifts() == false) {
 257     if (C->print_intrinsics()) {
 258       tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
 259                     NodeClassNames[sopc], type2name(type), num_elem);
 260     }
 261     return false;
 262   }
 263 
 264   // Check if mask unboxing is supported, this is a two step process which first loads the contents
 265   // of boolean array into vector followed by either lane expansion to match the lane size of masked
 266   // vector operation or populate the predicate register.
 267   if ((mask_use_type & VecMaskUseLoad) != 0) {
 268     if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type) ||
 269         !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
 270     #ifndef PRODUCT
 271       if (C->print_intrinsics()) {
 272         tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
 273                       NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
 274       }
 275     #endif
 276       return false;
 277     }
 278   }
 279 
 280   // Check if mask boxing is supported, this is a two step process which first stores the contents
 281   // of mask vector / predicate register into a boolean vector followed by vector store operation to
 282   // transfer the contents to underlined storage of mask boxes which is a boolean array.
 283   if ((mask_use_type & VecMaskUseStore) != 0) {
 284     if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type) ||
 285         !Matcher::match_rule_supported_vector(Op_StoreVector, num_elem, T_BOOLEAN)) {
 286     #ifndef PRODUCT
 287       if (C->print_intrinsics()) {
 288         tty->print_cr("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
 289                       NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
 290       }
 291     #endif
 292       return false;
 293     }
 294   }
 295 
 296   if ((mask_use_type & VecMaskUsePred) != 0) {
 297     bool is_supported = false;
 298     if (Matcher::has_predicated_vectors()) {
 299       if (VectorNode::is_vector_integral_negate(sopc)) {
 300         is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
 301       } else {
 302         is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
 303       }
 304     }
 305 
 306     if (!is_supported) {
 307     #ifndef PRODUCT
 308       if (C->print_intrinsics()) {
 309         tty->print_cr("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
 310                       NodeClassNames[sopc], type2name(type), num_elem);
 311       }
 312     #endif
 313       return false;
 314     }
 315   }
 316 
 317   return true;
 318 }
 319 
 320 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
 321   if (vec_klass->const_oop() == nullptr) {
 322     return false; // uninitialized or some kind of unsafe access
 323   }
 324   assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != nullptr, "klass instance expected");
 325   ciInstanceKlass* klass =  vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
 326   return klass->is_initialized();
 327 }
 328 
 329 // public static
 330 // <V extends Vector<E>,
 331 //  M extends VectorMask<E>,
 332 //  E>
 333 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 334 //           int length, V v, M m,
 335 //           UnaryOperation<V, M> defaultImpl)
 336 //
 337 // public static
 338 // <V,
 339 //  M extends VectorMask<E>,
 340 //  E>
 341 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 342 //            int length, V v1, V v2, M m,
 343 //            BinaryOperation<V, M> defaultImpl)
 344 //
 345 // public static
 346 // <V extends Vector<E>,
 347 //  M extends VectorMask<E>,
 348 //  E>
 349 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 350 //             int length, V v1, V v2, V v3, M m,
 351 //             TernaryOperation<V, M> defaultImpl)
 352 //
 353 bool LibraryCallKit::inline_vector_nary_operation(int n) {
 354   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
 355   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
 356   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
 357   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
 358   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
 359 
 360   if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
 361       !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
 362     if (C->print_intrinsics()) {
 363       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
 364                     NodeClassNames[argument(0)->Opcode()],
 365                     NodeClassNames[argument(1)->Opcode()],
 366                     NodeClassNames[argument(3)->Opcode()],
 367                     NodeClassNames[argument(4)->Opcode()]);
 368     }
 369     return false; // not enough info for intrinsification
 370   }
 371 
 372   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 373   if (!elem_type->is_primitive_type()) {
 374     if (C->print_intrinsics()) {
 375       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 376     }
 377     return false; // should be primitive type
 378   }
 379   if (!is_klass_initialized(vector_klass)) {
 380     if (C->print_intrinsics()) {
 381       tty->print_cr("  ** klass argument not initialized");
 382     }
 383     return false;
 384   }
 385 
 386   // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
 387   // is used to control this operation.
 388   const Type* vmask_type = gvn().type(argument(n + 5));
 389   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
 390   if (is_masked_op) {
 391     if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
 392       if (C->print_intrinsics()) {
 393         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
 394       }
 395       return false; // not enough info for intrinsification
 396     }
 397 
 398     if (!is_klass_initialized(mask_klass)) {
 399       if (C->print_intrinsics()) {
 400         tty->print_cr("  ** mask klass argument not initialized");
 401       }
 402       return false;
 403     }
 404 
 405     if (vmask_type->maybe_null()) {
 406       if (C->print_intrinsics()) {
 407         tty->print_cr("  ** null mask values are not allowed for masked op");
 408       }
 409       return false;
 410     }
 411   }
 412 
 413   BasicType elem_bt = elem_type->basic_type();
 414   int num_elem = vlen->get_con();
 415   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
 416   int sopc = VectorNode::opcode(opc, elem_bt);
 417   if ((opc != Op_CallLeafVector) && (sopc == 0)) {
 418     if (C->print_intrinsics()) {
 419       tty->print_cr("  ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
 420     }
 421     return false; // operation not supported
 422   }
 423   if (num_elem == 1) {
 424     if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
 425       if (C->print_intrinsics()) {
 426         tty->print_cr("  ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
 427                       n, opc, num_elem, type2name(elem_bt));
 428       }
 429       return false;
 430     }
 431   }
 432   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 433   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 434 
 435   if (is_vector_mask(vbox_klass)) {
 436     assert(!is_masked_op, "mask operations do not need mask to control");
 437   }
 438 
 439   if (opc == Op_CallLeafVector) {
 440     if (!UseVectorStubs) {
 441       if (C->print_intrinsics()) {
 442         tty->print_cr("  ** vector stubs support is disabled");
 443       }
 444       return false;
 445     }
 446     if (!Matcher::supports_vector_calling_convention()) {
 447       if (C->print_intrinsics()) {
 448         tty->print_cr("  ** no vector calling conventions supported");
 449       }
 450       return false;
 451     }
 452     if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
 453       if (C->print_intrinsics()) {
 454         tty->print_cr("  ** vector size (vlen=%d, etype=%s) is not supported",
 455                       num_elem, type2name(elem_bt));
 456       }
 457       return false;
 458     }
 459   }
 460 
 461   // When using mask, mask use type needs to be VecMaskUseLoad.
 462   VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
 463                                       : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
 464   if ((sopc != 0) && !arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
 465     if (C->print_intrinsics()) {
 466       tty->print_cr("  ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
 467                     n, sopc, num_elem, type2name(elem_bt),
 468                     is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
 469     }
 470     return false; // not supported
 471   }
 472 
 473   // Return true if current platform has implemented the masked operation with predicate feature.
 474   bool use_predicate = is_masked_op && sopc != 0 && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
 475   if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
 476     if (C->print_intrinsics()) {
 477       tty->print_cr("  ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
 478                     n, sopc, num_elem, type2name(elem_bt));
 479     }
 480     return false;
 481   }
 482 
 483   Node* opd1 = nullptr; Node* opd2 = nullptr; Node* opd3 = nullptr;
 484   switch (n) {
 485     case 3: {
 486       opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
 487       if (opd3 == nullptr) {
 488         if (C->print_intrinsics()) {
 489           tty->print_cr("  ** unbox failed v3=%s",
 490                         NodeClassNames[argument(7)->Opcode()]);
 491         }
 492         return false;
 493       }
 494       // fall-through
 495     }
 496     case 2: {
 497       opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
 498       if (opd2 == nullptr) {
 499         if (C->print_intrinsics()) {
 500           tty->print_cr("  ** unbox failed v2=%s",
 501                         NodeClassNames[argument(6)->Opcode()]);
 502         }
 503         return false;
 504       }
 505       // fall-through
 506     }
 507     case 1: {
 508       opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
 509       if (opd1 == nullptr) {
 510         if (C->print_intrinsics()) {
 511           tty->print_cr("  ** unbox failed v1=%s",
 512                         NodeClassNames[argument(5)->Opcode()]);
 513         }
 514         return false;
 515       }
 516       break;
 517     }
 518     default: fatal("unsupported arity: %d", n);
 519   }
 520 
 521   Node* mask = nullptr;
 522   if (is_masked_op) {
 523     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
 524     assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
 525     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
 526     mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
 527     if (mask == nullptr) {
 528       if (C->print_intrinsics()) {
 529         tty->print_cr("  ** unbox failed mask=%s",
 530                       NodeClassNames[argument(n + 5)->Opcode()]);
 531       }
 532       return false;
 533     }
 534   }
 535 
 536   Node* operation = nullptr;
 537   if (opc == Op_CallLeafVector) {
 538     assert(UseVectorStubs, "sanity");
 539     operation = gen_call_to_svml(opr->get_con(), elem_bt, num_elem, opd1, opd2);
 540     if (operation == nullptr) {
 541       if (C->print_intrinsics()) {
 542         tty->print_cr("  ** svml call failed for %s_%s_%d",
 543                          (elem_bt == T_FLOAT)?"float":"double",
 544                          VectorSupport::svmlname[opr->get_con() - VectorSupport::VECTOR_OP_SVML_START],
 545                          num_elem * type2aelembytes(elem_bt));
 546       }
 547       return false;
 548      }
 549   } else {
 550     const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
 551     switch (n) {
 552       case 1:
 553       case 2: {
 554         operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc));
 555         break;
 556       }
 557       case 3: {
 558         operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
 559         break;
 560       }
 561       default: fatal("unsupported arity: %d", n);
 562     }
 563   }
 564 
 565   if (is_masked_op && mask != nullptr) {
 566     if (use_predicate) {
 567       operation->add_req(mask);
 568       operation->add_flag(Node::Flag_is_predicated_vector);
 569     } else {
 570       operation->add_flag(Node::Flag_is_predicated_using_blend);
 571       operation = gvn().transform(operation);
 572       operation = new VectorBlendNode(opd1, operation, mask);
 573     }
 574   }
 575   operation = gvn().transform(operation);
 576 
 577   // Wrap it up in VectorBox to keep object type information.
 578   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
 579   set_result(vbox);
 580   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 581   return true;
 582 }
 583 
 584 // <Sh extends VectorShuffle<E>,  E>
 585 //  Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
 586 //                  int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
 587 bool LibraryCallKit::inline_vector_shuffle_iota() {
 588   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
 589   const TypeInt*     vlen          = gvn().type(argument(3))->isa_int();
 590   const TypeInt*     start_val     = gvn().type(argument(4))->isa_int();
 591   const TypeInt*     step_val      = gvn().type(argument(5))->isa_int();
 592   const TypeInt*     wrap          = gvn().type(argument(6))->isa_int();
 593 
 594   if (shuffle_klass == nullptr || shuffle_klass->const_oop() == nullptr ||
 595       vlen == nullptr || !vlen->is_con() || start_val == nullptr || step_val == nullptr ||
 596       wrap == nullptr || !wrap->is_con()) {
 597     return false; // not enough info for intrinsification
 598   }
 599 
 600   if (!is_klass_initialized(shuffle_klass)) {
 601     if (C->print_intrinsics()) {
 602       tty->print_cr("  ** klass argument not initialized");
 603     }
 604     return false;
 605   }
 606 
 607   int do_wrap = wrap->get_con();
 608   int num_elem = vlen->get_con();
 609   BasicType elem_bt = T_BYTE;
 610 
 611   bool effective_indices_in_range = false;
 612   if (start_val->is_con() && step_val->is_con()) {
 613     int effective_min_index = start_val->get_con();
 614     int effective_max_index = start_val->get_con() + step_val->get_con() * (num_elem - 1);
 615     effective_indices_in_range = effective_max_index >= effective_min_index && effective_min_index >= -128 && effective_max_index <= 127;
 616   }
 617 
 618   if (!do_wrap && !effective_indices_in_range) {
 619     // Disable instrinsification for unwrapped shuffle iota if start/step
 620     // values are non-constant OR if intermediate result overflows byte value range.
 621     return false;
 622   }
 623 
 624   if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed)           ||
 625       !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)            ||
 626       !arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
 627       !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed)) {
 628     return false;
 629   }
 630 
 631   if (!do_wrap &&
 632       (!arch_supports_vector(Op_SubVB, num_elem, elem_bt, VecMaskNotUsed)       ||
 633       !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskNotUsed)  ||
 634       !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskNotUsed))) {
 635     return false;
 636   }
 637 
 638   bool step_multiply = !step_val->is_con() || !is_power_of_2(step_val->get_con());
 639   if ((step_multiply && !arch_supports_vector(Op_MulVB, num_elem, elem_bt, VecMaskNotUsed)) ||
 640       (!step_multiply && !arch_supports_vector(Op_LShiftVB, num_elem, elem_bt, VecMaskNotUsed))) {
 641     return false;
 642   }
 643 
 644   const Type * type_bt = Type::get_const_basic_type(elem_bt);
 645   const TypeVect * vt  = TypeVect::make(type_bt, num_elem);
 646 
 647   Node* res = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
 648 
 649   Node* start = argument(4);
 650   Node* step  = argument(5);
 651 
 652   if (step_multiply) {
 653     Node* bcast_step     = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt));
 654     res = gvn().transform(VectorNode::make(Op_MulVB, res, bcast_step, vt));
 655   } else if (step_val->get_con() > 1) {
 656     Node* cnt = gvn().makecon(TypeInt::make(log2i_exact(step_val->get_con())));
 657     Node* shift_cnt = vector_shift_count(cnt, Op_LShiftI, elem_bt, num_elem);
 658     res = gvn().transform(VectorNode::make(Op_LShiftVB, res, shift_cnt, vt));
 659   }
 660 
 661   if (!start_val->is_con() || start_val->get_con() != 0) {
 662     Node* bcast_start    = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt));
 663     res = gvn().transform(VectorNode::make(Op_AddVB, res, bcast_start, vt));
 664   }
 665 
 666   Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1));
 667   Node * bcast_mod  = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt));
 668 
 669   if (do_wrap)  {
 670     // Wrap the indices greater than lane count.
 671      res = gvn().transform(VectorNode::make(Op_AndV, res, bcast_mod, vt));
 672   } else {
 673     ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::ugt));
 674     Node * lane_cnt  = gvn().makecon(TypeInt::make(num_elem));
 675     Node * bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt));
 676     const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
 677     Node* mask = gvn().transform(new VectorMaskCmpNode(BoolTest::ugt, bcast_lane_cnt, res, pred_node, vmask_type));
 678 
 679     // Make the indices greater than lane count as -ve values to match the java side implementation.
 680     res = gvn().transform(VectorNode::make(Op_AndV, res, bcast_mod, vt));
 681     Node * biased_val = gvn().transform(VectorNode::make(Op_SubVB, res, bcast_lane_cnt, vt));
 682     res = gvn().transform(new VectorBlendNode(biased_val, res, mask));
 683   }
 684 
 685   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 686   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 687 
 688   // Wrap it up in VectorBox to keep object type information.
 689   res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
 690   set_result(res);
 691   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 692   return true;
 693 }
 694 
 695 // <E, M>
 696 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
 697 //                          int length, M m, VectorMaskOp<M> defaultImpl)
 698 bool LibraryCallKit::inline_vector_mask_operation() {
 699   const TypeInt*     oper       = gvn().type(argument(0))->isa_int();
 700   const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
 701   const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
 702   const TypeInt*     vlen       = gvn().type(argument(3))->isa_int();
 703   Node*              mask       = argument(4);
 704 
 705   if (mask_klass == nullptr || elem_klass == nullptr || mask->is_top() || vlen == nullptr) {
 706     return false; // dead code
 707   }
 708 
 709   if (!is_klass_initialized(mask_klass)) {
 710     if (C->print_intrinsics()) {
 711       tty->print_cr("  ** klass argument not initialized");
 712     }
 713     return false;
 714   }
 715 
 716   int num_elem = vlen->get_con();
 717   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 718   BasicType elem_bt = elem_type->basic_type();
 719 
 720   int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
 721   if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
 722     if (C->print_intrinsics()) {
 723       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 724                     mopc, num_elem, type2name(elem_bt));
 725     }
 726     return false; // not supported
 727   }
 728 
 729   const Type* elem_ty = Type::get_const_basic_type(elem_bt);
 730   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
 731   const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
 732   Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
 733   if (mask_vec == nullptr) {
 734     if (C->print_intrinsics()) {
 735         tty->print_cr("  ** unbox failed mask=%s",
 736                       NodeClassNames[argument(4)->Opcode()]);
 737     }
 738     return false;
 739   }
 740 
 741   if (mask_vec->bottom_type()->isa_vectmask() == nullptr) {
 742     mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
 743   }
 744   const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
 745   Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
 746   if (mopc != Op_VectorMaskToLong) {
 747     maskoper = ConvI2L(maskoper);
 748   }
 749   set_result(maskoper);
 750 
 751   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 752   return true;
 753 }
 754 
 755 // public static
 756 // <V,
 757 //  Sh extends VectorShuffle<E>,
 758 //  E>
 759 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
 760 //                   Class<? extends Sh> shuffleClass, Sh s, int length,
 761 //                   ShuffleToVectorOperation<V, Sh, E> defaultImpl)
 762 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
 763   const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
 764   const TypeInstPtr* elem_klass    = gvn().type(argument(1))->isa_instptr();
 765   const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
 766   Node*              shuffle       = argument(3);
 767   const TypeInt*     vlen          = gvn().type(argument(4))->isa_int();
 768 
 769   if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) {
 770     return false; // dead code
 771   }
 772   if (!vlen->is_con() || vector_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr) {
 773     return false; // not enough info for intrinsification
 774   }
 775   if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
 776     if (C->print_intrinsics()) {
 777       tty->print_cr("  ** klass argument not initialized");
 778     }
 779     return false;
 780   }
 781 
 782   int num_elem = vlen->get_con();
 783   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 784   BasicType elem_bt = elem_type->basic_type();
 785 
 786   if (num_elem < 4) {
 787     return false;
 788   }
 789 
 790   int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE
 791   // Make sure that cast is implemented to particular type/size combination.
 792   if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
 793     if (C->print_intrinsics()) {
 794       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 795         cast_vopc, num_elem, type2name(elem_bt));
 796     }
 797     return false;
 798   }
 799 
 800   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 801   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 802 
 803   // Unbox shuffle with true flag to indicate its load shuffle to vector
 804   // shuffle is a byte array
 805   Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, T_BYTE, num_elem, true);
 806 
 807   // cast byte to target element type
 808   shuffle_vec = gvn().transform(VectorCastNode::make(cast_vopc, shuffle_vec, elem_bt, num_elem));
 809 
 810   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 811   const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 812 
 813   // Box vector
 814   Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem);
 815   set_result(res);
 816   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 817   return true;
 818 }
 819 
 820 // public static
 821 // <M,
 822 //  S extends VectorSpecies<E>,
 823 //  E>
 824 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
 825 //                    long bits, int mode, S s,
 826 //                    BroadcastOperation<M, E, S> defaultImpl)
 827 bool LibraryCallKit::inline_vector_frombits_coerced() {
 828   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 829   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 830   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 831   const TypeLong*    bits_type    = gvn().type(argument(3))->isa_long();
 832   // Mode argument determines the mode of operation it can take following values:-
 833   // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
 834   // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
 835   const TypeInt*     mode         = gvn().type(argument(5))->isa_int();
 836 
 837   if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || mode == nullptr ||
 838       bits_type == nullptr || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr ||
 839       !vlen->is_con() || !mode->is_con()) {
 840     if (C->print_intrinsics()) {
 841       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
 842                     NodeClassNames[argument(0)->Opcode()],
 843                     NodeClassNames[argument(1)->Opcode()],
 844                     NodeClassNames[argument(2)->Opcode()],
 845                     NodeClassNames[argument(5)->Opcode()]);
 846     }
 847     return false; // not enough info for intrinsification
 848   }
 849 
 850   if (!is_klass_initialized(vector_klass)) {
 851     if (C->print_intrinsics()) {
 852       tty->print_cr("  ** klass argument not initialized");
 853     }
 854     return false;
 855   }
 856   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 857   if (!elem_type->is_primitive_type()) {
 858     if (C->print_intrinsics()) {
 859       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 860     }
 861     return false; // should be primitive type
 862   }
 863   BasicType elem_bt = elem_type->basic_type();
 864   int num_elem = vlen->get_con();
 865   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 866   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 867 
 868   bool is_mask = is_vector_mask(vbox_klass);
 869   int  bcast_mode = mode->get_con();
 870   VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
 871   int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : Op_Replicate;
 872 
 873   if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
 874     if (C->print_intrinsics()) {
 875       tty->print_cr("  ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
 876                     num_elem, type2name(elem_bt),
 877                     is_mask ? 1 : 0,
 878                     bcast_mode);
 879     }
 880     return false; // not supported
 881   }
 882 
 883   Node* broadcast = nullptr;
 884   Node* bits = argument(3);
 885   Node* elem = bits;
 886 
 887   if (opc == Op_VectorLongToMask) {
 888     const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
 889     if (vt->isa_vectmask()) {
 890       broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
 891     } else {
 892       const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
 893       broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
 894       broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
 895     }
 896   } else {
 897     switch (elem_bt) {
 898       case T_BOOLEAN: // fall-through
 899       case T_BYTE:    // fall-through
 900       case T_SHORT:   // fall-through
 901       case T_CHAR:    // fall-through
 902       case T_INT: {
 903         elem = gvn().transform(new ConvL2INode(bits));
 904         break;
 905       }
 906       case T_DOUBLE: {
 907         elem = gvn().transform(new MoveL2DNode(bits));
 908         break;
 909       }
 910       case T_FLOAT: {
 911         bits = gvn().transform(new ConvL2INode(bits));
 912         elem = gvn().transform(new MoveI2FNode(bits));
 913         break;
 914       }
 915       case T_LONG: {
 916         // no conversion needed
 917         break;
 918       }
 919       default: fatal("%s", type2name(elem_bt));
 920     }
 921     broadcast = VectorNode::scalar2vector(elem, num_elem, Type::get_const_basic_type(elem_bt), is_mask);
 922     broadcast = gvn().transform(broadcast);
 923   }
 924 
 925   Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
 926   set_result(box);
 927   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 928   return true;
 929 }
 930 
 931 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type, bool mismatched_ms) {
 932   assert(arr_type != nullptr, "unexpected");
 933   BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
 934   if (elem_bt == arr_elem_bt) {
 935     return true;
 936   } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
 937     // Load/store of short vector from/to char[] is supported
 938     return true;
 939   } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
 940     // Load/store of byte vector from/to boolean[] is supported
 941     return true;
 942   } else {
 943     return mismatched_ms;
 944   }
 945 }
 946 
 947 //  public static
 948 //  <C,
 949 //   VM extends VectorPayload,
 950 //   E,
 951 //   S extends VectorSpecies<E>>
 952 //  VM load(Class<? extends VM> vmClass, Class<E> eClass,
 953 //          int length,
 954 //          Object base, long offset,            // Unsafe addressing
 955 //          boolean fromSegment,
 956 //          C container, long index, S s,        // Arguments for default implementation
 957 //          LoadOperation<C, VM, S> defaultImpl) {
 958 //  public static
 959 //  <C,
 960 //   V extends VectorPayload>
 961 //  void store(Class<?> vClass, Class<?> eClass,
 962 //             int length,
 963 //             Object base, long offset,        // Unsafe addressing
 964 //             boolean fromSegment,
 965 //             V v, C container, long index,    // Arguments for default implementation
 966 //             StoreVectorOperation<C, V> defaultImpl) {
 967 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
 968   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 969   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 970   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 971   const TypeInt*     from_ms      = gvn().type(argument(6))->isa_int();
 972 
 973   if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || !from_ms->is_con() ||
 974       vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
 975     if (C->print_intrinsics()) {
 976       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
 977                     NodeClassNames[argument(0)->Opcode()],
 978                     NodeClassNames[argument(1)->Opcode()],
 979                     NodeClassNames[argument(2)->Opcode()],
 980                     NodeClassNames[argument(6)->Opcode()]);
 981     }
 982     return false; // not enough info for intrinsification
 983   }
 984   if (!is_klass_initialized(vector_klass)) {
 985     if (C->print_intrinsics()) {
 986       tty->print_cr("  ** klass argument not initialized");
 987     }
 988     return false;
 989   }
 990 
 991   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 992   if (!elem_type->is_primitive_type()) {
 993     if (C->print_intrinsics()) {
 994       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 995     }
 996     return false; // should be primitive type
 997   }
 998   BasicType elem_bt = elem_type->basic_type();
 999   int num_elem = vlen->get_con();
1000 
1001   // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
1002   if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
1003     if (C->print_intrinsics()) {
1004       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
1005                     is_store, is_store ? "store" : "load",
1006                     num_elem, type2name(elem_bt));
1007     }
1008     return false; // not supported
1009   }
1010 
1011   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1012   bool is_mask = is_vector_mask(vbox_klass);
1013 
1014   Node* base = argument(3);
1015   Node* offset = ConvL2X(argument(4));
1016 
1017   // Save state and restore on bailout
1018   uint old_sp = sp();
1019   SafePointNode* old_map = clone_map();
1020 
1021   Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
1022 
1023   // The memory barrier checks are based on ones for unsafe access.
1024   // This is not 1-1 implementation.
1025   const Type *const base_type = gvn().type(base);
1026 
1027   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1028   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1029 
1030   const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
1031   const bool in_heap   = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
1032 
1033   const bool is_mixed_access = !in_heap && !in_native;
1034 
1035   const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == nullptr);
1036 
1037   const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
1038 
1039   bool mismatched_ms = from_ms->get_con() && !is_mask && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
1040   BasicType mem_elem_bt = mismatched_ms ? arr_type->elem()->array_element_basic_type() : elem_bt;
1041   int mem_num_elem = mismatched_ms ? (num_elem * type2aelembytes(elem_bt)) / type2aelembytes(mem_elem_bt) : num_elem;
1042   if (arr_type != nullptr && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
1043     if (C->print_intrinsics()) {
1044       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1045                     is_store, is_store ? "store" : "load",
1046                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1047     }
1048     set_map(old_map);
1049     set_sp(old_sp);
1050     return false;
1051   }
1052 
1053   // In case of mismatched memory segment accesses, we need to double check that the source type memory operations are supported by backend.
1054   if (mismatched_ms) {
1055     if (is_store) {
1056       if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskNotUsed)
1057           || !arch_supports_vector(Op_VectorReinterpret, mem_num_elem, mem_elem_bt, VecMaskNotUsed)) {
1058         if (C->print_intrinsics()) {
1059           tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
1060                         is_store, "store",
1061                         num_elem, type2name(elem_bt));
1062         }
1063         set_map(old_map);
1064         set_sp(old_sp);
1065         return false; // not supported
1066       }
1067     } else {
1068       if (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed)
1069           || !arch_supports_vector(Op_VectorReinterpret, num_elem, elem_bt, VecMaskNotUsed)) {
1070         if (C->print_intrinsics()) {
1071           tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
1072                         is_store, "load",
1073                         mem_num_elem, type2name(mem_elem_bt));
1074         }
1075         set_map(old_map);
1076         set_sp(old_sp);
1077         return false; // not supported
1078       }
1079     }
1080   }
1081   if (is_mask) {
1082     if (!is_store) {
1083       if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1084         set_map(old_map);
1085         set_sp(old_sp);
1086         return false; // not supported
1087       }
1088     } else {
1089       if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
1090         set_map(old_map);
1091         set_sp(old_sp);
1092         return false; // not supported
1093       }
1094     }
1095   }
1096 
1097   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1098 
1099   if (needs_cpu_membar) {
1100     insert_mem_bar(Op_MemBarCPUOrder);
1101   }
1102 
1103   if (is_store) {
1104     Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
1105     if (val == nullptr) {
1106       set_map(old_map);
1107       set_sp(old_sp);
1108       return false; // operand unboxing failed
1109     }
1110     set_all_memory(reset_memory());
1111 
1112     // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
1113     int store_num_elem = num_elem;
1114     if (mismatched_ms) {
1115       store_num_elem = mem_num_elem;
1116       const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, store_num_elem);
1117       val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1118     }
1119     if (is_mask) {
1120       val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
1121     }
1122     Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
1123     set_memory(vstore, addr_type);
1124   } else {
1125     // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
1126     Node* vload = nullptr;
1127     if (mismatched_ms) {
1128       vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1129       const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1130       vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1131     } else {
1132       // Special handle for masks
1133       if (is_mask) {
1134         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
1135         vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
1136       } else {
1137         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
1138       }
1139     }
1140     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1141     set_result(box);
1142   }
1143 
1144   destruct_map_clone(old_map);
1145 
1146   if (needs_cpu_membar) {
1147     insert_mem_bar(Op_MemBarCPUOrder);
1148   }
1149 
1150   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1151   return true;
1152 }
1153 
1154 //  public static
1155 //  <C,
1156 //   V extends Vector<?>,
1157 //   E,
1158 //   S extends VectorSpecies<E>,
1159 //   M extends VectorMask<E>>
1160 //  V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1161 //               int length, Object base, long offset,          // Unsafe addressing
1162 //               boolean fromSegment,
1163 //               M m, int offsetInRange,
1164 //               C container, long index, S s,                  // Arguments for default implementation
1165 //               LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1166 //  public static
1167 //  <C,
1168 //   V extends Vector<E>,
1169 //   M extends VectorMask<E>,
1170 //   E>
1171 //  void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1172 //                   int length,
1173 //                   Object base, long offset,                  // Unsafe addressing
1174 //                   boolean fromSegment,
1175 //                   V v, M m, C container, long index,         // Arguments for default implementation
1176 //                   StoreVectorMaskedOperation<C, V, M> defaultImpl) {
1177 
1178 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1179   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1180   const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1181   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1182   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1183   const TypeInt*     from_ms      = gvn().type(argument(7))->isa_int();
1184 
1185   if (vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1186       vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr || from_ms == nullptr ||
1187       elem_klass->const_oop() == nullptr || !vlen->is_con() || !from_ms->is_con()) {
1188     if (C->print_intrinsics()) {
1189       tty->print_cr("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1190                     NodeClassNames[argument(0)->Opcode()],
1191                     NodeClassNames[argument(1)->Opcode()],
1192                     NodeClassNames[argument(2)->Opcode()],
1193                     NodeClassNames[argument(3)->Opcode()],
1194                     NodeClassNames[argument(7)->Opcode()]);
1195     }
1196     return false; // not enough info for intrinsification
1197   }
1198   if (!is_klass_initialized(vector_klass)) {
1199     if (C->print_intrinsics()) {
1200       tty->print_cr("  ** klass argument not initialized");
1201     }
1202     return false;
1203   }
1204 
1205   if (!is_klass_initialized(mask_klass)) {
1206     if (C->print_intrinsics()) {
1207       tty->print_cr("  ** mask klass argument not initialized");
1208     }
1209     return false;
1210   }
1211 
1212   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1213   if (!elem_type->is_primitive_type()) {
1214     if (C->print_intrinsics()) {
1215       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1216     }
1217     return false; // should be primitive type
1218   }
1219 
1220   BasicType elem_bt = elem_type->basic_type();
1221   int num_elem = vlen->get_con();
1222 
1223   Node* base = argument(4);
1224   Node* offset = ConvL2X(argument(5));
1225 
1226   // Save state and restore on bailout
1227   uint old_sp = sp();
1228   SafePointNode* old_map = clone_map();
1229 
1230   Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1231   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1232   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1233 
1234   bool mismatched_ms = from_ms->get_con() && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
1235   BIG_ENDIAN_ONLY(if (mismatched_ms) return false;)
1236   // If there is no consistency between array and vector element types, it must be special byte array case
1237   if (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
1238     if (C->print_intrinsics()) {
1239       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
1240                     is_store, is_store ? "storeMasked" : "loadMasked",
1241                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1242     }
1243     set_map(old_map);
1244     set_sp(old_sp);
1245     return false;
1246   }
1247 
1248   int mem_num_elem = mismatched_ms ? num_elem * type2aelembytes(elem_bt) : num_elem;
1249   BasicType mem_elem_bt = mismatched_ms ? T_BYTE : elem_bt;
1250   bool supports_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
1251                                                 mem_num_elem, mem_elem_bt, VecMaskUseLoad);
1252 
1253   // If current arch does not support the predicated operations, we have to bail
1254   // out when current case uses the predicate feature.
1255   if (!supports_predicate) {
1256     bool needs_predicate = false;
1257     if (is_store) {
1258       // Masked vector store always uses the predicated store.
1259       needs_predicate = true;
1260     } else {
1261       // Masked vector load with IOOBE always uses the predicated load.
1262       const TypeInt* offset_in_range = gvn().type(argument(9))->isa_int();
1263       if (!offset_in_range->is_con()) {
1264         if (C->print_intrinsics()) {
1265           tty->print_cr("  ** missing constant: offsetInRange=%s",
1266                         NodeClassNames[argument(8)->Opcode()]);
1267         }
1268         set_map(old_map);
1269         set_sp(old_sp);
1270         return false;
1271       }
1272       needs_predicate = (offset_in_range->get_con() == 0);
1273     }
1274 
1275     if (needs_predicate) {
1276       if (C->print_intrinsics()) {
1277         tty->print_cr("  ** not supported: op=%s vlen=%d etype=%s mismatched_ms=%d",
1278                       is_store ? "storeMasked" : "loadMasked",
1279                       num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1280       }
1281       set_map(old_map);
1282       set_sp(old_sp);
1283       return false;
1284     }
1285   }
1286 
1287   // This only happens for masked vector load. If predicate is not supported, then check whether
1288   // the normal vector load and blend operations are supported by backend.
1289   if (!supports_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1290       !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1291     if (C->print_intrinsics()) {
1292       tty->print_cr("  ** not supported: op=loadMasked vlen=%d etype=%s mismatched_ms=%d",
1293                     num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1294     }
1295     set_map(old_map);
1296     set_sp(old_sp);
1297     return false;
1298   }
1299 
1300   // Since we are using byte array, we need to double check that the vector reinterpret operation
1301   // with byte type is supported by backend.
1302   if (mismatched_ms) {
1303     if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1304       if (C->print_intrinsics()) {
1305         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s mismatched_ms=1",
1306                       is_store, is_store ? "storeMasked" : "loadMasked",
1307                       num_elem, type2name(elem_bt));
1308       }
1309       set_map(old_map);
1310       set_sp(old_sp);
1311       return false;
1312     }
1313   }
1314 
1315   // Since it needs to unbox the mask, we need to double check that the related load operations
1316   // for mask are supported by backend.
1317   if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1318     if (C->print_intrinsics()) {
1319       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s",
1320                       is_store, is_store ? "storeMasked" : "loadMasked",
1321                       num_elem, type2name(elem_bt));
1322     }
1323     set_map(old_map);
1324     set_sp(old_sp);
1325     return false;
1326   }
1327 
1328   // Can base be null? Otherwise, always on-heap access.
1329   bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1330   if (can_access_non_heap) {
1331     insert_mem_bar(Op_MemBarCPUOrder);
1332   }
1333 
1334   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1335   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1336   assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1337   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1338   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1339 
1340   Node* mask = unbox_vector(is_store ? argument(9) : argument(8), mbox_type, elem_bt, num_elem);
1341   if (mask == nullptr) {
1342     if (C->print_intrinsics()) {
1343       tty->print_cr("  ** unbox failed mask=%s",
1344                     is_store ? NodeClassNames[argument(9)->Opcode()]
1345                              : NodeClassNames[argument(8)->Opcode()]);
1346     }
1347     set_map(old_map);
1348     set_sp(old_sp);
1349     return false;
1350   }
1351 
1352   if (is_store) {
1353     Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
1354     if (val == nullptr) {
1355       if (C->print_intrinsics()) {
1356         tty->print_cr("  ** unbox failed vector=%s",
1357                       NodeClassNames[argument(8)->Opcode()]);
1358       }
1359       set_map(old_map);
1360       set_sp(old_sp);
1361       return false; // operand unboxing failed
1362     }
1363     set_all_memory(reset_memory());
1364 
1365     if (mismatched_ms) {
1366       // Reinterpret the incoming vector to byte vector.
1367       const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1368       val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1369       // Reinterpret the vector mask to byte type.
1370       const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1371       const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1372       mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1373     }
1374     Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1375     set_memory(vstore, addr_type);
1376   } else {
1377     Node* vload = nullptr;
1378 
1379     if (mismatched_ms) {
1380       // Reinterpret the vector mask to byte type.
1381       const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1382       const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1383       mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1384     }
1385 
1386     if (supports_predicate) {
1387       // Generate masked load vector node if predicate feature is supported.
1388       const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1389       vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1390     } else {
1391       // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1392       Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1393       zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, Type::get_const_basic_type(mem_elem_bt)));
1394       vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1395       vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1396     }
1397 
1398     if (mismatched_ms) {
1399       const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1400       vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1401     }
1402 
1403     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1404     set_result(box);
1405   }
1406 
1407   destruct_map_clone(old_map);
1408 
1409   if (can_access_non_heap) {
1410     insert_mem_bar(Op_MemBarCPUOrder);
1411   }
1412 
1413   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1414   return true;
1415 }
1416 
1417 // <C,
1418 //  V extends Vector<?>,
1419 //  W extends Vector<Integer>,
1420 //  S extends VectorSpecies<E>,
1421 //  M extends VectorMask<E>,
1422 //  E>
1423 // V loadWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int length,
1424 //               Class<? extends Vector<Integer>> vectorIndexClass,
1425 //               Object base, long offset, // Unsafe addressing
1426 //               W index_vector, M m,
1427 //               C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
1428 //               LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1429 //
1430 //  <C,
1431 //   V extends Vector<E>,
1432 //   W extends Vector<Integer>,
1433 //   M extends VectorMask<E>,
1434 //   E>
1435 //  void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1436 //                    int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset,    // Unsafe addressing
1437 //                    W index_vector, V v, M m,
1438 //                    C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1439 //                    StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1440 //
1441 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1442   const TypeInstPtr* vector_klass     = gvn().type(argument(0))->isa_instptr();
1443   const TypeInstPtr* mask_klass       = gvn().type(argument(1))->isa_instptr();
1444   const TypeInstPtr* elem_klass       = gvn().type(argument(2))->isa_instptr();
1445   const TypeInt*     vlen             = gvn().type(argument(3))->isa_int();
1446   const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1447 
1448   if (vector_klass == nullptr || elem_klass == nullptr || vector_idx_klass == nullptr || vlen == nullptr ||
1449       vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || vector_idx_klass->const_oop() == nullptr || !vlen->is_con()) {
1450     if (C->print_intrinsics()) {
1451       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1452                     NodeClassNames[argument(0)->Opcode()],
1453                     NodeClassNames[argument(2)->Opcode()],
1454                     NodeClassNames[argument(3)->Opcode()],
1455                     NodeClassNames[argument(4)->Opcode()]);
1456     }
1457     return false; // not enough info for intrinsification
1458   }
1459 
1460   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1461     if (C->print_intrinsics()) {
1462       tty->print_cr("  ** klass argument not initialized");
1463     }
1464     return false;
1465   }
1466 
1467   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1468   if (!elem_type->is_primitive_type()) {
1469     if (C->print_intrinsics()) {
1470       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1471     }
1472     return false; // should be primitive type
1473   }
1474 
1475   BasicType elem_bt = elem_type->basic_type();
1476   int num_elem = vlen->get_con();
1477 
1478   const Type* vmask_type = gvn().type(is_scatter ? argument(10) : argument(9));
1479   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1480   if (is_masked_op) {
1481     if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1482       if (C->print_intrinsics()) {
1483         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1484       }
1485       return false; // not enough info for intrinsification
1486     }
1487 
1488     if (!is_klass_initialized(mask_klass)) {
1489       if (C->print_intrinsics()) {
1490         tty->print_cr("  ** mask klass argument not initialized");
1491       }
1492       return false;
1493     }
1494 
1495     if (vmask_type->maybe_null()) {
1496       if (C->print_intrinsics()) {
1497         tty->print_cr("  ** null mask values are not allowed for masked op");
1498       }
1499       return false;
1500     }
1501 
1502     // Check whether the predicated gather/scatter node is supported by architecture.
1503     if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt,
1504                               (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred))) {
1505       if (C->print_intrinsics()) {
1506         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1507                       is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1508                       num_elem, type2name(elem_bt));
1509       }
1510       return false; // not supported
1511     }
1512   } else {
1513     // Check whether the normal gather/scatter node is supported for non-masked operation.
1514     if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1515       if (C->print_intrinsics()) {
1516         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1517                       is_scatter, is_scatter ? "scatter" : "gather",
1518                       num_elem, type2name(elem_bt));
1519       }
1520       return false; // not supported
1521     }
1522   }
1523 
1524   // Check that the vector holding indices is supported by architecture
1525   if (!arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
1526       if (C->print_intrinsics()) {
1527         tty->print_cr("  ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1528                       is_scatter, is_scatter ? "scatter" : "gather",
1529                       num_elem, is_masked_op ? 1 : 0);
1530       }
1531       return false; // not supported
1532   }
1533 
1534   Node* base = argument(5);
1535   Node* offset = ConvL2X(argument(6));
1536 
1537   // Save state and restore on bailout
1538   uint old_sp = sp();
1539   SafePointNode* old_map = clone_map();
1540 
1541   Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1542 
1543   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1544   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1545 
1546   // The array must be consistent with vector type
1547   if (arr_type == nullptr || (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, false))) {
1548     if (C->print_intrinsics()) {
1549       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1550                     is_scatter, is_scatter ? "scatter" : "gather",
1551                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1552     }
1553     set_map(old_map);
1554     set_sp(old_sp);
1555     return false;
1556   }
1557 
1558   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1559   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1560   ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1561   if (vbox_idx_klass == nullptr) {
1562     set_map(old_map);
1563     set_sp(old_sp);
1564     return false;
1565   }
1566 
1567   const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1568   Node* index_vect = unbox_vector(argument(8), vbox_idx_type, T_INT, num_elem);
1569   if (index_vect == nullptr) {
1570     set_map(old_map);
1571     set_sp(old_sp);
1572     return false;
1573   }
1574 
1575   Node* mask = nullptr;
1576   if (is_masked_op) {
1577     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1578     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1579     mask = unbox_vector(is_scatter ? argument(10) : argument(9), mbox_type, elem_bt, num_elem);
1580     if (mask == nullptr) {
1581       if (C->print_intrinsics()) {
1582         tty->print_cr("  ** unbox failed mask=%s",
1583                     is_scatter ? NodeClassNames[argument(10)->Opcode()]
1584                                : NodeClassNames[argument(9)->Opcode()]);
1585       }
1586       set_map(old_map);
1587       set_sp(old_sp);
1588       return false;
1589     }
1590   }
1591 
1592   const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1593   if (is_scatter) {
1594     Node* val = unbox_vector(argument(9), vbox_type, elem_bt, num_elem);
1595     if (val == nullptr) {
1596       set_map(old_map);
1597       set_sp(old_sp);
1598       return false; // operand unboxing failed
1599     }
1600     set_all_memory(reset_memory());
1601 
1602     Node* vstore = nullptr;
1603     if (mask != nullptr) {
1604       vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, index_vect, mask));
1605     } else {
1606       vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1607     }
1608     set_memory(vstore, addr_type);
1609   } else {
1610     Node* vload = nullptr;
1611     if (mask != nullptr) {
1612       vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_vect, mask));
1613     } else {
1614       vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1615     }
1616     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1617     set_result(box);
1618   }
1619 
1620   destruct_map_clone(old_map);
1621 
1622   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1623   return true;
1624 }
1625 
1626 // public static
1627 // <V extends Vector<E>,
1628 //  M extends VectorMask<E>,
1629 //  E>
1630 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1631 //                       Class<E> elementType, int length, V v, M m,
1632 //                       ReductionOperation<V, M> defaultImpl)
1633 bool LibraryCallKit::inline_vector_reduction() {
1634   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
1635   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1636   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
1637   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
1638   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
1639 
1640   if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1641       !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1642     if (C->print_intrinsics()) {
1643       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1644                     NodeClassNames[argument(0)->Opcode()],
1645                     NodeClassNames[argument(1)->Opcode()],
1646                     NodeClassNames[argument(3)->Opcode()],
1647                     NodeClassNames[argument(4)->Opcode()]);
1648     }
1649     return false; // not enough info for intrinsification
1650   }
1651   if (!is_klass_initialized(vector_klass)) {
1652     if (C->print_intrinsics()) {
1653       tty->print_cr("  ** klass argument not initialized");
1654     }
1655     return false;
1656   }
1657   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1658   if (!elem_type->is_primitive_type()) {
1659     if (C->print_intrinsics()) {
1660       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1661     }
1662     return false; // should be primitive type
1663   }
1664 
1665   const Type* vmask_type = gvn().type(argument(6));
1666   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1667   if (is_masked_op) {
1668     if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1669       if (C->print_intrinsics()) {
1670         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1671       }
1672       return false; // not enough info for intrinsification
1673     }
1674 
1675     if (!is_klass_initialized(mask_klass)) {
1676       if (C->print_intrinsics()) {
1677         tty->print_cr("  ** mask klass argument not initialized");
1678       }
1679       return false;
1680     }
1681 
1682     if (vmask_type->maybe_null()) {
1683       if (C->print_intrinsics()) {
1684         tty->print_cr("  ** null mask values are not allowed for masked op");
1685       }
1686       return false;
1687     }
1688   }
1689 
1690   BasicType elem_bt = elem_type->basic_type();
1691   int num_elem = vlen->get_con();
1692   int opc  = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1693   int sopc = ReductionNode::opcode(opc, elem_bt);
1694 
1695   // When using mask, mask use type needs to be VecMaskUseLoad.
1696   if (!arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1697     if (C->print_intrinsics()) {
1698       tty->print_cr("  ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1699                     sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1700     }
1701     return false;
1702   }
1703 
1704   // Return true if current platform has implemented the masked operation with predicate feature.
1705   bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1706   if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1707     if (C->print_intrinsics()) {
1708       tty->print_cr("  ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1709                     sopc, num_elem, type2name(elem_bt));
1710     }
1711     return false;
1712   }
1713 
1714   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1715   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1716 
1717   Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1718   if (opd == nullptr) {
1719     return false; // operand unboxing failed
1720   }
1721 
1722   Node* mask = nullptr;
1723   if (is_masked_op) {
1724     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1725     assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1726     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1727     mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1728     if (mask == nullptr) {
1729       if (C->print_intrinsics()) {
1730         tty->print_cr("  ** unbox failed mask=%s",
1731                       NodeClassNames[argument(6)->Opcode()]);
1732       }
1733       return false;
1734     }
1735   }
1736 
1737   Node* init = ReductionNode::make_identity_con_scalar(gvn(), opc, elem_bt);
1738   Node* value = nullptr;
1739   if (mask == nullptr) {
1740     assert(!is_masked_op, "Masked op needs the mask value never null");
1741     value = ReductionNode::make(opc, nullptr, init, opd, elem_bt);
1742   } else {
1743     if (use_predicate) {
1744       value = ReductionNode::make(opc, nullptr, init, opd, elem_bt);
1745       value->add_req(mask);
1746       value->add_flag(Node::Flag_is_predicated_vector);
1747     } else {
1748       Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, Type::get_const_basic_type(elem_bt)));
1749       value = gvn().transform(new VectorBlendNode(reduce_identity, opd, mask));
1750       value = ReductionNode::make(opc, nullptr, init, value, elem_bt);
1751     }
1752   }
1753   value = gvn().transform(value);
1754 
1755   Node* bits = nullptr;
1756   switch (elem_bt) {
1757     case T_BYTE:
1758     case T_SHORT:
1759     case T_INT: {
1760       bits = gvn().transform(new ConvI2LNode(value));
1761       break;
1762     }
1763     case T_FLOAT: {
1764       value = gvn().transform(new MoveF2INode(value));
1765       bits  = gvn().transform(new ConvI2LNode(value));
1766       break;
1767     }
1768     case T_DOUBLE: {
1769       bits = gvn().transform(new MoveD2LNode(value));
1770       break;
1771     }
1772     case T_LONG: {
1773       bits = value; // no conversion needed
1774       break;
1775     }
1776     default: fatal("%s", type2name(elem_bt));
1777   }
1778   set_result(bits);
1779   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1780   return true;
1781 }
1782 
1783 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1784 //                                V v1, V v2,
1785 //                                BiFunction<V, V, Boolean> defaultImpl)
1786 //
1787 bool LibraryCallKit::inline_vector_test() {
1788   const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1789   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1790   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1791   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1792 
1793   if (cond == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1794       !cond->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1795     if (C->print_intrinsics()) {
1796       tty->print_cr("  ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1797                     NodeClassNames[argument(0)->Opcode()],
1798                     NodeClassNames[argument(1)->Opcode()],
1799                     NodeClassNames[argument(2)->Opcode()],
1800                     NodeClassNames[argument(3)->Opcode()]);
1801     }
1802     return false; // not enough info for intrinsification
1803   }
1804   if (!is_klass_initialized(vector_klass)) {
1805     if (C->print_intrinsics()) {
1806       tty->print_cr("  ** klass argument not initialized");
1807     }
1808     return false;
1809   }
1810   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1811   if (!elem_type->is_primitive_type()) {
1812     if (C->print_intrinsics()) {
1813       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1814     }
1815     return false; // should be primitive type
1816   }
1817   BasicType elem_bt = elem_type->basic_type();
1818   int num_elem = vlen->get_con();
1819   BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1820   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1821   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1822 
1823   if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1824     if (C->print_intrinsics()) {
1825       tty->print_cr("  ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1826                     cond->get_con(), num_elem, type2name(elem_bt),
1827                     is_vector_mask(vbox_klass));
1828     }
1829     return false;
1830   }
1831 
1832   Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1833   Node* opd2;
1834   if (Matcher::vectortest_needs_second_argument(booltest == BoolTest::overflow,
1835                                                 opd1->bottom_type()->isa_vectmask())) {
1836     opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1837   } else {
1838     opd2 = opd1;
1839   }
1840   if (opd1 == nullptr || opd2 == nullptr) {
1841     return false; // operand unboxing failed
1842   }
1843 
1844   Node* cmp = gvn().transform(new VectorTestNode(opd1, opd2, booltest));
1845   BoolTest::mask test = Matcher::vectortest_mask(booltest == BoolTest::overflow,
1846                                                  opd1->bottom_type()->isa_vectmask(), num_elem);
1847   Node* bol = gvn().transform(new BoolNode(cmp, test));
1848   Node* res = gvn().transform(new CMoveINode(bol, gvn().intcon(0), gvn().intcon(1), TypeInt::BOOL));
1849 
1850   set_result(res);
1851   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1852   return true;
1853 }
1854 
1855 // public static
1856 // <V extends Vector<E>,
1857 //  M extends VectorMask<E>,
1858 //  E>
1859 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1860 //         V v1, V v2, M m,
1861 //         VectorBlendOp<V, M, E> defaultImpl)
1862 bool LibraryCallKit::inline_vector_blend() {
1863   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1864   const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1865   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1866   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1867 
1868   if (mask_klass == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1869     return false; // dead code
1870   }
1871   if (mask_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr ||
1872       elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1873     if (C->print_intrinsics()) {
1874       tty->print_cr("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1875                     NodeClassNames[argument(0)->Opcode()],
1876                     NodeClassNames[argument(1)->Opcode()],
1877                     NodeClassNames[argument(2)->Opcode()],
1878                     NodeClassNames[argument(3)->Opcode()]);
1879     }
1880     return false; // not enough info for intrinsification
1881   }
1882   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1883     if (C->print_intrinsics()) {
1884       tty->print_cr("  ** klass argument not initialized");
1885     }
1886     return false;
1887   }
1888   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1889   if (!elem_type->is_primitive_type()) {
1890     if (C->print_intrinsics()) {
1891       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1892     }
1893     return false; // should be primitive type
1894   }
1895   BasicType elem_bt = elem_type->basic_type();
1896   BasicType mask_bt = elem_bt;
1897   int num_elem = vlen->get_con();
1898 
1899   if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1900     if (C->print_intrinsics()) {
1901       tty->print_cr("  ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1902                     num_elem, type2name(elem_bt));
1903     }
1904     return false; // not supported
1905   }
1906   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1907   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1908 
1909   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1910   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1911 
1912   Node* v1   = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1913   Node* v2   = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1914   Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1915 
1916   if (v1 == nullptr || v2 == nullptr || mask == nullptr) {
1917     return false; // operand unboxing failed
1918   }
1919 
1920   Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1921 
1922   Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1923   set_result(box);
1924   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1925   return true;
1926 }
1927 
1928 //  public static
1929 //  <V extends Vector<E>,
1930 //   M extends VectorMask<E>,
1931 //   E>
1932 //  M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1933 //            V v1, V v2, M m,
1934 //            VectorCompareOp<V,M> defaultImpl)
1935 bool LibraryCallKit::inline_vector_compare() {
1936   const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1937   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1938   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
1939   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
1940   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
1941 
1942   if (cond == nullptr || vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1943     return false; // dead code
1944   }
1945   if (!cond->is_con() || vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
1946       elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1947     if (C->print_intrinsics()) {
1948       tty->print_cr("  ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1949                     NodeClassNames[argument(0)->Opcode()],
1950                     NodeClassNames[argument(1)->Opcode()],
1951                     NodeClassNames[argument(2)->Opcode()],
1952                     NodeClassNames[argument(3)->Opcode()],
1953                     NodeClassNames[argument(4)->Opcode()]);
1954     }
1955     return false; // not enough info for intrinsification
1956   }
1957   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1958     if (C->print_intrinsics()) {
1959       tty->print_cr("  ** klass argument not initialized");
1960     }
1961     return false;
1962   }
1963   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1964   if (!elem_type->is_primitive_type()) {
1965     if (C->print_intrinsics()) {
1966       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1967     }
1968     return false; // should be primitive type
1969   }
1970 
1971   int num_elem = vlen->get_con();
1972   BasicType elem_bt = elem_type->basic_type();
1973   BasicType mask_bt = elem_bt;
1974 
1975   if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1976     if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1977       if (C->print_intrinsics()) {
1978         tty->print_cr("  ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1979                       cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1980       }
1981       return false;
1982     }
1983   }
1984 
1985   if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1986     if (C->print_intrinsics()) {
1987       tty->print_cr("  ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1988                     cond->get_con(), num_elem, type2name(elem_bt));
1989     }
1990     return false;
1991   }
1992 
1993   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1994   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1995 
1996   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1997   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1998 
1999   Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2000   Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
2001 
2002   bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
2003   Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : nullptr;
2004   if (is_masked_op && mask == nullptr) {
2005     if (C->print_intrinsics()) {
2006       tty->print_cr("  ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
2007                     cond->get_con(), num_elem, type2name(elem_bt));
2008     }
2009     return false;
2010   }
2011 
2012   bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
2013   if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
2014     if (C->print_intrinsics()) {
2015       tty->print_cr("  ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
2016                     cond->get_con(), num_elem, type2name(elem_bt));
2017     }
2018     return false;
2019   }
2020 
2021   if (v1 == nullptr || v2 == nullptr) {
2022     return false; // operand unboxing failed
2023   }
2024   BoolTest::mask pred = (BoolTest::mask)cond->get_con();
2025   ConINode* pred_node = (ConINode*)gvn().makecon(cond);
2026 
2027   const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
2028   Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
2029 
2030   if (is_masked_op) {
2031     if (use_predicate) {
2032       operation->add_req(mask);
2033       operation->add_flag(Node::Flag_is_predicated_vector);
2034     } else {
2035       operation = gvn().transform(operation);
2036       operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
2037     }
2038   }
2039 
2040   operation = gvn().transform(operation);
2041 
2042   Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
2043   set_result(box);
2044   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2045   return true;
2046 }
2047 
2048 // public static
2049 // <V extends Vector<E>,
2050 //  Sh extends VectorShuffle<E>,
2051 //  M extends VectorMask<E>,
2052 //  E>
2053 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
2054 //               V v1, Sh sh, M m,
2055 //               VectorRearrangeOp<V, Sh, M, E> defaultImpl)
2056 bool LibraryCallKit::inline_vector_rearrange() {
2057   const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
2058   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
2059   const TypeInstPtr* mask_klass    = gvn().type(argument(2))->isa_instptr();
2060   const TypeInstPtr* elem_klass    = gvn().type(argument(3))->isa_instptr();
2061   const TypeInt*     vlen          = gvn().type(argument(4))->isa_int();
2062 
2063   if (vector_klass == nullptr  || shuffle_klass == nullptr ||  elem_klass == nullptr || vlen == nullptr) {
2064     return false; // dead code
2065   }
2066   if (shuffle_klass->const_oop() == nullptr ||
2067       vector_klass->const_oop()  == nullptr ||
2068       elem_klass->const_oop()    == nullptr ||
2069       !vlen->is_con()) {
2070     if (C->print_intrinsics()) {
2071       tty->print_cr("  ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
2072                     NodeClassNames[argument(0)->Opcode()],
2073                     NodeClassNames[argument(1)->Opcode()],
2074                     NodeClassNames[argument(3)->Opcode()],
2075                     NodeClassNames[argument(4)->Opcode()]);
2076     }
2077     return false; // not enough info for intrinsification
2078   }
2079   if (!is_klass_initialized(vector_klass)  ||
2080       !is_klass_initialized(shuffle_klass)) {
2081     if (C->print_intrinsics()) {
2082       tty->print_cr("  ** klass argument not initialized");
2083     }
2084     return false;
2085   }
2086   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2087   if (!elem_type->is_primitive_type()) {
2088     if (C->print_intrinsics()) {
2089       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2090     }
2091     return false; // should be primitive type
2092   }
2093   BasicType elem_bt = elem_type->basic_type();
2094   BasicType shuffle_bt = elem_bt;
2095   int num_elem = vlen->get_con();
2096 
2097   if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
2098     if (C->print_intrinsics()) {
2099       tty->print_cr("  ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
2100                     num_elem, type2name(elem_bt));
2101     }
2102     return false; // not supported
2103   }
2104 
2105   bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
2106   bool use_predicate = is_masked_op;
2107   if (is_masked_op &&
2108       (mask_klass == nullptr ||
2109        mask_klass->const_oop() == nullptr ||
2110        !is_klass_initialized(mask_klass))) {
2111     if (C->print_intrinsics()) {
2112       tty->print_cr("  ** mask_klass argument not initialized");
2113     }
2114   }
2115   VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2116   if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
2117     use_predicate = false;
2118     if(!is_masked_op ||
2119        (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2120         !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)     ||
2121         !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
2122       if (C->print_intrinsics()) {
2123         tty->print_cr("  ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
2124                       num_elem, type2name(elem_bt));
2125       }
2126       return false; // not supported
2127     }
2128   }
2129   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2130   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2131 
2132   ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
2133   const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
2134 
2135   Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2136   Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
2137 
2138   if (v1 == nullptr || shuffle == nullptr) {
2139     return false; // operand unboxing failed
2140   }
2141 
2142   Node* mask = nullptr;
2143   if (is_masked_op) {
2144     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2145     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2146     mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2147     if (mask == nullptr) {
2148       if (C->print_intrinsics()) {
2149         tty->print_cr("  ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
2150                       num_elem, type2name(elem_bt));
2151       }
2152       return false;
2153     }
2154   }
2155 
2156   Node* rearrange = new VectorRearrangeNode(v1, shuffle);
2157   if (is_masked_op) {
2158     if (use_predicate) {
2159       rearrange->add_req(mask);
2160       rearrange->add_flag(Node::Flag_is_predicated_vector);
2161     } else {
2162       const TypeVect* vt = v1->bottom_type()->is_vect();
2163       rearrange = gvn().transform(rearrange);
2164       Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2165       Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, Type::get_const_basic_type(elem_bt)));
2166       rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2167     }
2168   }
2169   rearrange = gvn().transform(rearrange);
2170 
2171   Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2172   set_result(box);
2173   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2174   return true;
2175 }
2176 
2177 static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
2178   address addr = nullptr;
2179   assert(UseVectorStubs, "sanity");
2180   assert(name_ptr != nullptr, "unexpected");
2181   assert((vop >= VectorSupport::VECTOR_OP_SVML_START) && (vop <= VectorSupport::VECTOR_OP_SVML_END), "unexpected");
2182   int op = vop - VectorSupport::VECTOR_OP_SVML_START;
2183 
2184   switch(bits) {
2185     case 64:  //fallthough
2186     case 128: //fallthough
2187     case 256: //fallthough
2188     case 512:
2189       if (bt == T_FLOAT) {
2190         snprintf(name_ptr, name_len, "vector_%s_float%d", VectorSupport::svmlname[op], bits);
2191         addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op];
2192       } else {
2193         assert(bt == T_DOUBLE, "must be FP type only");
2194         snprintf(name_ptr, name_len, "vector_%s_double%d", VectorSupport::svmlname[op], bits);
2195         addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op];
2196       }
2197       break;
2198     default:
2199       snprintf(name_ptr, name_len, "invalid");
2200       addr = nullptr;
2201       Unimplemented();
2202       break;
2203   }
2204 
2205   return addr;
2206 }
2207 
2208 Node* LibraryCallKit::gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
2209   assert(UseVectorStubs, "sanity");
2210   assert(vector_api_op_id >= VectorSupport::VECTOR_OP_SVML_START && vector_api_op_id <= VectorSupport::VECTOR_OP_SVML_END, "need valid op id");
2211   assert(opd1 != nullptr, "must not be null");
2212   const TypeVect* vt = TypeVect::make(bt, num_elem);
2213   const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != nullptr ? 2 : 1, vt, vt);
2214   char name[100] = "";
2215 
2216   // Get address for svml method.
2217   address addr = get_svml_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
2218 
2219   if (addr == nullptr) {
2220     return nullptr;
2221   }
2222 
2223   assert(name[0] != '\0', "name must not be null");
2224   Node* operation = make_runtime_call(RC_VECTOR,
2225                                       call_type,
2226                                       addr,
2227                                       name,
2228                                       TypePtr::BOTTOM,
2229                                       opd1,
2230                                       opd2);
2231   return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
2232 }
2233 
2234 //  public static
2235 //  <V extends Vector<E>,
2236 //   M extends VectorMask<E>,
2237 //   E>
2238 //  V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2239 //                 Class<E> elementType, int length,
2240 //                 V v, int n, M m,
2241 //                 VectorBroadcastIntOp<V, M> defaultImpl)
2242 bool LibraryCallKit::inline_vector_broadcast_int() {
2243   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
2244   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2245   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
2246   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
2247   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
2248 
2249   if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
2250     return false; // dead code
2251   }
2252   if (!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2253     if (C->print_intrinsics()) {
2254       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2255                     NodeClassNames[argument(0)->Opcode()],
2256                     NodeClassNames[argument(1)->Opcode()],
2257                     NodeClassNames[argument(3)->Opcode()],
2258                     NodeClassNames[argument(4)->Opcode()]);
2259     }
2260     return false; // not enough info for intrinsification
2261   }
2262   if (!is_klass_initialized(vector_klass)) {
2263     if (C->print_intrinsics()) {
2264       tty->print_cr("  ** klass argument not initialized");
2265     }
2266     return false;
2267   }
2268 
2269   const Type* vmask_type = gvn().type(argument(7));
2270   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2271   if (is_masked_op) {
2272     if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
2273       if (C->print_intrinsics()) {
2274         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2275       }
2276       return false; // not enough info for intrinsification
2277     }
2278 
2279     if (!is_klass_initialized(mask_klass)) {
2280       if (C->print_intrinsics()) {
2281         tty->print_cr("  ** mask klass argument not initialized");
2282       }
2283       return false;
2284     }
2285 
2286     if (vmask_type->maybe_null()) {
2287       if (C->print_intrinsics()) {
2288         tty->print_cr("  ** null mask values are not allowed for masked op");
2289       }
2290       return false;
2291     }
2292   }
2293 
2294   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2295   if (!elem_type->is_primitive_type()) {
2296     if (C->print_intrinsics()) {
2297       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2298     }
2299     return false; // should be primitive type
2300   }
2301 
2302   int num_elem = vlen->get_con();
2303   BasicType elem_bt = elem_type->basic_type();
2304   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2305 
2306   bool is_shift  = VectorNode::is_shift_opcode(opc);
2307   bool is_rotate = VectorNode::is_rotate_opcode(opc);
2308 
2309   if (opc == 0 || (!is_shift && !is_rotate)) {
2310     if (C->print_intrinsics()) {
2311       tty->print_cr("  ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2312     }
2313     return false; // operation not supported
2314   }
2315 
2316   int sopc = VectorNode::opcode(opc, elem_bt);
2317   if (sopc == 0) {
2318     if (C->print_intrinsics()) {
2319       tty->print_cr("  ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2320     }
2321     return false; // operation not supported
2322   }
2323 
2324   Node* cnt  = argument(6);
2325   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2326   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2327   const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2328 
2329   // If CPU supports vector constant rotate instructions pass it directly
2330   bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2331                          Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2332   bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2333 
2334   VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2335   bool use_predicate = is_masked_op;
2336 
2337   if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2338     use_predicate = false;
2339     if (!is_masked_op ||
2340         (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2341          !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2342 
2343       if (C->print_intrinsics()) {
2344         tty->print_cr("  ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2345                       sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2346       }
2347       return false; // not supported
2348     }
2349   }
2350 
2351   Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2352   Node* opd2 = nullptr;
2353   if (is_shift) {
2354     opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2355   } else {
2356     assert(is_rotate, "unexpected operation");
2357     if (!is_const_rotate) {
2358       const Type * type_bt = Type::get_const_basic_type(elem_bt);
2359       cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2360       opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, type_bt));
2361     } else {
2362       // Constant shift value.
2363       opd2 = cnt;
2364     }
2365   }
2366 
2367   if (opd1 == nullptr || opd2 == nullptr) {
2368     return false;
2369   }
2370 
2371   Node* mask = nullptr;
2372   if (is_masked_op) {
2373     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2374     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2375     mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2376     if (mask == nullptr) {
2377       if (C->print_intrinsics()) {
2378         tty->print_cr("  ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2379       }
2380       return false;
2381     }
2382   }
2383 
2384   Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2385   if (is_masked_op && mask != nullptr) {
2386     if (use_predicate) {
2387       operation->add_req(mask);
2388       operation->add_flag(Node::Flag_is_predicated_vector);
2389     } else {
2390       operation = gvn().transform(operation);
2391       operation = new VectorBlendNode(opd1, operation, mask);
2392     }
2393   }
2394   operation = gvn().transform(operation);
2395   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2396   set_result(vbox);
2397   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2398   return true;
2399 }
2400 
2401 // public static <VOUT extends VectorPayload,
2402 //                 VIN extends VectorPayload,
2403 //                   S extends VectorSpecies>
2404 // VOUT convert(int oprId,
2405 //           Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2406 //           Class<?>   toVectorClass, Class<?>   toElementType, int   toVLen,
2407 //           VIN v, S s,
2408 //           VectorConvertOp<VOUT, VIN, S> defaultImpl)
2409 //
2410 bool LibraryCallKit::inline_vector_convert() {
2411   const TypeInt*     opr               = gvn().type(argument(0))->isa_int();
2412 
2413   const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2414   const TypeInstPtr* elem_klass_from   = gvn().type(argument(2))->isa_instptr();
2415   const TypeInt*     vlen_from         = gvn().type(argument(3))->isa_int();
2416 
2417   const TypeInstPtr* vector_klass_to   = gvn().type(argument(4))->isa_instptr();
2418   const TypeInstPtr* elem_klass_to     = gvn().type(argument(5))->isa_instptr();
2419   const TypeInt*     vlen_to           = gvn().type(argument(6))->isa_int();
2420 
2421   if (opr == nullptr ||
2422       vector_klass_from == nullptr || elem_klass_from == nullptr || vlen_from == nullptr ||
2423       vector_klass_to   == nullptr || elem_klass_to   == nullptr || vlen_to   == nullptr) {
2424     return false; // dead code
2425   }
2426   if (!opr->is_con() ||
2427       vector_klass_from->const_oop() == nullptr || elem_klass_from->const_oop() == nullptr || !vlen_from->is_con() ||
2428       vector_klass_to->const_oop() == nullptr || elem_klass_to->const_oop() == nullptr || !vlen_to->is_con()) {
2429     if (C->print_intrinsics()) {
2430       tty->print_cr("  ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2431                     NodeClassNames[argument(0)->Opcode()],
2432                     NodeClassNames[argument(1)->Opcode()],
2433                     NodeClassNames[argument(2)->Opcode()],
2434                     NodeClassNames[argument(3)->Opcode()],
2435                     NodeClassNames[argument(4)->Opcode()],
2436                     NodeClassNames[argument(5)->Opcode()],
2437                     NodeClassNames[argument(6)->Opcode()]);
2438     }
2439     return false; // not enough info for intrinsification
2440   }
2441   if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2442     if (C->print_intrinsics()) {
2443       tty->print_cr("  ** klass argument not initialized");
2444     }
2445     return false;
2446   }
2447 
2448   assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST  ||
2449          opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
2450          opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2451   bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2452   bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2453 
2454   ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2455   ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2456   if (is_vector_shuffle(vbox_klass_from)) {
2457     return false; // vector shuffles aren't supported
2458   }
2459   bool is_mask = is_vector_mask(vbox_klass_from);
2460 
2461   ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2462   if (!elem_type_from->is_primitive_type()) {
2463     return false; // should be primitive type
2464   }
2465   BasicType elem_bt_from = elem_type_from->basic_type();
2466   ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2467   if (!elem_type_to->is_primitive_type()) {
2468     return false; // should be primitive type
2469   }
2470   BasicType elem_bt_to = elem_type_to->basic_type();
2471 
2472   int num_elem_from = vlen_from->get_con();
2473   int num_elem_to = vlen_to->get_con();
2474 
2475   // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2476   // since we may need to change size.
2477   if (!arch_supports_vector(Op_VectorReinterpret,
2478                             num_elem_from,
2479                             elem_bt_from,
2480                             is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2481     if (C->print_intrinsics()) {
2482       tty->print_cr("  ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2483                     is_cast ? "cast" : "reinterpret",
2484                     num_elem_from, type2name(elem_bt_from), is_mask);
2485     }
2486     return false;
2487   }
2488 
2489   // Check whether we can support resizing/reinterpreting to the new size.
2490   if (!arch_supports_vector(Op_VectorReinterpret,
2491                             num_elem_to,
2492                             elem_bt_to,
2493                             is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2494     if (C->print_intrinsics()) {
2495       tty->print_cr("  ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2496                     is_cast ? "cast" : "reinterpret",
2497                     num_elem_to, type2name(elem_bt_to), is_mask);
2498     }
2499     return false;
2500   }
2501 
2502   // At this point, we know that both input and output vector registers are supported
2503   // by the architecture. Next check if the casted type is simply to same type - which means
2504   // that it is actually a resize and not a cast.
2505   if (is_cast && elem_bt_from == elem_bt_to) {
2506     is_cast = false;
2507   }
2508 
2509   const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2510 
2511   Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2512   if (opd1 == nullptr) {
2513     return false;
2514   }
2515 
2516   const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2517   const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2518 
2519   // Safety check to prevent casting if source mask is of type vector
2520   // and destination mask of type predicate vector and vice-versa.
2521   // From X86 standpoint, this case will only arise over KNL target,
2522   // where certain masks (depending on the species) are either propagated
2523   // through a vector or predicate register.
2524   if (is_mask &&
2525       ((src_type->isa_vectmask() == nullptr && dst_type->isa_vectmask()) ||
2526        (dst_type->isa_vectmask() == nullptr && src_type->isa_vectmask()))) {
2527     return false;
2528   }
2529 
2530   Node* op = opd1;
2531   if (is_cast) {
2532     assert(!is_mask || num_elem_from == num_elem_to, "vector mask cast needs the same elem num");
2533     int cast_vopc = VectorCastNode::opcode(-1, elem_bt_from, !is_ucast);
2534 
2535     // Make sure that vector cast is implemented to particular type/size combination if it is
2536     // not a mask casting.
2537     if (!is_mask && !arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2538       if (C->print_intrinsics()) {
2539         tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2540                       cast_vopc, num_elem_to, type2name(elem_bt_to), is_mask);
2541       }
2542       return false;
2543     }
2544 
2545     if (num_elem_from < num_elem_to) {
2546       // Since input and output number of elements are not consistent, we need to make sure we
2547       // properly size. Thus, first make a cast that retains the number of elements from source.
2548       int num_elem_for_cast = num_elem_from;
2549 
2550       // It is possible that arch does not support this intermediate vector size
2551       // TODO More complex logic required here to handle this corner case for the sizes.
2552       if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2553         if (C->print_intrinsics()) {
2554           tty->print_cr("  ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2555                         cast_vopc,
2556                         num_elem_for_cast, type2name(elem_bt_to), is_mask);
2557         }
2558         return false;
2559       }
2560 
2561       op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2562       // Now ensure that the destination gets properly resized to needed size.
2563       op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2564     } else if (num_elem_from > num_elem_to) {
2565       // Since number of elements from input is larger than output, simply reduce size of input
2566       // (we are supposed to drop top elements anyway).
2567       int num_elem_for_resize = num_elem_to;
2568 
2569       // It is possible that arch does not support this intermediate vector size
2570       // TODO More complex logic required here to handle this corner case for the sizes.
2571       if (!arch_supports_vector(Op_VectorReinterpret,
2572                                 num_elem_for_resize,
2573                                 elem_bt_from,
2574                                 VecMaskNotUsed)) {
2575         if (C->print_intrinsics()) {
2576           tty->print_cr("  ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2577                         num_elem_for_resize, type2name(elem_bt_from), is_mask);
2578         }
2579         return false;
2580       }
2581 
2582       const TypeVect* resize_type = TypeVect::make(elem_bt_from, num_elem_for_resize);
2583       op = gvn().transform(new VectorReinterpretNode(op, src_type, resize_type));
2584       op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2585     } else { // num_elem_from == num_elem_to
2586       if (is_mask) {
2587         // Make sure that cast for vector mask is implemented to particular type/size combination.
2588         if (!arch_supports_vector(Op_VectorMaskCast, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2589           if (C->print_intrinsics()) {
2590             tty->print_cr("  ** not supported: arity=1 op=maskcast vlen2=%d etype2=%s ismask=%d",
2591                           num_elem_to, type2name(elem_bt_to), is_mask);
2592           }
2593           return false;
2594         }
2595         op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2596       } else {
2597         // Since input and output number of elements match, and since we know this vector size is
2598         // supported, simply do a cast with no resize needed.
2599         op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2600       }
2601     }
2602   } else if (Type::cmp(src_type, dst_type) != 0) {
2603     assert(!is_cast, "must be reinterpret");
2604     op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2605   }
2606 
2607   const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2608   Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2609   set_result(vbox);
2610   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2611   return true;
2612 }
2613 
2614 //  public static
2615 //  <V extends Vector<E>,
2616 //   E>
2617 //  V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2618 //           V vec, int ix, long val,
2619 //           VecInsertOp<V> defaultImpl)
2620 bool LibraryCallKit::inline_vector_insert() {
2621   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2622   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2623   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2624   const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
2625 
2626   if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2627     return false; // dead code
2628   }
2629   if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2630     if (C->print_intrinsics()) {
2631       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2632                     NodeClassNames[argument(0)->Opcode()],
2633                     NodeClassNames[argument(1)->Opcode()],
2634                     NodeClassNames[argument(2)->Opcode()],
2635                     NodeClassNames[argument(4)->Opcode()]);
2636     }
2637     return false; // not enough info for intrinsification
2638   }
2639   if (!is_klass_initialized(vector_klass)) {
2640     if (C->print_intrinsics()) {
2641       tty->print_cr("  ** klass argument not initialized");
2642     }
2643     return false;
2644   }
2645   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2646   if (!elem_type->is_primitive_type()) {
2647     if (C->print_intrinsics()) {
2648       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2649     }
2650     return false; // should be primitive type
2651   }
2652   BasicType elem_bt = elem_type->basic_type();
2653   int num_elem = vlen->get_con();
2654   if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2655     if (C->print_intrinsics()) {
2656       tty->print_cr("  ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2657                     num_elem, type2name(elem_bt));
2658     }
2659     return false; // not supported
2660   }
2661 
2662   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2663   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2664 
2665   Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2666   if (opd == nullptr) {
2667     return false;
2668   }
2669 
2670   Node* insert_val = argument(5);
2671   assert(gvn().type(insert_val)->isa_long() != nullptr, "expected to be long");
2672 
2673   // Convert insert value back to its appropriate type.
2674   switch (elem_bt) {
2675     case T_BYTE:
2676       insert_val = gvn().transform(new ConvL2INode(insert_val));
2677       insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::BYTE));
2678       break;
2679     case T_SHORT:
2680       insert_val = gvn().transform(new ConvL2INode(insert_val));
2681       insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::SHORT));
2682       break;
2683     case T_INT:
2684       insert_val = gvn().transform(new ConvL2INode(insert_val));
2685       break;
2686     case T_FLOAT:
2687       insert_val = gvn().transform(new ConvL2INode(insert_val));
2688       insert_val = gvn().transform(new MoveI2FNode(insert_val));
2689       break;
2690     case T_DOUBLE:
2691       insert_val = gvn().transform(new MoveL2DNode(insert_val));
2692       break;
2693     case T_LONG:
2694       // no conversion needed
2695       break;
2696     default: fatal("%s", type2name(elem_bt)); break;
2697   }
2698 
2699   Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con()));
2700 
2701   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2702   set_result(vbox);
2703   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2704   return true;
2705 }
2706 
2707 //  public static
2708 //  <VM extends VectorPayload,
2709 //   E>
2710 //  long extract(Class<? extends VM> vClass, Class<E> eClass,
2711 //               int length,
2712 //               VM vm, int i,
2713 //               VecExtractOp<VM> defaultImpl)
2714 bool LibraryCallKit::inline_vector_extract() {
2715   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2716   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2717   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2718   const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
2719 
2720   if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2721     return false; // dead code
2722   }
2723   if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2724     if (C->print_intrinsics()) {
2725       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
2726                     NodeClassNames[argument(0)->Opcode()],
2727                     NodeClassNames[argument(1)->Opcode()],
2728                     NodeClassNames[argument(2)->Opcode()]);
2729     }
2730     return false; // not enough info for intrinsification
2731   }
2732   if (!is_klass_initialized(vector_klass)) {
2733     if (C->print_intrinsics()) {
2734       tty->print_cr("  ** klass argument not initialized");
2735     }
2736     return false;
2737   }
2738   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2739   if (!elem_type->is_primitive_type()) {
2740     if (C->print_intrinsics()) {
2741       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2742     }
2743     return false; // should be primitive type
2744   }
2745   BasicType elem_bt = elem_type->basic_type();
2746   int num_elem = vlen->get_con();
2747 
2748   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2749   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2750 
2751   Node* opd = nullptr;
2752 
2753   if (is_vector_mask(vbox_klass)) {
2754     // vbox_klass is mask. This is used for VectorMask.laneIsSet(int).
2755 
2756     Node* pos = argument(4); // can be variable
2757     if (arch_supports_vector(Op_ExtractUB, num_elem, elem_bt, VecMaskUseAll)) {
2758       // Transform mask to vector with type of boolean and utilize ExtractUB node.
2759       opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2760       if (opd == nullptr) {
2761         return false;
2762       }
2763       opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2764       opd = gvn().transform(new ExtractUBNode(opd, pos));
2765       opd = gvn().transform(new ConvI2LNode(opd));
2766     } else if (arch_supports_vector(Op_VectorMaskToLong, num_elem, elem_bt, VecMaskUseLoad)) {
2767       opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2768       if (opd == nullptr) {
2769         return false;
2770       }
2771       // VectorMaskToLongNode requires the input is either a mask or a vector with BOOLEAN type.
2772       if (opd->bottom_type()->isa_vectmask() == nullptr) {
2773         opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2774       }
2775       // ((toLong() >>> pos) & 1L
2776       opd = gvn().transform(new VectorMaskToLongNode(opd, TypeLong::LONG));
2777       opd = gvn().transform(new URShiftLNode(opd, pos));
2778       opd = gvn().transform(new AndLNode(opd, gvn().makecon(TypeLong::ONE)));
2779     } else {
2780       if (C->print_intrinsics()) {
2781         tty->print_cr("  ** Rejected mask extraction because architecture does not support it");
2782       }
2783       return false; // not supported
2784     }
2785   } else {
2786     // vbox_klass is vector. This is used for Vector.lane(int).
2787     if (!idx->is_con()) {
2788       if (C->print_intrinsics()) {
2789         tty->print_cr("  ** missing constant: idx=%s", NodeClassNames[argument(4)->Opcode()]);
2790       }
2791       return false; // not enough info for intrinsification
2792     }
2793 
2794     int vopc = ExtractNode::opcode(elem_bt);
2795     if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2796       if (C->print_intrinsics()) {
2797         tty->print_cr("  ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2798                       num_elem, type2name(elem_bt));
2799       }
2800       return false; // not supported
2801     }
2802 
2803     opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2804     if (opd == nullptr) {
2805       return false;
2806     }
2807     ConINode* idx_con = gvn().intcon(idx->get_con())->as_ConI();
2808 
2809     opd = gvn().transform(ExtractNode::make(opd, idx_con, elem_bt));
2810     switch (elem_bt) {
2811       case T_BYTE:
2812       case T_SHORT:
2813       case T_INT: {
2814         opd = gvn().transform(new ConvI2LNode(opd));
2815         break;
2816       }
2817       case T_FLOAT: {
2818         opd = gvn().transform(new MoveF2INode(opd));
2819         opd = gvn().transform(new ConvI2LNode(opd));
2820         break;
2821       }
2822       case T_DOUBLE: {
2823         opd = gvn().transform(new MoveD2LNode(opd));
2824         break;
2825       }
2826       case T_LONG: {
2827         // no conversion needed
2828         break;
2829       }
2830       default: fatal("%s", type2name(elem_bt));
2831     }
2832   }
2833   set_result(opd);
2834   return true;
2835 }
2836 
2837 // public static
2838 // <V extends Vector<E>,
2839 //  M extends VectorMask<E>,
2840 //  E>
2841 //  V compressExpandOp(int opr,
2842 //                    Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2843 //                    int length, V v, M m,
2844 //                    CompressExpandOperation<V, M> defaultImpl)
2845 bool LibraryCallKit::inline_vector_compress_expand() {
2846   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
2847   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2848   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
2849   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
2850   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
2851 
2852   if (vector_klass == nullptr || elem_klass == nullptr || mask_klass == nullptr || vlen == nullptr ||
2853       vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
2854       elem_klass->const_oop() == nullptr || !vlen->is_con() || !opr->is_con()) {
2855     if (C->print_intrinsics()) {
2856       tty->print_cr("  ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2857                     NodeClassNames[argument(0)->Opcode()],
2858                     NodeClassNames[argument(1)->Opcode()],
2859                     NodeClassNames[argument(2)->Opcode()],
2860                     NodeClassNames[argument(3)->Opcode()],
2861                     NodeClassNames[argument(4)->Opcode()]);
2862     }
2863     return false; // not enough info for intrinsification
2864   }
2865 
2866   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2867     if (C->print_intrinsics()) {
2868       tty->print_cr("  ** klass argument not initialized");
2869     }
2870     return false;
2871   }
2872 
2873   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2874   if (!elem_type->is_primitive_type()) {
2875     if (C->print_intrinsics()) {
2876       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2877     }
2878     return false; // should be primitive type
2879   }
2880 
2881   int num_elem = vlen->get_con();
2882   BasicType elem_bt = elem_type->basic_type();
2883   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2884 
2885   if (!arch_supports_vector(opc, num_elem, elem_bt, VecMaskUseLoad)) {
2886     if (C->print_intrinsics()) {
2887       tty->print_cr("  ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2888                     opc, num_elem, type2name(elem_bt));
2889     }
2890     return false; // not supported
2891   }
2892 
2893   Node* opd1 = nullptr;
2894   const TypeInstPtr* vbox_type = nullptr;
2895   if (opc != Op_CompressM) {
2896     ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2897     vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2898     opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2899     if (opd1 == nullptr) {
2900       if (C->print_intrinsics()) {
2901         tty->print_cr("  ** unbox failed vector=%s",
2902                       NodeClassNames[argument(5)->Opcode()]);
2903       }
2904       return false;
2905     }
2906   }
2907 
2908   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2909   assert(is_vector_mask(mbox_klass), "argument(6) should be a mask class");
2910   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2911 
2912   Node* mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2913   if (mask == nullptr) {
2914     if (C->print_intrinsics()) {
2915       tty->print_cr("  ** unbox failed mask=%s",
2916                     NodeClassNames[argument(6)->Opcode()]);
2917     }
2918     return false;
2919   }
2920 
2921   const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
2922   Node* operation = gvn().transform(VectorNode::make(opc, opd1, mask, vt));
2923 
2924   // Wrap it up in VectorBox to keep object type information.
2925   const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2926   Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2927   set_result(vbox);
2928   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2929   return true;
2930 }
2931 
2932 // public static
2933 // <V extends Vector<E>,
2934 //  E,
2935 //  S extends VectorSpecies<E>>
2936 //  V indexVector(Class<? extends V> vClass, Class<E> eClass,
2937 //                int length,
2938 //                V v, int step, S s,
2939 //                IndexOperation<V, S> defaultImpl)
2940 bool LibraryCallKit::inline_index_vector() {
2941   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2942   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2943   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2944 
2945   if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2946       vector_klass->const_oop() == nullptr || !vlen->is_con() ||
2947       elem_klass->const_oop() == nullptr) {
2948     if (C->print_intrinsics()) {
2949       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
2950                     NodeClassNames[argument(0)->Opcode()],
2951                     NodeClassNames[argument(1)->Opcode()],
2952                     NodeClassNames[argument(2)->Opcode()]);
2953     }
2954     return false; // not enough info for intrinsification
2955   }
2956 
2957   if (!is_klass_initialized(vector_klass)) {
2958     if (C->print_intrinsics()) {
2959       tty->print_cr("  ** klass argument not initialized");
2960     }
2961     return false;
2962   }
2963 
2964   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2965   if (!elem_type->is_primitive_type()) {
2966     if (C->print_intrinsics()) {
2967       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2968     }
2969     return false; // should be primitive type
2970   }
2971 
2972   int num_elem = vlen->get_con();
2973   BasicType elem_bt = elem_type->basic_type();
2974 
2975   // Check whether the iota index generation op is supported by the current hardware
2976   if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
2977     if (C->print_intrinsics()) {
2978       tty->print_cr("  ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2979     }
2980     return false; // not supported
2981   }
2982 
2983   int mul_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_MUL, elem_bt);
2984   int vmul_op = VectorNode::opcode(mul_op, elem_bt);
2985   bool needs_mul = true;
2986   Node* scale = argument(4);
2987   const TypeInt* scale_type = gvn().type(scale)->isa_int();
2988   // Multiply is not needed if the scale is a constant "1".
2989   if (scale_type && scale_type->is_con() && scale_type->get_con() == 1) {
2990     needs_mul = false;
2991   } else {
2992     // Check whether the vector multiply op is supported by the current hardware
2993     if (!arch_supports_vector(vmul_op, num_elem, elem_bt, VecMaskNotUsed)) {
2994       if (C->print_intrinsics()) {
2995         tty->print_cr("  ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2996       }
2997       return false; // not supported
2998     }
2999 
3000     // Check whether the scalar cast op is supported by the current hardware
3001     if (is_floating_point_type(elem_bt) || elem_bt == T_LONG) {
3002       int cast_op = elem_bt == T_LONG ? Op_ConvI2L :
3003                     elem_bt == T_FLOAT? Op_ConvI2F : Op_ConvI2D;
3004       if (!Matcher::match_rule_supported(cast_op)) {
3005         if (C->print_intrinsics()) {
3006           tty->print_cr("  ** Rejected op (%s) because architecture does not support it",
3007                         NodeClassNames[cast_op]);
3008         }
3009         return false; // not supported
3010       }
3011     }
3012   }
3013 
3014   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
3015   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
3016   Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
3017   if (opd == nullptr) {
3018     if (C->print_intrinsics()) {
3019       tty->print_cr("  ** unbox failed vector=%s",
3020                     NodeClassNames[argument(3)->Opcode()]);
3021     }
3022     return false;
3023   }
3024 
3025   int add_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_ADD, elem_bt);
3026   int vadd_op = VectorNode::opcode(add_op, elem_bt);
3027   bool needs_add = true;
3028   // The addition is not needed if all the element values of "opd" are zero
3029   if (VectorNode::is_all_zeros_vector(opd)) {
3030     needs_add = false;
3031   } else {
3032     // Check whether the vector addition op is supported by the current hardware
3033     if (!arch_supports_vector(vadd_op, num_elem, elem_bt, VecMaskNotUsed)) {
3034       if (C->print_intrinsics()) {
3035         tty->print_cr("  ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3036       }
3037       return false; // not supported
3038     }
3039   }
3040 
3041   // Compute the iota indice vector
3042   const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
3043   Node* index = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
3044 
3045   // Broadcast the "scale" to a vector, and multiply the "scale" with iota indice vector.
3046   if (needs_mul) {
3047     switch (elem_bt) {
3048       case T_BOOLEAN: // fall-through
3049       case T_BYTE:    // fall-through
3050       case T_SHORT:   // fall-through
3051       case T_CHAR:    // fall-through
3052       case T_INT: {
3053         // no conversion needed
3054         break;
3055       }
3056       case T_LONG: {
3057         scale = gvn().transform(new ConvI2LNode(scale));
3058         break;
3059       }
3060       case T_FLOAT: {
3061         scale = gvn().transform(new ConvI2FNode(scale));
3062         break;
3063       }
3064       case T_DOUBLE: {
3065         scale = gvn().transform(new ConvI2DNode(scale));
3066         break;
3067       }
3068       default: fatal("%s", type2name(elem_bt));
3069     }
3070     scale = gvn().transform(VectorNode::scalar2vector(scale, num_elem, Type::get_const_basic_type(elem_bt)));
3071     index = gvn().transform(VectorNode::make(vmul_op, index, scale, vt));
3072   }
3073 
3074   // Add "opd" if addition is needed.
3075   if (needs_add) {
3076     index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3077   }
3078   Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3079   set_result(vbox);
3080   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3081   return true;
3082 }
3083 
3084 // public static
3085 // <E,
3086 //  M extends VectorMask<E>>
3087 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3088 //                              long offset, long limit,
3089 //                              IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3090 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3091   const TypeInstPtr* mask_klass   = gvn().type(argument(0))->isa_instptr();
3092   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
3093   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
3094 
3095   if (mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
3096       mask_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
3097     if (C->print_intrinsics()) {
3098       tty->print_cr("  ** missing constant: mclass=%s etype=%s vlen=%s",
3099                     NodeClassNames[argument(0)->Opcode()],
3100                     NodeClassNames[argument(1)->Opcode()],
3101                     NodeClassNames[argument(2)->Opcode()]);
3102     }
3103     return false; // not enough info for intrinsification
3104   }
3105 
3106   if (!is_klass_initialized(mask_klass)) {
3107     if (C->print_intrinsics()) {
3108       tty->print_cr("  ** klass argument not initialized");
3109     }
3110     return false;
3111   }
3112 
3113   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3114   if (!elem_type->is_primitive_type()) {
3115     if (C->print_intrinsics()) {
3116       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
3117     }
3118     return false; // should be primitive type
3119   }
3120 
3121   int num_elem = vlen->get_con();
3122   BasicType elem_bt = elem_type->basic_type();
3123 
3124   // Check whether the necessary ops are supported by current hardware.
3125   bool supports_mask_gen = arch_supports_vector(Op_VectorMaskGen, num_elem, elem_bt, VecMaskUseStore);
3126   if (!supports_mask_gen) {
3127     if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
3128         !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed) ||
3129         !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
3130       if (C->print_intrinsics()) {
3131         tty->print_cr("  ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3132       }
3133       return false; // not supported
3134     }
3135 
3136     // Check whether the scalar cast operation is supported by current hardware.
3137     if (elem_bt != T_LONG) {
3138       int cast_op = is_integral_type(elem_bt) ? Op_ConvL2I
3139                                               : (elem_bt == T_FLOAT ? Op_ConvL2F : Op_ConvL2D);
3140       if (!Matcher::match_rule_supported(cast_op)) {
3141         if (C->print_intrinsics()) {
3142           tty->print_cr("  ** Rejected op (%s) because architecture does not support it",
3143                         NodeClassNames[cast_op]);
3144         }
3145         return false; // not supported
3146       }
3147     }
3148   }
3149 
3150   Node* offset = argument(3);
3151   Node* limit = argument(5);
3152   if (offset == nullptr || limit == nullptr) {
3153     if (C->print_intrinsics()) {
3154       tty->print_cr("  ** offset or limit argument is null");
3155     }
3156     return false; // not supported
3157   }
3158 
3159   ciKlass* box_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
3160   assert(is_vector_mask(box_klass), "argument(0) should be a mask class");
3161   const TypeInstPtr* box_type = TypeInstPtr::make_exact(TypePtr::NotNull, box_klass);
3162 
3163   // We assume "offset > 0 && limit >= offset && limit - offset < num_elem".
3164   // So directly get indexLimit with "indexLimit = limit - offset".
3165   Node* indexLimit = gvn().transform(new SubLNode(limit, offset));
3166   Node* mask = nullptr;
3167   if (supports_mask_gen) {
3168     mask = gvn().transform(VectorMaskGenNode::make(indexLimit, elem_bt, num_elem));
3169   } else {
3170     // Generate the vector mask based on "mask = iota < indexLimit".
3171     // Broadcast "indexLimit" to a vector.
3172     switch (elem_bt) {
3173       case T_BOOLEAN: // fall-through
3174       case T_BYTE:    // fall-through
3175       case T_SHORT:   // fall-through
3176       case T_CHAR:    // fall-through
3177       case T_INT: {
3178         indexLimit = gvn().transform(new ConvL2INode(indexLimit));
3179         break;
3180       }
3181       case T_DOUBLE: {
3182         indexLimit = gvn().transform(new ConvL2DNode(indexLimit));
3183         break;
3184       }
3185       case T_FLOAT: {
3186         indexLimit = gvn().transform(new ConvL2FNode(indexLimit));
3187         break;
3188       }
3189       case T_LONG: {
3190         // no conversion needed
3191         break;
3192       }
3193       default: fatal("%s", type2name(elem_bt));
3194     }
3195     indexLimit = gvn().transform(VectorNode::scalar2vector(indexLimit, num_elem, Type::get_const_basic_type(elem_bt)));
3196 
3197     // Load the "iota" vector.
3198     const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
3199     Node* iota = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
3200 
3201     // Compute the vector mask with "mask = iota < indexLimit".
3202     ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::lt));
3203     const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
3204     mask = gvn().transform(new VectorMaskCmpNode(BoolTest::lt, iota, indexLimit, pred_node, vmask_type));
3205   }
3206   Node* vbox = box_vector(mask, box_type, elem_bt, num_elem);
3207   set_result(vbox);
3208   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3209   return true;
3210 }