1 /*
   2  * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "ci/ciSymbols.hpp"
  27 #include "classfile/vmSymbols.hpp"
  28 #include "opto/library_call.hpp"
  29 #include "opto/runtime.hpp"
  30 #include "opto/vectornode.hpp"
  31 #include "prims/vectorSupport.hpp"
  32 #include "runtime/stubRoutines.hpp"
  33 
  34 #ifdef ASSERT
  35 static bool is_vector(ciKlass* klass) {
  36   return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
  37 }
  38 
  39 static bool check_vbox(const TypeInstPtr* vbox_type) {
  40   assert(vbox_type->klass_is_exact(), "");
  41 
  42   ciInstanceKlass* ik = vbox_type->klass()->as_instance_klass();
  43   assert(is_vector(ik), "not a vector");
  44 
  45   ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
  46   assert(fd1 != NULL, "element type info is missing");
  47 
  48   ciConstant val1 = fd1->constant_value();
  49   BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
  50   assert(is_java_primitive(elem_bt), "element type info is missing");
  51 
  52   ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
  53   assert(fd2 != NULL, "vector length info is missing");
  54 
  55   ciConstant val2 = fd2->constant_value();
  56   assert(val2.as_int() > 0, "vector length info is missing");
  57 
  58   return true;
  59 }
  60 #endif
  61 
  62 static bool is_vector_mask(ciKlass* klass) {
  63   return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
  64 }
  65 
  66 static bool is_vector_shuffle(ciKlass* klass) {
  67   return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
  68 }
  69 
  70 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
  71                                                  VectorMaskUseType mask_use_type, bool has_scalar_args) {
  72   bool is_supported = true;
  73 
  74   // has_scalar_args flag is true only for non-constant scalar shift count,
  75   // since in this case shift needs to be broadcasted.
  76   if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
  77        (has_scalar_args &&
  78          !arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed))) {
  79     is_supported = false;
  80   }
  81 
  82   if (is_supported) {
  83     // Check whether mask unboxing is supported.
  84     if ((mask_use_type & VecMaskUseLoad) != 0) {
  85       if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt)) {
  86       #ifndef PRODUCT
  87         if (C->print_intrinsics()) {
  88           tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
  89                         NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
  90         }
  91       #endif
  92         return false;
  93       }
  94     }
  95 
  96     if ((mask_use_type & VecMaskUsePred) != 0) {
  97       if (!Matcher::has_predicated_vectors() ||
  98           !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
  99       #ifndef PRODUCT
 100         if (C->print_intrinsics()) {
 101           tty->print_cr("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
 102                         NodeClassNames[opc], type2name(elem_bt), num_elem);
 103         }
 104       #endif
 105         return false;
 106       }
 107     }
 108   }
 109 
 110   int lshiftopc, rshiftopc;
 111   switch(elem_bt) {
 112     case T_BYTE:
 113       lshiftopc = Op_LShiftI;
 114       rshiftopc = Op_URShiftB;
 115       break;
 116     case T_SHORT:
 117       lshiftopc = Op_LShiftI;
 118       rshiftopc = Op_URShiftS;
 119       break;
 120     case T_INT:
 121       lshiftopc = Op_LShiftI;
 122       rshiftopc = Op_URShiftI;
 123       break;
 124     case T_LONG:
 125       lshiftopc = Op_LShiftL;
 126       rshiftopc = Op_URShiftL;
 127       break;
 128     default:
 129       assert(false, "Unexpected type");
 130   }
 131   int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
 132   int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
 133   if (!is_supported &&
 134       arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
 135       arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
 136       arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
 137     is_supported = true;
 138   }
 139   return is_supported;
 140 }
 141 
 142 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
 143   assert(EnableVectorSupport, "");
 144 
 145   PreserveReexecuteState preexecs(this);
 146   jvms()->set_should_reexecute(true);
 147 
 148   VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
 149   set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
 150   make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
 151   set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
 152   set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
 153   Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
 154 
 155   assert(check_vbox(vbox_type), "");
 156   const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->klass()));
 157   VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
 158   return gvn().transform(vbox);
 159 }
 160 
 161 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) {
 162   assert(EnableVectorSupport, "");
 163   const TypeInstPtr* vbox_type_v = gvn().type(v)->is_instptr();
 164   if (vbox_type->klass() != vbox_type_v->klass()) {
 165     return NULL; // arguments don't agree on vector shapes
 166   }
 167   if (vbox_type_v->maybe_null()) {
 168     return NULL; // no nulls are allowed
 169   }
 170   assert(check_vbox(vbox_type), "");
 171   const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->klass()));
 172   Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector));
 173   return unbox;
 174 }
 175 
 176 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
 177   assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
 178   juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
 179   Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
 180   Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
 181   return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
 182 }
 183 
 184 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
 185   // Check that the operation is valid.
 186   if (sopc <= 0) {
 187 #ifndef PRODUCT
 188     if (C->print_intrinsics()) {
 189       tty->print_cr("  ** Rejected intrinsification because no valid vector op could be extracted");
 190     }
 191 #endif
 192     return false;
 193   }
 194 
 195   if (VectorNode::is_vector_rotate(sopc)) {
 196     if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
 197 #ifndef PRODUCT
 198       if (C->print_intrinsics()) {
 199         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
 200                       NodeClassNames[sopc], type2name(type), num_elem);
 201       }
 202 #endif
 203       return false;
 204     }
 205   } else {
 206     // Check that architecture supports this op-size-type combination.
 207     if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
 208 #ifndef PRODUCT
 209       if (C->print_intrinsics()) {
 210         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support it",
 211                       NodeClassNames[sopc], type2name(type), num_elem);
 212       }
 213 #endif
 214       return false;
 215     } else {
 216       assert(Matcher::match_rule_supported(sopc), "must be supported");
 217     }
 218   }
 219 
 220   if (num_elem == 1) {
 221     if (mask_use_type != VecMaskNotUsed) {
 222 #ifndef PRODUCT
 223       if (C->print_intrinsics()) {
 224         tty->print_cr("  ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
 225                       NodeClassNames[sopc], type2name(type), num_elem);
 226       }
 227 #endif
 228       return false;
 229     }
 230 
 231     if (sopc != 0) {
 232       if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
 233 #ifndef PRODUCT
 234         if (C->print_intrinsics()) {
 235           tty->print_cr("  ** Not a svml call or load/store vector op (%s,%s,%d)",
 236                         NodeClassNames[sopc], type2name(type), num_elem);
 237         }
 238 #endif
 239         return false;
 240       }
 241     }
 242   }
 243 
 244   if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
 245       Matcher::supports_vector_variable_shifts() == false) {
 246     if (C->print_intrinsics()) {
 247       tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
 248                     NodeClassNames[sopc], type2name(type), num_elem);
 249     }
 250     return false;
 251   }
 252 
 253   // Check whether mask unboxing is supported.
 254   if ((mask_use_type & VecMaskUseLoad) != 0) {
 255     if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type)) {
 256     #ifndef PRODUCT
 257       if (C->print_intrinsics()) {
 258         tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
 259                       NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
 260       }
 261     #endif
 262       return false;
 263     }
 264   }
 265 
 266   // Check whether mask boxing is supported.
 267   if ((mask_use_type & VecMaskUseStore) != 0) {
 268     if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type)) {
 269     #ifndef PRODUCT
 270       if (C->print_intrinsics()) {
 271         tty->print_cr("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
 272                       NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
 273       }
 274     #endif
 275       return false;
 276     }
 277   }
 278 
 279   if ((mask_use_type & VecMaskUsePred) != 0) {
 280     if (!Matcher::has_predicated_vectors() ||
 281         !Matcher::match_rule_supported_vector_masked(sopc, num_elem, type)) {
 282     #ifndef PRODUCT
 283       if (C->print_intrinsics()) {
 284         tty->print_cr("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
 285                       NodeClassNames[sopc], type2name(type), num_elem);
 286       }
 287     #endif
 288       return false;
 289     }
 290   }
 291 
 292   return true;
 293 }
 294 
 295 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
 296   if (vec_klass->const_oop() == NULL) {
 297     return false; // uninitialized or some kind of unsafe access
 298   }
 299   assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != NULL, "klass instance expected");
 300   ciInstanceKlass* klass =  vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
 301   return klass->is_initialized();
 302 }
 303 
 304 // public static
 305 // <V extends Vector<E>,
 306 //  M extends VectorMask<E>,
 307 //  E>
 308 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 309 //           int length, V v, M m,
 310 //           UnaryOperation<V, M> defaultImpl)
 311 //
 312 // public static
 313 // <V,
 314 //  M extends VectorMask<E>,
 315 //  E>
 316 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 317 //            int length, V v1, V v2, M m,
 318 //            BinaryOperation<V, M> defaultImpl)
 319 //
 320 // public static
 321 // <V extends Vector<E>,
 322 //  M extends VectorMask<E>,
 323 //  E>
 324 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 325 //             int length, V v1, V v2, V v3, M m,
 326 //             TernaryOperation<V, M> defaultImpl)
 327 //
 328 bool LibraryCallKit::inline_vector_nary_operation(int n) {
 329   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
 330   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
 331   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
 332   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
 333   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
 334 
 335   if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
 336       !opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 337     if (C->print_intrinsics()) {
 338       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
 339                     NodeClassNames[argument(0)->Opcode()],
 340                     NodeClassNames[argument(1)->Opcode()],
 341                     NodeClassNames[argument(3)->Opcode()],
 342                     NodeClassNames[argument(4)->Opcode()]);
 343     }
 344     return false; // not enough info for intrinsification
 345   }
 346 
 347   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 348   if (!elem_type->is_primitive_type()) {
 349     if (C->print_intrinsics()) {
 350       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 351     }
 352     return false; // should be primitive type
 353   }
 354   if (!is_klass_initialized(vector_klass)) {
 355     if (C->print_intrinsics()) {
 356       tty->print_cr("  ** klass argument not initialized");
 357     }
 358     return false;
 359   }
 360 
 361   // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
 362   // is used to control this operation.
 363   const Type* vmask_type = gvn().type(argument(n + 5));
 364   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
 365   if (is_masked_op) {
 366     if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
 367       if (C->print_intrinsics()) {
 368         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
 369       }
 370       return false; // not enough info for intrinsification
 371     }
 372 
 373     if (!is_klass_initialized(mask_klass)) {
 374       if (C->print_intrinsics()) {
 375         tty->print_cr("  ** mask klass argument not initialized");
 376       }
 377       return false;
 378     }
 379 
 380     if (vmask_type->maybe_null()) {
 381       if (C->print_intrinsics()) {
 382         tty->print_cr("  ** null mask values are not allowed for masked op");
 383       }
 384       return false;
 385     }
 386   }
 387 
 388   BasicType elem_bt = elem_type->basic_type();
 389   int num_elem = vlen->get_con();
 390   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
 391   int sopc = VectorNode::opcode(opc, elem_bt);
 392   if ((opc != Op_CallLeafVector) && (sopc == 0)) {
 393     if (C->print_intrinsics()) {
 394       tty->print_cr("  ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
 395     }
 396     return false; // operation not supported
 397   }
 398   if (num_elem == 1) {
 399     if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
 400       if (C->print_intrinsics()) {
 401         tty->print_cr("  ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
 402                       n, opc, num_elem, type2name(elem_bt));
 403       }
 404       return false;
 405     }
 406   }
 407   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 408   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 409 
 410   if (is_vector_mask(vbox_klass)) {
 411     assert(!is_masked_op, "mask operations do not need mask to control");
 412   }
 413 
 414   if (opc == Op_CallLeafVector) {
 415     if (!UseVectorStubs) {
 416       if (C->print_intrinsics()) {
 417         tty->print_cr("  ** vector stubs support is disabled");
 418       }
 419       return false;
 420     }
 421     if (!Matcher::supports_vector_calling_convention()) {
 422       if (C->print_intrinsics()) {
 423         tty->print_cr("  ** no vector calling conventions supported");
 424       }
 425       return false;
 426     }
 427     if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
 428       if (C->print_intrinsics()) {
 429         tty->print_cr("  ** vector size (vlen=%d, etype=%s) is not supported",
 430                       num_elem, type2name(elem_bt));
 431       }
 432       return false;
 433     }
 434   }
 435 
 436   // When using mask, mask use type needs to be VecMaskUseLoad.
 437   VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
 438                                       : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
 439   if ((sopc != 0) && !arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
 440     if (C->print_intrinsics()) {
 441       tty->print_cr("  ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
 442                     n, sopc, num_elem, type2name(elem_bt),
 443                     is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
 444     }
 445     return false; // not supported
 446   }
 447 
 448   // Return true if current platform has implemented the masked operation with predicate feature.
 449   bool use_predicate = is_masked_op && sopc != 0 && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
 450   if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
 451     if (C->print_intrinsics()) {
 452       tty->print_cr("  ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
 453                     n, sopc, num_elem, type2name(elem_bt));
 454     }
 455     return false;
 456   }
 457 
 458   Node* opd1 = NULL; Node* opd2 = NULL; Node* opd3 = NULL;
 459   switch (n) {
 460     case 3: {
 461       opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
 462       if (opd3 == NULL) {
 463         if (C->print_intrinsics()) {
 464           tty->print_cr("  ** unbox failed v3=%s",
 465                         NodeClassNames[argument(7)->Opcode()]);
 466         }
 467         return false;
 468       }
 469       // fall-through
 470     }
 471     case 2: {
 472       opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
 473       if (opd2 == NULL) {
 474         if (C->print_intrinsics()) {
 475           tty->print_cr("  ** unbox failed v2=%s",
 476                         NodeClassNames[argument(6)->Opcode()]);
 477         }
 478         return false;
 479       }
 480       // fall-through
 481     }
 482     case 1: {
 483       opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
 484       if (opd1 == NULL) {
 485         if (C->print_intrinsics()) {
 486           tty->print_cr("  ** unbox failed v1=%s",
 487                         NodeClassNames[argument(5)->Opcode()]);
 488         }
 489         return false;
 490       }
 491       break;
 492     }
 493     default: fatal("unsupported arity: %d", n);
 494   }
 495 
 496   Node* mask = NULL;
 497   if (is_masked_op) {
 498     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
 499     assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
 500     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
 501     mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
 502     if (mask == NULL) {
 503       if (C->print_intrinsics()) {
 504         tty->print_cr("  ** unbox failed mask=%s",
 505                       NodeClassNames[argument(n + 5)->Opcode()]);
 506       }
 507       return false;
 508     }
 509   }
 510 
 511   Node* operation = NULL;
 512   if (opc == Op_CallLeafVector) {
 513     assert(UseVectorStubs, "sanity");
 514     operation = gen_call_to_svml(opr->get_con(), elem_bt, num_elem, opd1, opd2);
 515     if (operation == NULL) {
 516       if (C->print_intrinsics()) {
 517         tty->print_cr("  ** svml call failed for %s_%s_%d",
 518                          (elem_bt == T_FLOAT)?"float":"double",
 519                          VectorSupport::svmlname[opr->get_con() - VectorSupport::VECTOR_OP_SVML_START],
 520                          num_elem * type2aelembytes(elem_bt));
 521       }
 522       return false;
 523      }
 524   } else {
 525     const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
 526     switch (n) {
 527       case 1:
 528       case 2: {
 529         operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass));
 530         break;
 531       }
 532       case 3: {
 533         operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
 534         break;
 535       }
 536       default: fatal("unsupported arity: %d", n);
 537     }
 538   }
 539 
 540   if (is_masked_op && mask != NULL) {
 541     if (use_predicate) {
 542       operation->add_req(mask);
 543       operation->add_flag(Node::Flag_is_predicated_vector);
 544     } else {
 545       operation = gvn().transform(operation);
 546       operation = new VectorBlendNode(opd1, operation, mask);
 547     }
 548   }
 549   operation = gvn().transform(operation);
 550 
 551   // Wrap it up in VectorBox to keep object type information.
 552   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
 553   set_result(vbox);
 554   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 555   return true;
 556 }
 557 
 558 // <Sh extends VectorShuffle<E>,  E>
 559 //  Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
 560 //                  int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
 561 bool LibraryCallKit::inline_vector_shuffle_iota() {
 562   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
 563   const TypeInt*     vlen          = gvn().type(argument(3))->isa_int();
 564   const TypeInt*     start_val     = gvn().type(argument(4))->isa_int();
 565   const TypeInt*     step_val      = gvn().type(argument(5))->isa_int();
 566   const TypeInt*     wrap          = gvn().type(argument(6))->isa_int();
 567 
 568   Node* start = argument(4);
 569   Node* step  = argument(5);
 570 
 571   if (shuffle_klass == NULL || vlen == NULL || start_val == NULL || step_val == NULL || wrap == NULL) {
 572     return false; // dead code
 573   }
 574   if (!vlen->is_con() || !is_power_of_2(vlen->get_con()) ||
 575       shuffle_klass->const_oop() == NULL || !wrap->is_con()) {
 576     return false; // not enough info for intrinsification
 577   }
 578   if (!is_klass_initialized(shuffle_klass)) {
 579     if (C->print_intrinsics()) {
 580       tty->print_cr("  ** klass argument not initialized");
 581     }
 582     return false;
 583   }
 584 
 585   int do_wrap = wrap->get_con();
 586   int num_elem = vlen->get_con();
 587   BasicType elem_bt = T_BYTE;
 588 
 589   if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed)) {
 590     return false;
 591   }
 592   if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed)) {
 593     return false;
 594   }
 595   if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) {
 596     return false;
 597   }
 598   if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
 599     return false;
 600   }
 601   if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
 602     return false;
 603   }
 604   if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
 605     return false;
 606   }
 607 
 608   const Type * type_bt = Type::get_const_basic_type(elem_bt);
 609   const TypeVect * vt  = TypeVect::make(type_bt, num_elem);
 610 
 611   Node* res =  gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
 612 
 613   if(!step_val->is_con() || !is_power_of_2(step_val->get_con())) {
 614     Node* bcast_step     = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt));
 615     res = gvn().transform(VectorNode::make(Op_MulI, res, bcast_step, num_elem, elem_bt));
 616   } else if (step_val->get_con() > 1) {
 617     Node* cnt = gvn().makecon(TypeInt::make(log2i_exact(step_val->get_con())));
 618     Node* shift_cnt = vector_shift_count(cnt, Op_LShiftI, elem_bt, num_elem);
 619     res = gvn().transform(VectorNode::make(Op_LShiftVB, res, shift_cnt, vt));
 620   }
 621 
 622   if (!start_val->is_con() || start_val->get_con() != 0) {
 623     Node* bcast_start    = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt));
 624     res = gvn().transform(VectorNode::make(Op_AddI, res, bcast_start, num_elem, elem_bt));
 625   }
 626 
 627   Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1));
 628   Node * bcast_mod  = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt));
 629   if(do_wrap)  {
 630     // Wrap the indices greater than lane count.
 631     res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
 632   } else {
 633     ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::ge));
 634     Node * lane_cnt  = gvn().makecon(TypeInt::make(num_elem));
 635     Node * bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt));
 636     const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
 637     Node* mask = gvn().transform(new VectorMaskCmpNode(BoolTest::ge, bcast_lane_cnt, res, pred_node, vmask_type));
 638 
 639     // Make the indices greater than lane count as -ve values. This matches the java side implementation.
 640     res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
 641     Node * biased_val = gvn().transform(VectorNode::make(Op_SubI, res, bcast_lane_cnt, num_elem, elem_bt));
 642     res = gvn().transform(new VectorBlendNode(biased_val, res, mask));
 643   }
 644 
 645   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 646   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 647 
 648   // Wrap it up in VectorBox to keep object type information.
 649   res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
 650   set_result(res);
 651   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 652   return true;
 653 }
 654 
 655 // <E, M>
 656 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
 657 //                          int length, M m, VectorMaskOp<M> defaultImpl)
 658 bool LibraryCallKit::inline_vector_mask_operation() {
 659   const TypeInt*     oper       = gvn().type(argument(0))->isa_int();
 660   const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
 661   const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
 662   const TypeInt*     vlen       = gvn().type(argument(3))->isa_int();
 663   Node*              mask       = argument(4);
 664 
 665   if (mask_klass == NULL || elem_klass == NULL || mask->is_top() || vlen == NULL) {
 666     return false; // dead code
 667   }
 668 
 669   if (!is_klass_initialized(mask_klass)) {
 670     if (C->print_intrinsics()) {
 671       tty->print_cr("  ** klass argument not initialized");
 672     }
 673     return false;
 674   }
 675 
 676   int num_elem = vlen->get_con();
 677   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 678   BasicType elem_bt = elem_type->basic_type();
 679 
 680   if (!arch_supports_vector(Op_LoadVector, num_elem, T_BOOLEAN, VecMaskNotUsed)) {
 681     if (C->print_intrinsics()) {
 682       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 683                     Op_LoadVector, num_elem, type2name(T_BOOLEAN));
 684     }
 685     return false; // not supported
 686   }
 687 
 688   int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
 689   if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskNotUsed)) {
 690     if (C->print_intrinsics()) {
 691       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 692                     mopc, num_elem, type2name(elem_bt));
 693     }
 694     return false; // not supported
 695   }
 696 
 697   const Type* elem_ty = Type::get_const_basic_type(elem_bt);
 698   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
 699   const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
 700   Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
 701   if (mask_vec->bottom_type()->isa_vectmask() == NULL) {
 702     mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
 703   }
 704   const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
 705   Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
 706   if (mopc != Op_VectorMaskToLong) {
 707     maskoper = ConvI2L(maskoper);
 708   }
 709   set_result(maskoper);
 710 
 711   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 712   return true;
 713 }
 714 
 715 // public static
 716 // <V,
 717 //  Sh extends VectorShuffle<E>,
 718 //  E>
 719 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
 720 //                   Class<? extends Sh> shuffleClass, Sh s, int length,
 721 //                   ShuffleToVectorOperation<V, Sh, E> defaultImpl)
 722 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
 723   const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
 724   const TypeInstPtr* elem_klass    = gvn().type(argument(1))->isa_instptr();
 725   const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
 726   Node*              shuffle       = argument(3);
 727   const TypeInt*     vlen          = gvn().type(argument(4))->isa_int();
 728 
 729   if (vector_klass == NULL || elem_klass == NULL || shuffle_klass == NULL || shuffle->is_top() || vlen == NULL) {
 730     return false; // dead code
 731   }
 732   if (!vlen->is_con() || vector_klass->const_oop() == NULL || shuffle_klass->const_oop() == NULL) {
 733     return false; // not enough info for intrinsification
 734   }
 735   if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
 736     if (C->print_intrinsics()) {
 737       tty->print_cr("  ** klass argument not initialized");
 738     }
 739     return false;
 740   }
 741 
 742   int num_elem = vlen->get_con();
 743   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 744   BasicType elem_bt = elem_type->basic_type();
 745 
 746   if (num_elem < 4) {
 747     return false;
 748   }
 749 
 750   int cast_vopc = VectorCastNode::opcode(T_BYTE); // from shuffle of type T_BYTE
 751   // Make sure that cast is implemented to particular type/size combination.
 752   if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
 753     if (C->print_intrinsics()) {
 754       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 755         cast_vopc, num_elem, type2name(elem_bt));
 756     }
 757     return false;
 758   }
 759 
 760   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 761   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 762 
 763   // Unbox shuffle with true flag to indicate its load shuffle to vector
 764   // shuffle is a byte array
 765   Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, T_BYTE, num_elem, true);
 766 
 767   // cast byte to target element type
 768   shuffle_vec = gvn().transform(VectorCastNode::make(cast_vopc, shuffle_vec, elem_bt, num_elem));
 769 
 770   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 771   const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 772 
 773   // Box vector
 774   Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem);
 775   set_result(res);
 776   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 777   return true;
 778 }
 779 
 780 // public static
 781 // <M,
 782 //  S extends VectorSpecies<E>,
 783 //  E>
 784 // M broadcastCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
 785 //                    long bits, S s,
 786 //                    BroadcastOperation<M, E, S> defaultImpl)
 787 bool LibraryCallKit::inline_vector_broadcast_coerced() {
 788   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 789   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 790   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 791 
 792   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
 793       vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 794     if (C->print_intrinsics()) {
 795       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
 796                     NodeClassNames[argument(0)->Opcode()],
 797                     NodeClassNames[argument(1)->Opcode()],
 798                     NodeClassNames[argument(2)->Opcode()]);
 799     }
 800     return false; // not enough info for intrinsification
 801   }
 802 
 803   if (!is_klass_initialized(vector_klass)) {
 804     if (C->print_intrinsics()) {
 805       tty->print_cr("  ** klass argument not initialized");
 806     }
 807     return false;
 808   }
 809   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 810   if (!elem_type->is_primitive_type()) {
 811     if (C->print_intrinsics()) {
 812       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 813     }
 814     return false; // should be primitive type
 815   }
 816   BasicType elem_bt = elem_type->basic_type();
 817   int num_elem = vlen->get_con();
 818   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 819   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 820 
 821   // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
 822   if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt,
 823                             (is_vector_mask(vbox_klass) ? VecMaskUseStore : VecMaskNotUsed), true /*has_scalar_args*/)) {
 824     if (C->print_intrinsics()) {
 825       tty->print_cr("  ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d",
 826                     num_elem, type2name(elem_bt),
 827                     is_vector_mask(vbox_klass) ? 1 : 0);
 828     }
 829     return false; // not supported
 830   }
 831 
 832   Node* bits = argument(3); // long
 833   Node* elem = NULL;
 834   switch (elem_bt) {
 835     case T_BOOLEAN: // fall-through
 836     case T_BYTE:    // fall-through
 837     case T_SHORT:   // fall-through
 838     case T_CHAR:    // fall-through
 839     case T_INT: {
 840       elem = gvn().transform(new ConvL2INode(bits));
 841       break;
 842     }
 843     case T_DOUBLE: {
 844       elem = gvn().transform(new MoveL2DNode(bits));
 845       break;
 846     }
 847     case T_FLOAT: {
 848       bits = gvn().transform(new ConvL2INode(bits));
 849       elem = gvn().transform(new MoveI2FNode(bits));
 850       break;
 851     }
 852     case T_LONG: {
 853       elem = bits; // no conversion needed
 854       break;
 855     }
 856     default: fatal("%s", type2name(elem_bt));
 857   }
 858 
 859   Node* broadcast = VectorNode::scalar2vector(elem, num_elem, Type::get_const_basic_type(elem_bt), is_vector_mask(vbox_klass));
 860   broadcast = gvn().transform(broadcast);
 861 
 862   Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
 863   set_result(box);
 864   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 865   return true;
 866 }
 867 
 868 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type) {
 869   assert(arr_type != NULL, "unexpected");
 870   BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
 871   if (elem_bt == arr_elem_bt) {
 872     return true;
 873   } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
 874     // Load/store of short vector from/to char[] is supported
 875     return true;
 876   } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
 877     // Load/store of byte vector from/to boolean[] is supported
 878     return true;
 879   } else {
 880     return false;
 881   }
 882 }
 883 
 884 // public static
 885 // <C,
 886 //  VM,
 887 //  E,
 888 //  S extends VectorSpecies<E>>
 889 // VM load(Class<? extends VM> vmClass, Class<E> elementType, int length,
 890 //         Object base, long offset,    // Unsafe addressing
 891 //         C container, int index, S s,     // Arguments for default implementation
 892 //         LoadOperation<C, VM, E, S> defaultImpl)
 893 //
 894 // public static
 895 // <C,
 896 //  V extends Vector<?>>
 897 // void store(Class<?> vectorClass, Class<?> elementType, int length,
 898 //            Object base, long offset,    // Unsafe addressing
 899 //            V v,
 900 //            C container, int index,      // Arguments for default implementation
 901 //            StoreVectorOperation<C, V> defaultImpl)
 902 
 903 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
 904   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 905   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 906   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 907 
 908   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
 909       vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 910     if (C->print_intrinsics()) {
 911       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
 912                     NodeClassNames[argument(0)->Opcode()],
 913                     NodeClassNames[argument(1)->Opcode()],
 914                     NodeClassNames[argument(2)->Opcode()]);
 915     }
 916     return false; // not enough info for intrinsification
 917   }
 918   if (!is_klass_initialized(vector_klass)) {
 919     if (C->print_intrinsics()) {
 920       tty->print_cr("  ** klass argument not initialized");
 921     }
 922     return false;
 923   }
 924 
 925   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 926   if (!elem_type->is_primitive_type()) {
 927     if (C->print_intrinsics()) {
 928       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 929     }
 930     return false; // should be primitive type
 931   }
 932   BasicType elem_bt = elem_type->basic_type();
 933   int num_elem = vlen->get_con();
 934 
 935   // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
 936   if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
 937     if (C->print_intrinsics()) {
 938       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
 939                     is_store, is_store ? "store" : "load",
 940                     num_elem, type2name(elem_bt));
 941     }
 942     return false; // not supported
 943   }
 944 
 945   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 946   bool is_mask = is_vector_mask(vbox_klass);
 947 
 948   Node* base = argument(3);
 949   Node* offset = ConvL2X(argument(4));
 950 
 951   // Save state and restore on bailout
 952   uint old_sp = sp();
 953   SafePointNode* old_map = clone_map();
 954 
 955   Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
 956 
 957   // The memory barrier checks are based on ones for unsafe access.
 958   // This is not 1-1 implementation.
 959   const Type *const base_type = gvn().type(base);
 960 
 961   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
 962   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
 963 
 964   const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
 965   const bool in_heap   = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
 966 
 967   const bool is_mixed_access = !in_heap && !in_native;
 968 
 969   const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == NULL);
 970 
 971   const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
 972 
 973   // Now handle special case where load/store happens from/to byte array but element type is not byte.
 974   bool using_byte_array = arr_type != NULL && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE;
 975   // Handle loading masks.
 976   // If there is no consistency between array and vector element types, it must be special byte array case or loading masks
 977   if (arr_type != NULL && !using_byte_array && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type)) {
 978     if (C->print_intrinsics()) {
 979       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
 980                     is_store, is_store ? "store" : "load",
 981                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
 982     }
 983     set_map(old_map);
 984     set_sp(old_sp);
 985     return false;
 986   }
 987   // Since we are using byte array, we need to double check that the byte operations are supported by backend.
 988   if (using_byte_array) {
 989     int byte_num_elem = num_elem * type2aelembytes(elem_bt);
 990     if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, byte_num_elem, T_BYTE, VecMaskNotUsed)
 991         || !arch_supports_vector(Op_VectorReinterpret, byte_num_elem, T_BYTE, VecMaskNotUsed)) {
 992       if (C->print_intrinsics()) {
 993         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
 994                       is_store, is_store ? "store" : "load",
 995                       byte_num_elem, type2name(elem_bt));
 996       }
 997       set_map(old_map);
 998       set_sp(old_sp);
 999       return false; // not supported
1000     }
1001   }
1002   if (is_mask) {
1003     if (!arch_supports_vector(Op_LoadVector, num_elem, T_BOOLEAN, VecMaskNotUsed)) {
1004       if (C->print_intrinsics()) {
1005         tty->print_cr("  ** not supported: arity=%d op=%s/mask vlen=%d etype=bit ismask=no",
1006                       is_store, is_store ? "store" : "load",
1007                       num_elem);
1008       }
1009       set_map(old_map);
1010       set_sp(old_sp);
1011       return false; // not supported
1012     }
1013     if (!is_store) {
1014       if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1015         set_map(old_map);
1016         set_sp(old_sp);
1017         return false; // not supported
1018       }
1019     } else {
1020       if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
1021         set_map(old_map);
1022         set_sp(old_sp);
1023         return false; // not supported
1024       }
1025     }
1026   }
1027 
1028   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1029 
1030   if (needs_cpu_membar) {
1031     insert_mem_bar(Op_MemBarCPUOrder);
1032   }
1033 
1034   if (is_store) {
1035     Node* val = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1036     if (val == NULL) {
1037       set_map(old_map);
1038       set_sp(old_sp);
1039       return false; // operand unboxing failed
1040     }
1041     set_all_memory(reset_memory());
1042 
1043     // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
1044     int store_num_elem = num_elem;
1045     if (using_byte_array) {
1046       store_num_elem = num_elem * type2aelembytes(elem_bt);
1047       const TypeVect* to_vect_type = TypeVect::make(T_BYTE, store_num_elem);
1048       val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1049     }
1050 
1051     Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
1052     set_memory(vstore, addr_type);
1053   } else {
1054     // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
1055     Node* vload = NULL;
1056     if (using_byte_array) {
1057       int load_num_elem = num_elem * type2aelembytes(elem_bt);
1058       vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, load_num_elem, T_BYTE));
1059       const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1060       vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1061     } else {
1062       // Special handle for masks
1063       if (is_mask) {
1064         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
1065         vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
1066       } else {
1067         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
1068       }
1069     }
1070     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1071     set_result(box);
1072   }
1073 
1074   old_map->destruct(&_gvn);
1075 
1076   if (needs_cpu_membar) {
1077     insert_mem_bar(Op_MemBarCPUOrder);
1078   }
1079 
1080   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1081   return true;
1082 }
1083 
1084 // public static
1085 // <C,
1086 //  V extends Vector<?>,
1087 //  E,
1088 //  S extends VectorSpecies<E>,
1089 //  M extends VectorMask<E>>
1090 // V loadMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1091 //              int length, Object base, long offset, M m,
1092 //              C container, int index, S s,  // Arguments for default implementation
1093 //              LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1094 //
1095 // public static
1096 // <C,
1097 //  V extends Vector<E>,
1098 //  M extends VectorMask<E>,
1099 //  E>
1100 // void storeMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1101 //                  int length, Object base, long offset,
1102 //                  V v, M m,
1103 //                  C container, int index,  // Arguments for default implementation
1104 //                  StoreVectorMaskedOperation<C, V, M, E> defaultImpl) {
1105 //
1106 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1107   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1108   const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1109   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1110   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1111 
1112   if (vector_klass == NULL || mask_klass == NULL || elem_klass == NULL || vlen == NULL ||
1113       vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
1114       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1115     if (C->print_intrinsics()) {
1116       tty->print_cr("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1117                     NodeClassNames[argument(0)->Opcode()],
1118                     NodeClassNames[argument(1)->Opcode()],
1119                     NodeClassNames[argument(2)->Opcode()],
1120                     NodeClassNames[argument(3)->Opcode()]);
1121     }
1122     return false; // not enough info for intrinsification
1123   }
1124   if (!is_klass_initialized(vector_klass)) {
1125     if (C->print_intrinsics()) {
1126       tty->print_cr("  ** klass argument not initialized");
1127     }
1128     return false;
1129   }
1130 
1131   if (!is_klass_initialized(mask_klass)) {
1132     if (C->print_intrinsics()) {
1133       tty->print_cr("  ** mask klass argument not initialized");
1134     }
1135     return false;
1136   }
1137 
1138   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1139   if (!elem_type->is_primitive_type()) {
1140     if (C->print_intrinsics()) {
1141       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1142     }
1143     return false; // should be primitive type
1144   }
1145 
1146   BasicType elem_bt = elem_type->basic_type();
1147   int num_elem = vlen->get_con();
1148 
1149   Node* base = argument(4);
1150   Node* offset = ConvL2X(argument(5));
1151 
1152   // Save state and restore on bailout
1153   uint old_sp = sp();
1154   SafePointNode* old_map = clone_map();
1155 
1156   Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1157   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1158   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1159 
1160   // Now handle special case where load/store happens from/to byte array but element type is not byte.
1161   bool using_byte_array = arr_type != NULL && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE;
1162   // If there is no consistency between array and vector element types, it must be special byte array case
1163   if (arr_type != NULL && !using_byte_array && !elem_consistent_with_arr(elem_bt, arr_type)) {
1164     if (C->print_intrinsics()) {
1165       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
1166                     is_store, is_store ? "storeMasked" : "loadMasked",
1167                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1168     }
1169     set_map(old_map);
1170     set_sp(old_sp);
1171     return false;
1172   }
1173 
1174   int mem_num_elem = using_byte_array ? num_elem * type2aelembytes(elem_bt) : num_elem;
1175   BasicType mem_elem_bt = using_byte_array ? T_BYTE : elem_bt;
1176   bool use_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
1177                                             mem_num_elem, mem_elem_bt,
1178                                             (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred));
1179   // Masked vector store operation needs the architecture predicate feature. We need to check
1180   // whether the predicated vector operation is supported by backend.
1181   if (is_store && !use_predicate) {
1182     if (C->print_intrinsics()) {
1183       tty->print_cr("  ** not supported: op=storeMasked vlen=%d etype=%s using_byte_array=%d",
1184                     num_elem, type2name(elem_bt), using_byte_array ? 1 : 0);
1185     }
1186     set_map(old_map);
1187     set_sp(old_sp);
1188     return false;
1189   }
1190 
1191   // This only happens for masked vector load. If predicate is not supported, then check whether
1192   // the normal vector load and blend operations are supported by backend.
1193   if (!use_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1194       !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1195     if (C->print_intrinsics()) {
1196       tty->print_cr("  ** not supported: op=loadMasked vlen=%d etype=%s using_byte_array=%d",
1197                     num_elem, type2name(elem_bt), using_byte_array ? 1 : 0);
1198     }
1199     set_map(old_map);
1200     set_sp(old_sp);
1201     return false;
1202   }
1203 
1204   // Since we are using byte array, we need to double check that the vector reinterpret operation
1205   // with byte type is supported by backend.
1206   if (using_byte_array) {
1207     if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1208       if (C->print_intrinsics()) {
1209         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s using_byte_array=1",
1210                       is_store, is_store ? "storeMasked" : "loadMasked",
1211                       num_elem, type2name(elem_bt));
1212       }
1213       set_map(old_map);
1214       set_sp(old_sp);
1215       return false;
1216     }
1217   }
1218 
1219   // Since it needs to unbox the mask, we need to double check that the related load operations
1220   // for mask are supported by backend.
1221   if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1222     if (C->print_intrinsics()) {
1223       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s",
1224                       is_store, is_store ? "storeMasked" : "loadMasked",
1225                       num_elem, type2name(elem_bt));
1226     }
1227     set_map(old_map);
1228     set_sp(old_sp);
1229     return false;
1230   }
1231 
1232   // Can base be NULL? Otherwise, always on-heap access.
1233   bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1234   if (can_access_non_heap) {
1235     insert_mem_bar(Op_MemBarCPUOrder);
1236   }
1237 
1238   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1239   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1240   assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1241   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1242   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1243 
1244   Node* mask = unbox_vector(is_store ? argument(8) : argument(7), mbox_type, elem_bt, num_elem);
1245   if (mask == NULL) {
1246     if (C->print_intrinsics()) {
1247       tty->print_cr("  ** unbox failed mask=%s",
1248                     is_store ? NodeClassNames[argument(8)->Opcode()]
1249                              : NodeClassNames[argument(7)->Opcode()]);
1250     }
1251     set_map(old_map);
1252     set_sp(old_sp);
1253     return false;
1254   }
1255 
1256   if (is_store) {
1257     Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
1258     if (val == NULL) {
1259       if (C->print_intrinsics()) {
1260         tty->print_cr("  ** unbox failed vector=%s",
1261                       NodeClassNames[argument(7)->Opcode()]);
1262       }
1263       set_map(old_map);
1264       set_sp(old_sp);
1265       return false; // operand unboxing failed
1266     }
1267     set_all_memory(reset_memory());
1268 
1269     if (using_byte_array) {
1270       // Reinterpret the incoming vector to byte vector.
1271       const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1272       val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1273       // Reinterpret the vector mask to byte type.
1274       const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1275       const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1276       mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1277     }
1278     Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1279     set_memory(vstore, addr_type);
1280   } else {
1281     Node* vload = NULL;
1282 
1283     if (using_byte_array) {
1284       // Reinterpret the vector mask to byte type.
1285       const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1286       const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1287       mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1288     }
1289 
1290     if (use_predicate) {
1291       // Generate masked load vector node if predicate feature is supported.
1292       const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1293       vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1294     } else {
1295       // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1296       Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1297       zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, Type::get_const_basic_type(mem_elem_bt)));
1298       vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1299       vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1300     }
1301 
1302     if (using_byte_array) {
1303       const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1304       vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1305     }
1306 
1307     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1308     set_result(box);
1309   }
1310 
1311   old_map->destruct(&_gvn);
1312 
1313   if (can_access_non_heap) {
1314     insert_mem_bar(Op_MemBarCPUOrder);
1315   }
1316 
1317   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1318   return true;
1319 }
1320 
1321 // <C,
1322 //  V extends Vector<?>,
1323 //  W extends Vector<Integer>,
1324 //  S extends VectorSpecies<E>,
1325 //  M extends VectorMask<E>,
1326 //  E>
1327 // V loadWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int length,
1328 //               Class<? extends Vector<Integer>> vectorIndexClass,
1329 //               Object base, long offset, // Unsafe addressing
1330 //               W index_vector, M m,
1331 //               C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
1332 //               LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1333 //
1334 //  <C,
1335 //   V extends Vector<E>,
1336 //   W extends Vector<Integer>,
1337 //   M extends VectorMask<E>,
1338 //   E>
1339 //  void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1340 //                    int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset,    // Unsafe addressing
1341 //                    W index_vector, V v, M m,
1342 //                    C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1343 //                    StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1344 //
1345 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1346   const TypeInstPtr* vector_klass     = gvn().type(argument(0))->isa_instptr();
1347   const TypeInstPtr* mask_klass       = gvn().type(argument(1))->isa_instptr();
1348   const TypeInstPtr* elem_klass       = gvn().type(argument(2))->isa_instptr();
1349   const TypeInt*     vlen             = gvn().type(argument(3))->isa_int();
1350   const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1351 
1352   if (vector_klass == NULL || elem_klass == NULL || vector_idx_klass == NULL || vlen == NULL ||
1353       vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || vector_idx_klass->const_oop() == NULL || !vlen->is_con()) {
1354     if (C->print_intrinsics()) {
1355       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1356                     NodeClassNames[argument(0)->Opcode()],
1357                     NodeClassNames[argument(2)->Opcode()],
1358                     NodeClassNames[argument(3)->Opcode()],
1359                     NodeClassNames[argument(4)->Opcode()]);
1360     }
1361     return false; // not enough info for intrinsification
1362   }
1363 
1364   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1365     if (C->print_intrinsics()) {
1366       tty->print_cr("  ** klass argument not initialized");
1367     }
1368     return false;
1369   }
1370 
1371   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1372   if (!elem_type->is_primitive_type()) {
1373     if (C->print_intrinsics()) {
1374       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1375     }
1376     return false; // should be primitive type
1377   }
1378 
1379   BasicType elem_bt = elem_type->basic_type();
1380   int num_elem = vlen->get_con();
1381 
1382   const Type* vmask_type = gvn().type(is_scatter ? argument(10) : argument(9));
1383   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1384   if (is_masked_op) {
1385     if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
1386       if (C->print_intrinsics()) {
1387         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1388       }
1389       return false; // not enough info for intrinsification
1390     }
1391 
1392     if (!is_klass_initialized(mask_klass)) {
1393       if (C->print_intrinsics()) {
1394         tty->print_cr("  ** mask klass argument not initialized");
1395       }
1396       return false;
1397     }
1398 
1399     if (vmask_type->maybe_null()) {
1400       if (C->print_intrinsics()) {
1401         tty->print_cr("  ** null mask values are not allowed for masked op");
1402       }
1403       return false;
1404     }
1405 
1406     // Check whether the predicated gather/scatter node is supported by architecture.
1407     if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt,
1408                               (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred))) {
1409       if (C->print_intrinsics()) {
1410         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1411                       is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1412                       num_elem, type2name(elem_bt));
1413       }
1414       return false; // not supported
1415     }
1416   } else {
1417     // Check whether the normal gather/scatter node is supported for non-masked operation.
1418     if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1419       if (C->print_intrinsics()) {
1420         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1421                       is_scatter, is_scatter ? "scatter" : "gather",
1422                       num_elem, type2name(elem_bt));
1423       }
1424       return false; // not supported
1425     }
1426   }
1427 
1428   // Check that the vector holding indices is supported by architecture
1429   if (!arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
1430       if (C->print_intrinsics()) {
1431         tty->print_cr("  ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1432                       is_scatter, is_scatter ? "scatter" : "gather",
1433                       num_elem, is_masked_op ? 1 : 0);
1434       }
1435       return false; // not supported
1436   }
1437 
1438   Node* base = argument(5);
1439   Node* offset = ConvL2X(argument(6));
1440 
1441   // Save state and restore on bailout
1442   uint old_sp = sp();
1443   SafePointNode* old_map = clone_map();
1444 
1445   Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1446 
1447   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1448   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1449 
1450   // The array must be consistent with vector type
1451   if (arr_type == NULL || (arr_type != NULL && !elem_consistent_with_arr(elem_bt, arr_type))) {
1452     if (C->print_intrinsics()) {
1453       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1454                     is_scatter, is_scatter ? "scatter" : "gather",
1455                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1456     }
1457     set_map(old_map);
1458     set_sp(old_sp);
1459     return false;
1460   }
1461 
1462   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1463   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1464   ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1465   if (vbox_idx_klass == NULL) {
1466     set_map(old_map);
1467     set_sp(old_sp);
1468     return false;
1469   }
1470 
1471   const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1472   Node* index_vect = unbox_vector(argument(8), vbox_idx_type, T_INT, num_elem);
1473   if (index_vect == NULL) {
1474     set_map(old_map);
1475     set_sp(old_sp);
1476     return false;
1477   }
1478 
1479   Node* mask = NULL;
1480   if (is_masked_op) {
1481     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1482     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1483     mask = unbox_vector(is_scatter ? argument(10) : argument(9), mbox_type, elem_bt, num_elem);
1484     if (mask == NULL) {
1485       if (C->print_intrinsics()) {
1486         tty->print_cr("  ** unbox failed mask=%s",
1487                     is_scatter ? NodeClassNames[argument(10)->Opcode()]
1488                                : NodeClassNames[argument(9)->Opcode()]);
1489       }
1490       set_map(old_map);
1491       set_sp(old_sp);
1492       return false;
1493     }
1494   }
1495 
1496   const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1497   if (is_scatter) {
1498     Node* val = unbox_vector(argument(9), vbox_type, elem_bt, num_elem);
1499     if (val == NULL) {
1500       set_map(old_map);
1501       set_sp(old_sp);
1502       return false; // operand unboxing failed
1503     }
1504     set_all_memory(reset_memory());
1505 
1506     Node* vstore = NULL;
1507     if (mask != NULL) {
1508       vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, index_vect, mask));
1509     } else {
1510       vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1511     }
1512     set_memory(vstore, addr_type);
1513   } else {
1514     Node* vload = NULL;
1515     if (mask != NULL) {
1516       vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_vect, mask));
1517     } else {
1518       vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1519     }
1520     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1521     set_result(box);
1522   }
1523 
1524   old_map->destruct(&_gvn);
1525 
1526   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1527   return true;
1528 }
1529 
1530 // public static
1531 // <V extends Vector<E>,
1532 //  M extends VectorMask<E>,
1533 //  E>
1534 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1535 //                       Class<E> elementType, int length, V v, M m,
1536 //                       ReductionOperation<V, M> defaultImpl)
1537 bool LibraryCallKit::inline_vector_reduction() {
1538   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
1539   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1540   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
1541   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
1542   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
1543 
1544   if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
1545       !opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1546     if (C->print_intrinsics()) {
1547       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1548                     NodeClassNames[argument(0)->Opcode()],
1549                     NodeClassNames[argument(1)->Opcode()],
1550                     NodeClassNames[argument(3)->Opcode()],
1551                     NodeClassNames[argument(4)->Opcode()]);
1552     }
1553     return false; // not enough info for intrinsification
1554   }
1555   if (!is_klass_initialized(vector_klass)) {
1556     if (C->print_intrinsics()) {
1557       tty->print_cr("  ** klass argument not initialized");
1558     }
1559     return false;
1560   }
1561   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1562   if (!elem_type->is_primitive_type()) {
1563     if (C->print_intrinsics()) {
1564       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1565     }
1566     return false; // should be primitive type
1567   }
1568 
1569   const Type* vmask_type = gvn().type(argument(6));
1570   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1571   if (is_masked_op) {
1572     if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
1573       if (C->print_intrinsics()) {
1574         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1575       }
1576       return false; // not enough info for intrinsification
1577     }
1578 
1579     if (!is_klass_initialized(mask_klass)) {
1580       if (C->print_intrinsics()) {
1581         tty->print_cr("  ** mask klass argument not initialized");
1582       }
1583       return false;
1584     }
1585 
1586     if (vmask_type->maybe_null()) {
1587       if (C->print_intrinsics()) {
1588         tty->print_cr("  ** null mask values are not allowed for masked op");
1589       }
1590       return false;
1591     }
1592   }
1593 
1594   BasicType elem_bt = elem_type->basic_type();
1595   int num_elem = vlen->get_con();
1596   int opc  = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1597   int sopc = ReductionNode::opcode(opc, elem_bt);
1598 
1599   // When using mask, mask use type needs to be VecMaskUseLoad.
1600   if (!arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1601     if (C->print_intrinsics()) {
1602       tty->print_cr("  ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1603                     sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1604     }
1605     return false;
1606   }
1607 
1608   // Return true if current platform has implemented the masked operation with predicate feature.
1609   bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1610   if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1611     if (C->print_intrinsics()) {
1612       tty->print_cr("  ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1613                     sopc, num_elem, type2name(elem_bt));
1614     }
1615     return false;
1616   }
1617 
1618   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1619   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1620 
1621   Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1622   if (opd == NULL) {
1623     return false; // operand unboxing failed
1624   }
1625 
1626   Node* mask = NULL;
1627   if (is_masked_op) {
1628     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1629     assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1630     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1631     mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1632     if (mask == NULL) {
1633       if (C->print_intrinsics()) {
1634         tty->print_cr("  ** unbox failed mask=%s",
1635                       NodeClassNames[argument(6)->Opcode()]);
1636       }
1637       return false;
1638     }
1639   }
1640 
1641   Node* init = ReductionNode::make_reduction_input(gvn(), opc, elem_bt);
1642   Node* value = NULL;
1643   if (mask == NULL) {
1644     assert(!is_masked_op, "Masked op needs the mask value never null");
1645     value = ReductionNode::make(opc, NULL, init, opd, elem_bt);
1646   } else {
1647     if (use_predicate) {
1648       value = ReductionNode::make(opc, NULL, init, opd, elem_bt);
1649       value->add_req(mask);
1650       value->add_flag(Node::Flag_is_predicated_vector);
1651     } else {
1652       Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, Type::get_const_basic_type(elem_bt)));
1653       value = gvn().transform(new VectorBlendNode(reduce_identity, opd, mask));
1654       value = ReductionNode::make(opc, NULL, init, value, elem_bt);
1655     }
1656   }
1657   value = gvn().transform(value);
1658 
1659   Node* bits = NULL;
1660   switch (elem_bt) {
1661     case T_BYTE:
1662     case T_SHORT:
1663     case T_INT: {
1664       bits = gvn().transform(new ConvI2LNode(value));
1665       break;
1666     }
1667     case T_FLOAT: {
1668       value = gvn().transform(new MoveF2INode(value));
1669       bits  = gvn().transform(new ConvI2LNode(value));
1670       break;
1671     }
1672     case T_DOUBLE: {
1673       bits = gvn().transform(new MoveD2LNode(value));
1674       break;
1675     }
1676     case T_LONG: {
1677       bits = value; // no conversion needed
1678       break;
1679     }
1680     default: fatal("%s", type2name(elem_bt));
1681   }
1682   set_result(bits);
1683   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1684   return true;
1685 }
1686 
1687 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1688 //                                V v1, V v2,
1689 //                                BiFunction<V, V, Boolean> defaultImpl)
1690 //
1691 bool LibraryCallKit::inline_vector_test() {
1692   const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1693   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1694   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1695   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1696 
1697   if (cond == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
1698       !cond->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1699     if (C->print_intrinsics()) {
1700       tty->print_cr("  ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1701                     NodeClassNames[argument(0)->Opcode()],
1702                     NodeClassNames[argument(1)->Opcode()],
1703                     NodeClassNames[argument(2)->Opcode()],
1704                     NodeClassNames[argument(3)->Opcode()]);
1705     }
1706     return false; // not enough info for intrinsification
1707   }
1708   if (!is_klass_initialized(vector_klass)) {
1709     if (C->print_intrinsics()) {
1710       tty->print_cr("  ** klass argument not initialized");
1711     }
1712     return false;
1713   }
1714   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1715   if (!elem_type->is_primitive_type()) {
1716     if (C->print_intrinsics()) {
1717       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1718     }
1719     return false; // should be primitive type
1720   }
1721   BasicType elem_bt = elem_type->basic_type();
1722   int num_elem = vlen->get_con();
1723   BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1724   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1725   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1726 
1727   if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1728     if (C->print_intrinsics()) {
1729       tty->print_cr("  ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1730                     cond->get_con(), num_elem, type2name(elem_bt),
1731                     is_vector_mask(vbox_klass));
1732     }
1733     return false;
1734   }
1735 
1736   Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1737   Node* opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1738   if (opd1 == NULL || opd2 == NULL) {
1739     return false; // operand unboxing failed
1740   }
1741   Node* test = new VectorTestNode(opd1, opd2, booltest);
1742   test = gvn().transform(test);
1743 
1744   set_result(test);
1745   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1746   return true;
1747 }
1748 
1749 // public static
1750 // <V extends Vector<E>,
1751 //  M extends VectorMask<E>,
1752 //  E>
1753 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1754 //         V v1, V v2, M m,
1755 //         VectorBlendOp<V, M, E> defaultImpl)
1756 bool LibraryCallKit::inline_vector_blend() {
1757   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1758   const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1759   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1760   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1761 
1762   if (mask_klass == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL) {
1763     return false; // dead code
1764   }
1765   if (mask_klass->const_oop() == NULL || vector_klass->const_oop() == NULL ||
1766       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1767     if (C->print_intrinsics()) {
1768       tty->print_cr("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1769                     NodeClassNames[argument(0)->Opcode()],
1770                     NodeClassNames[argument(1)->Opcode()],
1771                     NodeClassNames[argument(2)->Opcode()],
1772                     NodeClassNames[argument(3)->Opcode()]);
1773     }
1774     return false; // not enough info for intrinsification
1775   }
1776   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1777     if (C->print_intrinsics()) {
1778       tty->print_cr("  ** klass argument not initialized");
1779     }
1780     return false;
1781   }
1782   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1783   if (!elem_type->is_primitive_type()) {
1784     if (C->print_intrinsics()) {
1785       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1786     }
1787     return false; // should be primitive type
1788   }
1789   BasicType elem_bt = elem_type->basic_type();
1790   BasicType mask_bt = elem_bt;
1791   int num_elem = vlen->get_con();
1792 
1793   if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1794     if (C->print_intrinsics()) {
1795       tty->print_cr("  ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1796                     num_elem, type2name(elem_bt));
1797     }
1798     return false; // not supported
1799   }
1800   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1801   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1802 
1803   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1804   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1805 
1806   Node* v1   = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1807   Node* v2   = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1808   Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1809 
1810   if (v1 == NULL || v2 == NULL || mask == NULL) {
1811     return false; // operand unboxing failed
1812   }
1813 
1814   Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1815 
1816   Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1817   set_result(box);
1818   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1819   return true;
1820 }
1821 
1822 //  public static
1823 //  <V extends Vector<E>,
1824 //   M extends VectorMask<E>,
1825 //   E>
1826 //  M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1827 //            V v1, V v2, M m,
1828 //            VectorCompareOp<V,M> defaultImpl)
1829 bool LibraryCallKit::inline_vector_compare() {
1830   const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1831   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1832   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
1833   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
1834   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
1835 
1836   if (cond == NULL || vector_klass == NULL || mask_klass == NULL || elem_klass == NULL || vlen == NULL) {
1837     return false; // dead code
1838   }
1839   if (!cond->is_con() || vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
1840       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1841     if (C->print_intrinsics()) {
1842       tty->print_cr("  ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1843                     NodeClassNames[argument(0)->Opcode()],
1844                     NodeClassNames[argument(1)->Opcode()],
1845                     NodeClassNames[argument(2)->Opcode()],
1846                     NodeClassNames[argument(3)->Opcode()],
1847                     NodeClassNames[argument(4)->Opcode()]);
1848     }
1849     return false; // not enough info for intrinsification
1850   }
1851   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1852     if (C->print_intrinsics()) {
1853       tty->print_cr("  ** klass argument not initialized");
1854     }
1855     return false;
1856   }
1857   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1858   if (!elem_type->is_primitive_type()) {
1859     if (C->print_intrinsics()) {
1860       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1861     }
1862     return false; // should be primitive type
1863   }
1864 
1865   int num_elem = vlen->get_con();
1866   BasicType elem_bt = elem_type->basic_type();
1867   BasicType mask_bt = elem_bt;
1868 
1869   if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1870     if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1871       if (C->print_intrinsics()) {
1872         tty->print_cr("  ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1873                       cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1874       }
1875       return false;
1876     }
1877   }
1878 
1879   if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1880     if (C->print_intrinsics()) {
1881       tty->print_cr("  ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1882                     cond->get_con(), num_elem, type2name(elem_bt));
1883     }
1884     return false;
1885   }
1886 
1887   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1888   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1889 
1890   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1891   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1892 
1893   Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1894   Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1895 
1896   bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1897   Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : NULL;
1898   if (is_masked_op && mask == NULL) {
1899     if (C->print_intrinsics()) {
1900       tty->print_cr("  ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1901                     cond->get_con(), num_elem, type2name(elem_bt));
1902     }
1903     return false;
1904   }
1905 
1906   bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1907   if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1908     if (C->print_intrinsics()) {
1909       tty->print_cr("  ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1910                     cond->get_con(), num_elem, type2name(elem_bt));
1911     }
1912     return false;
1913   }
1914 
1915   if (v1 == NULL || v2 == NULL) {
1916     return false; // operand unboxing failed
1917   }
1918   BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1919   ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1920 
1921   const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1922   Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1923 
1924   if (is_masked_op) {
1925     if (use_predicate) {
1926       operation->add_req(mask);
1927       operation->add_flag(Node::Flag_is_predicated_vector);
1928     } else {
1929       operation = gvn().transform(operation);
1930       operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1931     }
1932   }
1933 
1934   operation = gvn().transform(operation);
1935 
1936   Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1937   set_result(box);
1938   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1939   return true;
1940 }
1941 
1942 // public static
1943 // <V extends Vector<E>,
1944 //  Sh extends VectorShuffle<E>,
1945 //  M extends VectorMask<E>,
1946 //  E>
1947 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1948 //               V v1, Sh sh, M m,
1949 //               VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1950 bool LibraryCallKit::inline_vector_rearrange() {
1951   const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
1952   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1953   const TypeInstPtr* mask_klass    = gvn().type(argument(2))->isa_instptr();
1954   const TypeInstPtr* elem_klass    = gvn().type(argument(3))->isa_instptr();
1955   const TypeInt*     vlen          = gvn().type(argument(4))->isa_int();
1956 
1957   if (vector_klass == NULL  || shuffle_klass == NULL ||  elem_klass == NULL || vlen == NULL) {
1958     return false; // dead code
1959   }
1960   if (shuffle_klass->const_oop() == NULL ||
1961       vector_klass->const_oop()  == NULL ||
1962       elem_klass->const_oop()    == NULL ||
1963       !vlen->is_con()) {
1964     if (C->print_intrinsics()) {
1965       tty->print_cr("  ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1966                     NodeClassNames[argument(0)->Opcode()],
1967                     NodeClassNames[argument(1)->Opcode()],
1968                     NodeClassNames[argument(3)->Opcode()],
1969                     NodeClassNames[argument(4)->Opcode()]);
1970     }
1971     return false; // not enough info for intrinsification
1972   }
1973   if (!is_klass_initialized(vector_klass)  ||
1974       !is_klass_initialized(shuffle_klass)) {
1975     if (C->print_intrinsics()) {
1976       tty->print_cr("  ** klass argument not initialized");
1977     }
1978     return false;
1979   }
1980   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1981   if (!elem_type->is_primitive_type()) {
1982     if (C->print_intrinsics()) {
1983       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1984     }
1985     return false; // should be primitive type
1986   }
1987   BasicType elem_bt = elem_type->basic_type();
1988   BasicType shuffle_bt = elem_bt;
1989   int num_elem = vlen->get_con();
1990 
1991   if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
1992     if (C->print_intrinsics()) {
1993       tty->print_cr("  ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1994                     num_elem, type2name(elem_bt));
1995     }
1996     return false; // not supported
1997   }
1998 
1999   bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
2000   bool use_predicate = is_masked_op;
2001   if (is_masked_op &&
2002       (mask_klass == NULL ||
2003        mask_klass->const_oop() == NULL ||
2004        !is_klass_initialized(mask_klass))) {
2005     if (C->print_intrinsics()) {
2006       tty->print_cr("  ** mask_klass argument not initialized");
2007     }
2008   }
2009   VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2010   if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
2011     use_predicate = false;
2012     if(!is_masked_op ||
2013        (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2014         !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)     ||
2015         !arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed))) {
2016       if (C->print_intrinsics()) {
2017         tty->print_cr("  ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
2018                       num_elem, type2name(elem_bt));
2019       }
2020       return false; // not supported
2021     }
2022   }
2023   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2024   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2025 
2026   ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
2027   const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
2028 
2029   Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2030   Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
2031 
2032   if (v1 == NULL || shuffle == NULL) {
2033     return false; // operand unboxing failed
2034   }
2035 
2036   Node* mask = NULL;
2037   if (is_masked_op) {
2038     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2039     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2040     mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2041     if (mask == NULL) {
2042       if (C->print_intrinsics()) {
2043         tty->print_cr("  ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
2044                       num_elem, type2name(elem_bt));
2045       }
2046       return false;
2047     }
2048   }
2049 
2050   Node* rearrange = new VectorRearrangeNode(v1, shuffle);
2051   if (is_masked_op) {
2052     if (use_predicate) {
2053       rearrange->add_req(mask);
2054       rearrange->add_flag(Node::Flag_is_predicated_vector);
2055     } else {
2056       const TypeVect* vt = v1->bottom_type()->is_vect();
2057       rearrange = gvn().transform(rearrange);
2058       Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2059       Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, Type::get_const_basic_type(elem_bt)));
2060       rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2061     }
2062   }
2063   rearrange = gvn().transform(rearrange);
2064 
2065   Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2066   set_result(box);
2067   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2068   return true;
2069 }
2070 
2071 static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
2072   address addr = NULL;
2073   assert(UseVectorStubs, "sanity");
2074   assert(name_ptr != NULL, "unexpected");
2075   assert((vop >= VectorSupport::VECTOR_OP_SVML_START) && (vop <= VectorSupport::VECTOR_OP_SVML_END), "unexpected");
2076   int op = vop - VectorSupport::VECTOR_OP_SVML_START;
2077 
2078   switch(bits) {
2079     case 64:  //fallthough
2080     case 128: //fallthough
2081     case 256: //fallthough
2082     case 512:
2083       if (bt == T_FLOAT) {
2084         snprintf(name_ptr, name_len, "vector_%s_float%d", VectorSupport::svmlname[op], bits);
2085         addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op];
2086       } else {
2087         assert(bt == T_DOUBLE, "must be FP type only");
2088         snprintf(name_ptr, name_len, "vector_%s_double%d", VectorSupport::svmlname[op], bits);
2089         addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op];
2090       }
2091       break;
2092     default:
2093       snprintf(name_ptr, name_len, "invalid");
2094       addr = NULL;
2095       Unimplemented();
2096       break;
2097   }
2098 
2099   return addr;
2100 }
2101 
2102 Node* LibraryCallKit::gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
2103   assert(UseVectorStubs, "sanity");
2104   assert(vector_api_op_id >= VectorSupport::VECTOR_OP_SVML_START && vector_api_op_id <= VectorSupport::VECTOR_OP_SVML_END, "need valid op id");
2105   assert(opd1 != NULL, "must not be null");
2106   const TypeVect* vt = TypeVect::make(bt, num_elem);
2107   const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != NULL ? 2 : 1, vt, vt);
2108   char name[100] = "";
2109 
2110   // Get address for svml method.
2111   address addr = get_svml_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
2112 
2113   if (addr == NULL) {
2114     return NULL;
2115   }
2116 
2117   assert(name != NULL, "name must not be null");
2118   Node* operation = make_runtime_call(RC_VECTOR,
2119                                       call_type,
2120                                       addr,
2121                                       name,
2122                                       TypePtr::BOTTOM,
2123                                       opd1,
2124                                       opd2);
2125   return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
2126 }
2127 
2128 //  public static
2129 //  <V extends Vector<E>,
2130 //   M extends VectorMask<E>,
2131 //   E>
2132 //  V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2133 //                 Class<E> elementType, int length,
2134 //                 V v, int n, M m,
2135 //                 VectorBroadcastIntOp<V, M> defaultImpl)
2136 bool LibraryCallKit::inline_vector_broadcast_int() {
2137   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
2138   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2139   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
2140   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
2141   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
2142 
2143   if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL) {
2144     return false; // dead code
2145   }
2146   if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
2147     if (C->print_intrinsics()) {
2148       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2149                     NodeClassNames[argument(0)->Opcode()],
2150                     NodeClassNames[argument(1)->Opcode()],
2151                     NodeClassNames[argument(3)->Opcode()],
2152                     NodeClassNames[argument(4)->Opcode()]);
2153     }
2154     return false; // not enough info for intrinsification
2155   }
2156   if (!is_klass_initialized(vector_klass)) {
2157     if (C->print_intrinsics()) {
2158       tty->print_cr("  ** klass argument not initialized");
2159     }
2160     return false;
2161   }
2162 
2163   const Type* vmask_type = gvn().type(argument(7));
2164   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2165   if (is_masked_op) {
2166     if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
2167       if (C->print_intrinsics()) {
2168         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2169       }
2170       return false; // not enough info for intrinsification
2171     }
2172 
2173     if (!is_klass_initialized(mask_klass)) {
2174       if (C->print_intrinsics()) {
2175         tty->print_cr("  ** mask klass argument not initialized");
2176       }
2177       return false;
2178     }
2179 
2180     if (vmask_type->maybe_null()) {
2181       if (C->print_intrinsics()) {
2182         tty->print_cr("  ** null mask values are not allowed for masked op");
2183       }
2184       return false;
2185     }
2186   }
2187 
2188   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2189   if (!elem_type->is_primitive_type()) {
2190     if (C->print_intrinsics()) {
2191       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2192     }
2193     return false; // should be primitive type
2194   }
2195 
2196   int num_elem = vlen->get_con();
2197   BasicType elem_bt = elem_type->basic_type();
2198   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2199 
2200   bool is_shift  = VectorNode::is_shift_opcode(opc);
2201   bool is_rotate = VectorNode::is_rotate_opcode(opc);
2202 
2203   if (opc == 0 || (!is_shift && !is_rotate)) {
2204     if (C->print_intrinsics()) {
2205       tty->print_cr("  ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2206     }
2207     return false; // operation not supported
2208   }
2209 
2210   int sopc = VectorNode::opcode(opc, elem_bt);
2211   if (sopc == 0) {
2212     if (C->print_intrinsics()) {
2213       tty->print_cr("  ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2214     }
2215     return false; // operation not supported
2216   }
2217 
2218   Node* cnt  = argument(6);
2219   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2220   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2221   const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2222 
2223   // If CPU supports vector constant rotate instructions pass it directly
2224   bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2225                          Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2226   bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2227 
2228   VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2229   bool use_predicate = is_masked_op;
2230 
2231   if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2232     use_predicate = false;
2233     if (!is_masked_op ||
2234         (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2235          !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2236 
2237       if (C->print_intrinsics()) {
2238         tty->print_cr("  ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2239                       sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2240       }
2241       return false; // not supported
2242     }
2243   }
2244 
2245   Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2246   Node* opd2 = NULL;
2247   if (is_shift) {
2248     opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2249   } else {
2250     assert(is_rotate, "unexpected operation");
2251     if (!is_const_rotate) {
2252       const Type * type_bt = Type::get_const_basic_type(elem_bt);
2253       cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2254       opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, type_bt));
2255     } else {
2256       // Constant shift value.
2257       opd2 = cnt;
2258     }
2259   }
2260 
2261   if (opd1 == NULL || opd2 == NULL) {
2262     return false;
2263   }
2264 
2265   Node* mask = NULL;
2266   if (is_masked_op) {
2267     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2268     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2269     mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2270     if (mask == NULL) {
2271       if (C->print_intrinsics()) {
2272         tty->print_cr("  ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2273       }
2274       return false;
2275     }
2276   }
2277 
2278   Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2279   if (is_masked_op && mask != NULL) {
2280     if (use_predicate) {
2281       operation->add_req(mask);
2282       operation->add_flag(Node::Flag_is_predicated_vector);
2283     } else {
2284       operation = gvn().transform(operation);
2285       operation = new VectorBlendNode(opd1, operation, mask);
2286     }
2287   }
2288   operation = gvn().transform(operation);
2289   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2290   set_result(vbox);
2291   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2292   return true;
2293 }
2294 
2295 // public static <VOUT extends VectorPayload,
2296 //                 VIN extends VectorPayload,
2297 //                   S extends VectorSpecies>
2298 // VOUT convert(int oprId,
2299 //           Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2300 //           Class<?>   toVectorClass, Class<?>   toElementType, int   toVLen,
2301 //           VIN v, S s,
2302 //           VectorConvertOp<VOUT, VIN, S> defaultImpl)
2303 //
2304 bool LibraryCallKit::inline_vector_convert() {
2305   const TypeInt*     opr               = gvn().type(argument(0))->isa_int();
2306 
2307   const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2308   const TypeInstPtr* elem_klass_from   = gvn().type(argument(2))->isa_instptr();
2309   const TypeInt*     vlen_from         = gvn().type(argument(3))->isa_int();
2310 
2311   const TypeInstPtr* vector_klass_to   = gvn().type(argument(4))->isa_instptr();
2312   const TypeInstPtr* elem_klass_to     = gvn().type(argument(5))->isa_instptr();
2313   const TypeInt*     vlen_to           = gvn().type(argument(6))->isa_int();
2314 
2315   if (opr == NULL ||
2316       vector_klass_from == NULL || elem_klass_from == NULL || vlen_from == NULL ||
2317       vector_klass_to   == NULL || elem_klass_to   == NULL || vlen_to   == NULL) {
2318     return false; // dead code
2319   }
2320   if (!opr->is_con() ||
2321       vector_klass_from->const_oop() == NULL || elem_klass_from->const_oop() == NULL || !vlen_from->is_con() ||
2322       vector_klass_to->const_oop() == NULL || elem_klass_to->const_oop() == NULL || !vlen_to->is_con()) {
2323     if (C->print_intrinsics()) {
2324       tty->print_cr("  ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2325                     NodeClassNames[argument(0)->Opcode()],
2326                     NodeClassNames[argument(1)->Opcode()],
2327                     NodeClassNames[argument(2)->Opcode()],
2328                     NodeClassNames[argument(3)->Opcode()],
2329                     NodeClassNames[argument(4)->Opcode()],
2330                     NodeClassNames[argument(5)->Opcode()],
2331                     NodeClassNames[argument(6)->Opcode()]);
2332     }
2333     return false; // not enough info for intrinsification
2334   }
2335   if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2336     if (C->print_intrinsics()) {
2337       tty->print_cr("  ** klass argument not initialized");
2338     }
2339     return false;
2340   }
2341 
2342   assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
2343          opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2344   bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST);
2345 
2346   ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2347   ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2348   if (is_vector_shuffle(vbox_klass_from)) {
2349     return false; // vector shuffles aren't supported
2350   }
2351   bool is_mask = is_vector_mask(vbox_klass_from);
2352 
2353   ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2354   if (!elem_type_from->is_primitive_type()) {
2355     return false; // should be primitive type
2356   }
2357   BasicType elem_bt_from = elem_type_from->basic_type();
2358   ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2359   if (!elem_type_to->is_primitive_type()) {
2360     return false; // should be primitive type
2361   }
2362   BasicType elem_bt_to = elem_type_to->basic_type();
2363 
2364   int num_elem_from = vlen_from->get_con();
2365   int num_elem_to = vlen_to->get_con();
2366 
2367   // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2368   // since we may need to change size.
2369   if (!arch_supports_vector(Op_VectorReinterpret,
2370                             num_elem_from,
2371                             elem_bt_from,
2372                             is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2373     if (C->print_intrinsics()) {
2374       tty->print_cr("  ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2375                     is_cast ? "cast" : "reinterpret",
2376                     num_elem_from, type2name(elem_bt_from), is_mask);
2377     }
2378     return false;
2379   }
2380 
2381   // Check whether we can support resizing/reinterpreting to the new size.
2382   if (!arch_supports_vector(Op_VectorReinterpret,
2383                             num_elem_to,
2384                             elem_bt_to,
2385                             is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2386     if (C->print_intrinsics()) {
2387       tty->print_cr("  ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2388                     is_cast ? "cast" : "reinterpret",
2389                     num_elem_to, type2name(elem_bt_to), is_mask);
2390     }
2391     return false;
2392   }
2393 
2394   // At this point, we know that both input and output vector registers are supported
2395   // by the architecture. Next check if the casted type is simply to same type - which means
2396   // that it is actually a resize and not a cast.
2397   if (is_cast && elem_bt_from == elem_bt_to) {
2398     is_cast = false;
2399   }
2400 
2401   const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2402 
2403   Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2404   if (opd1 == NULL) {
2405     return false;
2406   }
2407 
2408   const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2409   const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2410 
2411   // Safety check to prevent casting if source mask is of type vector
2412   // and destination mask of type predicate vector and vice-versa.
2413   // From X86 standpoint, this case will only arise over KNL target,
2414   // where certain masks (depending on the species) are either propagated
2415   // through a vector or predicate register.
2416   if (is_mask &&
2417       ((src_type->isa_vectmask() == NULL && dst_type->isa_vectmask()) ||
2418        (dst_type->isa_vectmask() == NULL && src_type->isa_vectmask()))) {
2419     return false;
2420   }
2421 
2422   Node* op = opd1;
2423   if (is_cast) {
2424     BasicType new_elem_bt_to = elem_bt_to;
2425     BasicType new_elem_bt_from = elem_bt_from;
2426     if (is_mask && is_floating_point_type(elem_bt_from)) {
2427       new_elem_bt_from = elem_bt_from == T_FLOAT ? T_INT : T_LONG;
2428     }
2429     int cast_vopc = VectorCastNode::opcode(new_elem_bt_from);
2430     // Make sure that cast is implemented to particular type/size combination.
2431     if (!arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2432       if (C->print_intrinsics()) {
2433         tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2434                       cast_vopc,
2435                       num_elem_to, type2name(elem_bt_to), is_mask);
2436       }
2437       return false;
2438     }
2439 
2440     if (num_elem_from < num_elem_to) {
2441       // Since input and output number of elements are not consistent, we need to make sure we
2442       // properly size. Thus, first make a cast that retains the number of elements from source.
2443       int num_elem_for_cast = num_elem_from;
2444 
2445       // It is possible that arch does not support this intermediate vector size
2446       // TODO More complex logic required here to handle this corner case for the sizes.
2447       if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2448         if (C->print_intrinsics()) {
2449           tty->print_cr("  ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2450                         cast_vopc,
2451                         num_elem_for_cast, type2name(elem_bt_to), is_mask);
2452         }
2453         return false;
2454       }
2455 
2456       op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2457       // Now ensure that the destination gets properly resized to needed size.
2458       op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2459     } else if (num_elem_from > num_elem_to) {
2460       // Since number elements from input is larger than output, simply reduce size of input (we are supposed to
2461       // drop top elements anyway).
2462       int num_elem_for_resize = num_elem_to;
2463 
2464       // It is possible that arch does not support this intermediate vector size
2465       // TODO More complex logic required here to handle this corner case for the sizes.
2466       if (!arch_supports_vector(Op_VectorReinterpret,
2467                                 num_elem_for_resize,
2468                                 elem_bt_from,
2469                                 VecMaskNotUsed)) {
2470         if (C->print_intrinsics()) {
2471           tty->print_cr("  ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2472                         num_elem_for_resize, type2name(elem_bt_from), is_mask);
2473         }
2474         return false;
2475       }
2476 
2477       op = gvn().transform(new VectorReinterpretNode(op,
2478                                                      src_type,
2479                                                      TypeVect::make(elem_bt_from,
2480                                                                     num_elem_for_resize)));
2481       op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2482     } else {
2483       if (is_mask) {
2484         if ((dst_type->isa_vectmask() && src_type->isa_vectmask()) ||
2485             (type2aelembytes(elem_bt_from) == type2aelembytes(elem_bt_to))) {
2486           op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2487         } else {
2488           // Special handling for casting operation involving floating point types.
2489           // Case A) F -> X :=  F -> VectorMaskCast (F->I/L [NOP]) -> VectorCast[I/L]2X
2490           // Case B) X -> F :=  X -> VectorCastX2[I/L] -> VectorMaskCast ([I/L]->F [NOP])
2491           // Case C) F -> F :=  VectorMaskCast (F->I/L [NOP]) -> VectorCast[I/L]2[L/I] -> VectotMaskCast (L/I->F [NOP])
2492           if (is_floating_point_type(elem_bt_from)) {
2493             const TypeVect* new_src_type = TypeVect::make(new_elem_bt_from, num_elem_to, is_mask);
2494             op = gvn().transform(new VectorMaskCastNode(op, new_src_type));
2495           }
2496           if (is_floating_point_type(elem_bt_to)) {
2497             new_elem_bt_to = elem_bt_to == T_FLOAT ? T_INT : T_LONG;
2498           }
2499           op = gvn().transform(VectorCastNode::make(cast_vopc, op, new_elem_bt_to, num_elem_to));
2500           if (new_elem_bt_to != elem_bt_to) {
2501             op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2502           }
2503         }
2504       } else {
2505         // Since input and output number of elements match, and since we know this vector size is
2506         // supported, simply do a cast with no resize needed.
2507         op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2508       }
2509     }
2510   } else if (Type::cmp(src_type, dst_type) != 0) {
2511     assert(!is_cast, "must be reinterpret");
2512     op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2513   }
2514 
2515   const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2516   Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2517   set_result(vbox);
2518   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2519   return true;
2520 }
2521 
2522 //  public static
2523 //  <V extends Vector<E>,
2524 //   E>
2525 //  V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2526 //           V vec, int ix, long val,
2527 //           VecInsertOp<V> defaultImpl)
2528 bool LibraryCallKit::inline_vector_insert() {
2529   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2530   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2531   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2532   const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
2533 
2534   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || idx == NULL) {
2535     return false; // dead code
2536   }
2537   if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
2538     if (C->print_intrinsics()) {
2539       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2540                     NodeClassNames[argument(0)->Opcode()],
2541                     NodeClassNames[argument(1)->Opcode()],
2542                     NodeClassNames[argument(2)->Opcode()],
2543                     NodeClassNames[argument(4)->Opcode()]);
2544     }
2545     return false; // not enough info for intrinsification
2546   }
2547   if (!is_klass_initialized(vector_klass)) {
2548     if (C->print_intrinsics()) {
2549       tty->print_cr("  ** klass argument not initialized");
2550     }
2551     return false;
2552   }
2553   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2554   if (!elem_type->is_primitive_type()) {
2555     if (C->print_intrinsics()) {
2556       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2557     }
2558     return false; // should be primitive type
2559   }
2560   BasicType elem_bt = elem_type->basic_type();
2561   int num_elem = vlen->get_con();
2562   if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2563     if (C->print_intrinsics()) {
2564       tty->print_cr("  ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2565                     num_elem, type2name(elem_bt));
2566     }
2567     return false; // not supported
2568   }
2569 
2570   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2571   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2572 
2573   Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2574   if (opd == NULL) {
2575     return false;
2576   }
2577 
2578   Node* insert_val = argument(5);
2579   assert(gvn().type(insert_val)->isa_long() != NULL, "expected to be long");
2580 
2581   // Convert insert value back to its appropriate type.
2582   switch (elem_bt) {
2583     case T_BYTE:
2584       insert_val = gvn().transform(new ConvL2INode(insert_val));
2585       insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::BYTE));
2586       break;
2587     case T_SHORT:
2588       insert_val = gvn().transform(new ConvL2INode(insert_val));
2589       insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::SHORT));
2590       break;
2591     case T_INT:
2592       insert_val = gvn().transform(new ConvL2INode(insert_val));
2593       break;
2594     case T_FLOAT:
2595       insert_val = gvn().transform(new ConvL2INode(insert_val));
2596       insert_val = gvn().transform(new MoveI2FNode(insert_val));
2597       break;
2598     case T_DOUBLE:
2599       insert_val = gvn().transform(new MoveL2DNode(insert_val));
2600       break;
2601     case T_LONG:
2602       // no conversion needed
2603       break;
2604     default: fatal("%s", type2name(elem_bt)); break;
2605   }
2606 
2607   Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con()));
2608 
2609   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2610   set_result(vbox);
2611   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2612   return true;
2613 }
2614 
2615 //  public static
2616 //  <V extends Vector<E>,
2617 //   E>
2618 //  long extract(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2619 //               V vec, int ix,
2620 //               VecExtractOp<V> defaultImpl)
2621 bool LibraryCallKit::inline_vector_extract() {
2622   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2623   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2624   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2625   const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
2626 
2627   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || idx == NULL) {
2628     return false; // dead code
2629   }
2630   if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
2631     if (C->print_intrinsics()) {
2632       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2633                     NodeClassNames[argument(0)->Opcode()],
2634                     NodeClassNames[argument(1)->Opcode()],
2635                     NodeClassNames[argument(2)->Opcode()],
2636                     NodeClassNames[argument(4)->Opcode()]);
2637     }
2638     return false; // not enough info for intrinsification
2639   }
2640   if (!is_klass_initialized(vector_klass)) {
2641     if (C->print_intrinsics()) {
2642       tty->print_cr("  ** klass argument not initialized");
2643     }
2644     return false;
2645   }
2646   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2647   if (!elem_type->is_primitive_type()) {
2648     if (C->print_intrinsics()) {
2649       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2650     }
2651     return false; // should be primitive type
2652   }
2653   BasicType elem_bt = elem_type->basic_type();
2654   int num_elem = vlen->get_con();
2655   int vopc = ExtractNode::opcode(elem_bt);
2656   if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2657     if (C->print_intrinsics()) {
2658       tty->print_cr("  ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2659                     num_elem, type2name(elem_bt));
2660     }
2661     return false; // not supported
2662   }
2663 
2664   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2665   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2666 
2667   Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2668   if (opd == NULL) {
2669     return false;
2670   }
2671 
2672   Node* operation = gvn().transform(ExtractNode::make(opd, idx->get_con(), elem_bt));
2673 
2674   Node* bits = NULL;
2675   switch (elem_bt) {
2676     case T_BYTE:
2677     case T_SHORT:
2678     case T_INT: {
2679       bits = gvn().transform(new ConvI2LNode(operation));
2680       break;
2681     }
2682     case T_FLOAT: {
2683       bits = gvn().transform(new MoveF2INode(operation));
2684       bits = gvn().transform(new ConvI2LNode(bits));
2685       break;
2686     }
2687     case T_DOUBLE: {
2688       bits = gvn().transform(new MoveD2LNode(operation));
2689       break;
2690     }
2691     case T_LONG: {
2692       bits = operation; // no conversion needed
2693       break;
2694     }
2695     default: fatal("%s", type2name(elem_bt));
2696   }
2697 
2698   set_result(bits);
2699   return true;
2700 }
2701