1 /*
   2  * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "ci/ciSymbols.hpp"
  27 #include "classfile/vmSymbols.hpp"
  28 #include "opto/library_call.hpp"
  29 #include "opto/runtime.hpp"
  30 #include "opto/vectornode.hpp"
  31 #include "prims/vectorSupport.hpp"
  32 #include "runtime/stubRoutines.hpp"
  33 
  34 #ifdef ASSERT
  35 static bool is_vector(ciKlass* klass) {
  36   return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
  37 }
  38 
  39 static bool check_vbox(const TypeInstPtr* vbox_type) {
  40   assert(vbox_type->klass_is_exact(), "");
  41 
  42   ciInstanceKlass* ik = vbox_type->klass()->as_instance_klass();
  43   assert(is_vector(ik), "not a vector");
  44 
  45   ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
  46   assert(fd1 != NULL, "element type info is missing");
  47 
  48   ciConstant val1 = fd1->constant_value();
  49   BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
  50   assert(is_java_primitive(elem_bt), "element type info is missing");
  51 
  52   ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
  53   assert(fd2 != NULL, "vector length info is missing");
  54 
  55   ciConstant val2 = fd2->constant_value();
  56   assert(val2.as_int() > 0, "vector length info is missing");
  57 
  58   return true;
  59 }
  60 #endif
  61 
  62 static bool is_vector_mask(ciKlass* klass) {
  63   return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
  64 }
  65 
  66 static bool is_vector_shuffle(ciKlass* klass) {
  67   return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
  68 }
  69 
  70 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
  71                                                  VectorMaskUseType mask_use_type, bool has_scalar_args) {
  72   bool is_supported = true;
  73 
  74   // has_scalar_args flag is true only for non-constant scalar shift count,
  75   // since in this case shift needs to be broadcasted.
  76   if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
  77        (has_scalar_args &&
  78          !arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed))) {
  79     is_supported = false;
  80   }
  81 
  82   if (is_supported) {
  83     // Check whether mask unboxing is supported.
  84     if ((mask_use_type & VecMaskUseLoad) != 0) {
  85       if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt)) {
  86       #ifndef PRODUCT
  87         if (C->print_intrinsics()) {
  88           tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
  89                         NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
  90         }
  91       #endif
  92         return false;
  93       }
  94     }
  95 
  96     if ((mask_use_type & VecMaskUsePred) != 0) {
  97       if (!Matcher::has_predicated_vectors() ||
  98           !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
  99       #ifndef PRODUCT
 100         if (C->print_intrinsics()) {
 101           tty->print_cr("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
 102                         NodeClassNames[opc], type2name(elem_bt), num_elem);
 103         }
 104       #endif
 105         return false;
 106       }
 107     }
 108   }
 109 
 110   int lshiftopc, rshiftopc;
 111   switch(elem_bt) {
 112     case T_BYTE:
 113       lshiftopc = Op_LShiftI;
 114       rshiftopc = Op_URShiftB;
 115       break;
 116     case T_SHORT:
 117       lshiftopc = Op_LShiftI;
 118       rshiftopc = Op_URShiftS;
 119       break;
 120     case T_INT:
 121       lshiftopc = Op_LShiftI;
 122       rshiftopc = Op_URShiftI;
 123       break;
 124     case T_LONG:
 125       lshiftopc = Op_LShiftL;
 126       rshiftopc = Op_URShiftL;
 127       break;
 128     default:
 129       assert(false, "Unexpected type");
 130   }
 131   int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
 132   int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
 133   if (!is_supported &&
 134       arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
 135       arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
 136       arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
 137     is_supported = true;
 138   }
 139   return is_supported;
 140 }
 141 
 142 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
 143   assert(EnableVectorSupport, "");
 144 
 145   PreserveReexecuteState preexecs(this);
 146   jvms()->set_should_reexecute(true);
 147 
 148   VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
 149   set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
 150   make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
 151   set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
 152   set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
 153   Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
 154 
 155   assert(check_vbox(vbox_type), "");
 156   const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->klass()));
 157   VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
 158   return gvn().transform(vbox);
 159 }
 160 
 161 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) {
 162   assert(EnableVectorSupport, "");
 163   const TypeInstPtr* vbox_type_v = gvn().type(v)->is_instptr();
 164   if (vbox_type->klass() != vbox_type_v->klass()) {
 165     return NULL; // arguments don't agree on vector shapes
 166   }
 167   if (vbox_type_v->maybe_null()) {
 168     return NULL; // no nulls are allowed
 169   }
 170   assert(check_vbox(vbox_type), "");
 171   const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->klass()));
 172   Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector));
 173   return unbox;
 174 }
 175 
 176 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
 177   assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
 178   juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
 179   Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
 180   Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
 181   return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
 182 }
 183 
 184 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
 185   // Check that the operation is valid.
 186   if (sopc <= 0) {
 187 #ifndef PRODUCT
 188     if (C->print_intrinsics()) {
 189       tty->print_cr("  ** Rejected intrinsification because no valid vector op could be extracted");
 190     }
 191 #endif
 192     return false;
 193   }
 194 
 195   if (VectorNode::is_vector_rotate(sopc)) {
 196     if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
 197 #ifndef PRODUCT
 198       if (C->print_intrinsics()) {
 199         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
 200                       NodeClassNames[sopc], type2name(type), num_elem);
 201       }
 202 #endif
 203       return false;
 204     }
 205   } else {
 206     // Check that architecture supports this op-size-type combination.
 207     if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
 208 #ifndef PRODUCT
 209       if (C->print_intrinsics()) {
 210         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support it",
 211                       NodeClassNames[sopc], type2name(type), num_elem);
 212       }
 213 #endif
 214       return false;
 215     } else {
 216       assert(Matcher::match_rule_supported(sopc), "must be supported");
 217     }
 218   }
 219 
 220   if (num_elem == 1) {
 221     if (mask_use_type != VecMaskNotUsed) {
 222 #ifndef PRODUCT
 223       if (C->print_intrinsics()) {
 224         tty->print_cr("  ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
 225                       NodeClassNames[sopc], type2name(type), num_elem);
 226       }
 227 #endif
 228       return false;
 229     }
 230 
 231     if (sopc != 0) {
 232       if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
 233 #ifndef PRODUCT
 234         if (C->print_intrinsics()) {
 235           tty->print_cr("  ** Not a svml call or load/store vector op (%s,%s,%d)",
 236                         NodeClassNames[sopc], type2name(type), num_elem);
 237         }
 238 #endif
 239         return false;
 240       }
 241     }
 242   }
 243 
 244   if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
 245       Matcher::supports_vector_variable_shifts() == false) {
 246     if (C->print_intrinsics()) {
 247       tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
 248                     NodeClassNames[sopc], type2name(type), num_elem);
 249     }
 250     return false;
 251   }
 252 
 253   // Check whether mask unboxing is supported.
 254   if ((mask_use_type & VecMaskUseLoad) != 0) {
 255     if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type)) {
 256     #ifndef PRODUCT
 257       if (C->print_intrinsics()) {
 258         tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
 259                       NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
 260       }
 261     #endif
 262       return false;
 263     }
 264   }
 265 
 266   // Check whether mask boxing is supported.
 267   if ((mask_use_type & VecMaskUseStore) != 0) {
 268     if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type)) {
 269     #ifndef PRODUCT
 270       if (C->print_intrinsics()) {
 271         tty->print_cr("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
 272                       NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
 273       }
 274     #endif
 275       return false;
 276     }
 277   }
 278 
 279   if ((mask_use_type & VecMaskUsePred) != 0) {
 280     if (!Matcher::has_predicated_vectors() ||
 281         !Matcher::match_rule_supported_vector_masked(sopc, num_elem, type)) {
 282     #ifndef PRODUCT
 283       if (C->print_intrinsics()) {
 284         tty->print_cr("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
 285                       NodeClassNames[sopc], type2name(type), num_elem);
 286       }
 287     #endif
 288       return false;
 289     }
 290   }
 291 
 292   return true;
 293 }
 294 
 295 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
 296   if (vec_klass->const_oop() == NULL) {
 297     return false; // uninitialized or some kind of unsafe access
 298   }
 299   assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != NULL, "klass instance expected");
 300   ciInstanceKlass* klass =  vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
 301   return klass->is_initialized();
 302 }
 303 
 304 // public static
 305 // <V extends Vector<E>,
 306 //  M extends VectorMask<E>,
 307 //  E>
 308 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 309 //           int length, V v, M m,
 310 //           UnaryOperation<V, M> defaultImpl)
 311 //
 312 // public static
 313 // <V,
 314 //  M extends VectorMask<E>,
 315 //  E>
 316 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 317 //            int length, V v1, V v2, M m,
 318 //            BinaryOperation<V, M> defaultImpl)
 319 //
 320 // public static
 321 // <V extends Vector<E>,
 322 //  M extends VectorMask<E>,
 323 //  E>
 324 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 325 //             int length, V v1, V v2, V v3, M m,
 326 //             TernaryOperation<V, M> defaultImpl)
 327 //
 328 bool LibraryCallKit::inline_vector_nary_operation(int n) {
 329   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
 330   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
 331   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
 332   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
 333   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
 334 
 335   if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
 336       !opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 337     if (C->print_intrinsics()) {
 338       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
 339                     NodeClassNames[argument(0)->Opcode()],
 340                     NodeClassNames[argument(1)->Opcode()],
 341                     NodeClassNames[argument(3)->Opcode()],
 342                     NodeClassNames[argument(4)->Opcode()]);
 343     }
 344     return false; // not enough info for intrinsification
 345   }
 346 
 347   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 348   if (!elem_type->is_primitive_type()) {
 349     if (C->print_intrinsics()) {
 350       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 351     }
 352     return false; // should be primitive type
 353   }
 354   if (!is_klass_initialized(vector_klass)) {
 355     if (C->print_intrinsics()) {
 356       tty->print_cr("  ** klass argument not initialized");
 357     }
 358     return false;
 359   }
 360 
 361   // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
 362   // is used to control this operation.
 363   const Type* vmask_type = gvn().type(argument(n + 5));
 364   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
 365   if (is_masked_op) {
 366     if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
 367       if (C->print_intrinsics()) {
 368         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
 369       }
 370       return false; // not enough info for intrinsification
 371     }
 372 
 373     if (!is_klass_initialized(mask_klass)) {
 374       if (C->print_intrinsics()) {
 375         tty->print_cr("  ** mask klass argument not initialized");
 376       }
 377       return false;
 378     }
 379 
 380     if (vmask_type->maybe_null()) {
 381       if (C->print_intrinsics()) {
 382         tty->print_cr("  ** null mask values are not allowed for masked op");
 383       }
 384       return false;
 385     }
 386   }
 387 
 388   BasicType elem_bt = elem_type->basic_type();
 389   int num_elem = vlen->get_con();
 390   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
 391   int sopc = VectorNode::opcode(opc, elem_bt);
 392   if ((opc != Op_CallLeafVector) && (sopc == 0)) {
 393     if (C->print_intrinsics()) {
 394       tty->print_cr("  ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
 395     }
 396     return false; // operation not supported
 397   }
 398   if (num_elem == 1) {
 399     if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
 400       if (C->print_intrinsics()) {
 401         tty->print_cr("  ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
 402                       n, opc, num_elem, type2name(elem_bt));
 403       }
 404       return false;
 405     }
 406   }
 407   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 408   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 409 
 410   if (is_vector_mask(vbox_klass)) {
 411     assert(!is_masked_op, "mask operations do not need mask to control");
 412   }
 413 
 414   if (opc == Op_CallLeafVector) {
 415     if (!UseVectorStubs) {
 416       if (C->print_intrinsics()) {
 417         tty->print_cr("  ** vector stubs support is disabled");
 418       }
 419       return false;
 420     }
 421     if (!Matcher::supports_vector_calling_convention()) {
 422       if (C->print_intrinsics()) {
 423         tty->print_cr("  ** no vector calling conventions supported");
 424       }
 425       return false;
 426     }
 427     if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
 428       if (C->print_intrinsics()) {
 429         tty->print_cr("  ** vector size (vlen=%d, etype=%s) is not supported",
 430                       num_elem, type2name(elem_bt));
 431       }
 432       return false;
 433     }
 434   }
 435 
 436   // When using mask, mask use type needs to be VecMaskUseLoad.
 437   VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
 438                                       : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
 439   if ((sopc != 0) && !arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
 440     if (C->print_intrinsics()) {
 441       tty->print_cr("  ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
 442                     n, sopc, num_elem, type2name(elem_bt),
 443                     is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
 444     }
 445     return false; // not supported
 446   }
 447 
 448   // Return true if current platform has implemented the masked operation with predicate feature.
 449   bool use_predicate = is_masked_op && sopc != 0 && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
 450   if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
 451     if (C->print_intrinsics()) {
 452       tty->print_cr("  ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
 453                     n, sopc, num_elem, type2name(elem_bt));
 454     }
 455     return false;
 456   }
 457 
 458   Node* opd1 = NULL; Node* opd2 = NULL; Node* opd3 = NULL;
 459   switch (n) {
 460     case 3: {
 461       opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
 462       if (opd3 == NULL) {
 463         if (C->print_intrinsics()) {
 464           tty->print_cr("  ** unbox failed v3=%s",
 465                         NodeClassNames[argument(7)->Opcode()]);
 466         }
 467         return false;
 468       }
 469       // fall-through
 470     }
 471     case 2: {
 472       opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
 473       if (opd2 == NULL) {
 474         if (C->print_intrinsics()) {
 475           tty->print_cr("  ** unbox failed v2=%s",
 476                         NodeClassNames[argument(6)->Opcode()]);
 477         }
 478         return false;
 479       }
 480       // fall-through
 481     }
 482     case 1: {
 483       opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
 484       if (opd1 == NULL) {
 485         if (C->print_intrinsics()) {
 486           tty->print_cr("  ** unbox failed v1=%s",
 487                         NodeClassNames[argument(5)->Opcode()]);
 488         }
 489         return false;
 490       }
 491       break;
 492     }
 493     default: fatal("unsupported arity: %d", n);
 494   }
 495 
 496   Node* mask = NULL;
 497   if (is_masked_op) {
 498     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
 499     assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
 500     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
 501     mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
 502     if (mask == NULL) {
 503       if (C->print_intrinsics()) {
 504         tty->print_cr("  ** unbox failed mask=%s",
 505                       NodeClassNames[argument(n + 5)->Opcode()]);
 506       }
 507       return false;
 508     }
 509   }
 510 
 511   Node* operation = NULL;
 512   if (opc == Op_CallLeafVector) {
 513     assert(UseVectorStubs, "sanity");
 514     operation = gen_call_to_svml(opr->get_con(), elem_bt, num_elem, opd1, opd2);
 515     if (operation == NULL) {
 516       if (C->print_intrinsics()) {
 517         tty->print_cr("  ** svml call failed for %s_%s_%d",
 518                          (elem_bt == T_FLOAT)?"float":"double",
 519                          VectorSupport::svmlname[opr->get_con() - VectorSupport::VECTOR_OP_SVML_START],
 520                          num_elem * type2aelembytes(elem_bt));
 521       }
 522       return false;
 523      }
 524   } else {
 525     const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
 526     switch (n) {
 527       case 1:
 528       case 2: {
 529         operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc));
 530         break;
 531       }
 532       case 3: {
 533         operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
 534         break;
 535       }
 536       default: fatal("unsupported arity: %d", n);
 537     }
 538   }
 539 
 540   if (is_masked_op && mask != NULL) {
 541     if (use_predicate) {
 542       operation->add_req(mask);
 543       operation->add_flag(Node::Flag_is_predicated_vector);
 544     } else {
 545       operation = gvn().transform(operation);
 546       operation = new VectorBlendNode(opd1, operation, mask);
 547     }
 548   }
 549   operation = gvn().transform(operation);
 550 
 551   // Wrap it up in VectorBox to keep object type information.
 552   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
 553   set_result(vbox);
 554   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 555   return true;
 556 }
 557 
 558 // <Sh extends VectorShuffle<E>,  E>
 559 //  Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
 560 //                  int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
 561 bool LibraryCallKit::inline_vector_shuffle_iota() {
 562   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
 563   const TypeInt*     vlen          = gvn().type(argument(3))->isa_int();
 564   const TypeInt*     start_val     = gvn().type(argument(4))->isa_int();
 565   const TypeInt*     step_val      = gvn().type(argument(5))->isa_int();
 566   const TypeInt*     wrap          = gvn().type(argument(6))->isa_int();
 567 
 568   Node* start = argument(4);
 569   Node* step  = argument(5);
 570 
 571   if (shuffle_klass == NULL || vlen == NULL || start_val == NULL || step_val == NULL || wrap == NULL) {
 572     return false; // dead code
 573   }
 574   if (!vlen->is_con() || !is_power_of_2(vlen->get_con()) ||
 575       shuffle_klass->const_oop() == NULL || !wrap->is_con()) {
 576     return false; // not enough info for intrinsification
 577   }
 578   if (!is_klass_initialized(shuffle_klass)) {
 579     if (C->print_intrinsics()) {
 580       tty->print_cr("  ** klass argument not initialized");
 581     }
 582     return false;
 583   }
 584 
 585   int do_wrap = wrap->get_con();
 586   int num_elem = vlen->get_con();
 587   BasicType elem_bt = T_BYTE;
 588 
 589   if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed)) {
 590     return false;
 591   }
 592   if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed)) {
 593     return false;
 594   }
 595   if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) {
 596     return false;
 597   }
 598   if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
 599     return false;
 600   }
 601   if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
 602     return false;
 603   }
 604   if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
 605     return false;
 606   }
 607 
 608   const Type * type_bt = Type::get_const_basic_type(elem_bt);
 609   const TypeVect * vt  = TypeVect::make(type_bt, num_elem);
 610 
 611   Node* res =  gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
 612 
 613   if(!step_val->is_con() || !is_power_of_2(step_val->get_con())) {
 614     Node* bcast_step     = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt));
 615     res = gvn().transform(VectorNode::make(Op_MulI, res, bcast_step, num_elem, elem_bt));
 616   } else if (step_val->get_con() > 1) {
 617     Node* cnt = gvn().makecon(TypeInt::make(log2i_exact(step_val->get_con())));
 618     Node* shift_cnt = vector_shift_count(cnt, Op_LShiftI, elem_bt, num_elem);
 619     res = gvn().transform(VectorNode::make(Op_LShiftVB, res, shift_cnt, vt));
 620   }
 621 
 622   if (!start_val->is_con() || start_val->get_con() != 0) {
 623     Node* bcast_start    = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt));
 624     res = gvn().transform(VectorNode::make(Op_AddI, res, bcast_start, num_elem, elem_bt));
 625   }
 626 
 627   Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1));
 628   Node * bcast_mod  = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt));
 629   if(do_wrap)  {
 630     // Wrap the indices greater than lane count.
 631     res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
 632   } else {
 633     ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::ge));
 634     Node * lane_cnt  = gvn().makecon(TypeInt::make(num_elem));
 635     Node * bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt));
 636     const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
 637     Node* mask = gvn().transform(new VectorMaskCmpNode(BoolTest::ge, bcast_lane_cnt, res, pred_node, vmask_type));
 638 
 639     // Make the indices greater than lane count as -ve values. This matches the java side implementation.
 640     res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
 641     Node * biased_val = gvn().transform(VectorNode::make(Op_SubI, res, bcast_lane_cnt, num_elem, elem_bt));
 642     res = gvn().transform(new VectorBlendNode(biased_val, res, mask));
 643   }
 644 
 645   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 646   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 647 
 648   // Wrap it up in VectorBox to keep object type information.
 649   res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
 650   set_result(res);
 651   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 652   return true;
 653 }
 654 
 655 // <E, M>
 656 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
 657 //                          int length, M m, VectorMaskOp<M> defaultImpl)
 658 bool LibraryCallKit::inline_vector_mask_operation() {
 659   const TypeInt*     oper       = gvn().type(argument(0))->isa_int();
 660   const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
 661   const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
 662   const TypeInt*     vlen       = gvn().type(argument(3))->isa_int();
 663   Node*              mask       = argument(4);
 664 
 665   if (mask_klass == NULL || elem_klass == NULL || mask->is_top() || vlen == NULL) {
 666     return false; // dead code
 667   }
 668 
 669   if (!is_klass_initialized(mask_klass)) {
 670     if (C->print_intrinsics()) {
 671       tty->print_cr("  ** klass argument not initialized");
 672     }
 673     return false;
 674   }
 675 
 676   int num_elem = vlen->get_con();
 677   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 678   BasicType elem_bt = elem_type->basic_type();
 679 
 680   if (!arch_supports_vector(Op_LoadVector, num_elem, T_BOOLEAN, VecMaskNotUsed)) {
 681     if (C->print_intrinsics()) {
 682       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 683                     Op_LoadVector, num_elem, type2name(T_BOOLEAN));
 684     }
 685     return false; // not supported
 686   }
 687 
 688   int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
 689   if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskNotUsed)) {
 690     if (C->print_intrinsics()) {
 691       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 692                     mopc, num_elem, type2name(elem_bt));
 693     }
 694     return false; // not supported
 695   }
 696 
 697   const Type* elem_ty = Type::get_const_basic_type(elem_bt);
 698   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
 699   const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
 700   Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
 701   if (mask_vec == NULL) {
 702     if (C->print_intrinsics()) {
 703         tty->print_cr("  ** unbox failed mask=%s",
 704                       NodeClassNames[argument(4)->Opcode()]);
 705     }
 706     return false;
 707   }
 708 
 709   if (mask_vec->bottom_type()->isa_vectmask() == NULL) {
 710     mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
 711   }
 712   const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
 713   Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
 714   if (mopc != Op_VectorMaskToLong) {
 715     maskoper = ConvI2L(maskoper);
 716   }
 717   set_result(maskoper);
 718 
 719   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 720   return true;
 721 }
 722 
 723 // public static
 724 // <V,
 725 //  Sh extends VectorShuffle<E>,
 726 //  E>
 727 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
 728 //                   Class<? extends Sh> shuffleClass, Sh s, int length,
 729 //                   ShuffleToVectorOperation<V, Sh, E> defaultImpl)
 730 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
 731   const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
 732   const TypeInstPtr* elem_klass    = gvn().type(argument(1))->isa_instptr();
 733   const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
 734   Node*              shuffle       = argument(3);
 735   const TypeInt*     vlen          = gvn().type(argument(4))->isa_int();
 736 
 737   if (vector_klass == NULL || elem_klass == NULL || shuffle_klass == NULL || shuffle->is_top() || vlen == NULL) {
 738     return false; // dead code
 739   }
 740   if (!vlen->is_con() || vector_klass->const_oop() == NULL || shuffle_klass->const_oop() == NULL) {
 741     return false; // not enough info for intrinsification
 742   }
 743   if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
 744     if (C->print_intrinsics()) {
 745       tty->print_cr("  ** klass argument not initialized");
 746     }
 747     return false;
 748   }
 749 
 750   int num_elem = vlen->get_con();
 751   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 752   BasicType elem_bt = elem_type->basic_type();
 753 
 754   if (num_elem < 4) {
 755     return false;
 756   }
 757 
 758   int cast_vopc = VectorCastNode::opcode(T_BYTE); // from shuffle of type T_BYTE
 759   // Make sure that cast is implemented to particular type/size combination.
 760   if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
 761     if (C->print_intrinsics()) {
 762       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 763         cast_vopc, num_elem, type2name(elem_bt));
 764     }
 765     return false;
 766   }
 767 
 768   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 769   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 770 
 771   // Unbox shuffle with true flag to indicate its load shuffle to vector
 772   // shuffle is a byte array
 773   Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, T_BYTE, num_elem, true);
 774 
 775   // cast byte to target element type
 776   shuffle_vec = gvn().transform(VectorCastNode::make(cast_vopc, shuffle_vec, elem_bt, num_elem));
 777 
 778   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 779   const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 780 
 781   // Box vector
 782   Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem);
 783   set_result(res);
 784   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 785   return true;
 786 }
 787 
 788 // public static
 789 // <M,
 790 //  S extends VectorSpecies<E>,
 791 //  E>
 792 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
 793 //                    long bits, int mode, S s,
 794 //                    BroadcastOperation<M, E, S> defaultImpl)
 795 bool LibraryCallKit::inline_vector_frombits_coerced() {
 796   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 797   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 798   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 799   const TypeLong*    bits_type    = gvn().type(argument(3))->isa_long();
 800   // Mode argument determines the mode of operation it can take following values:-
 801   // MODE_BROADCAST for vector Vector.boradcast and VectorMask.maskAll operations.
 802   // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
 803   const TypeInt*     mode         = gvn().type(argument(5))->isa_int();
 804 
 805   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || mode == NULL ||
 806       bits_type == NULL || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL ||
 807       !vlen->is_con() || !mode->is_con()) {
 808     if (C->print_intrinsics()) {
 809       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
 810                     NodeClassNames[argument(0)->Opcode()],
 811                     NodeClassNames[argument(1)->Opcode()],
 812                     NodeClassNames[argument(2)->Opcode()],
 813                     NodeClassNames[argument(5)->Opcode()]);
 814     }
 815     return false; // not enough info for intrinsification
 816   }
 817 
 818   if (!is_klass_initialized(vector_klass)) {
 819     if (C->print_intrinsics()) {
 820       tty->print_cr("  ** klass argument not initialized");
 821     }
 822     return false;
 823   }
 824   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 825   if (!elem_type->is_primitive_type()) {
 826     if (C->print_intrinsics()) {
 827       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 828     }
 829     return false; // should be primitive type
 830   }
 831   BasicType elem_bt = elem_type->basic_type();
 832   int num_elem = vlen->get_con();
 833   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 834   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 835 
 836   bool is_mask = is_vector_mask(vbox_klass);
 837   int  bcast_mode = mode->get_con();
 838   VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
 839   int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : VectorNode::replicate_opcode(elem_bt);
 840 
 841   if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
 842     if (C->print_intrinsics()) {
 843       tty->print_cr("  ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
 844                     num_elem, type2name(elem_bt),
 845                     is_mask ? 1 : 0,
 846                     bcast_mode);
 847     }
 848     return false; // not supported
 849   }
 850 
 851   Node* broadcast = NULL;
 852   Node* bits = argument(3);
 853   Node* elem = bits;
 854 
 855   if (opc == Op_VectorLongToMask) {
 856     const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
 857     if (vt->isa_vectmask()) {
 858       broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
 859     } else {
 860       const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
 861       broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
 862       broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
 863     }
 864   } else {
 865     switch (elem_bt) {
 866       case T_BOOLEAN: // fall-through
 867       case T_BYTE:    // fall-through
 868       case T_SHORT:   // fall-through
 869       case T_CHAR:    // fall-through
 870       case T_INT: {
 871         elem = gvn().transform(new ConvL2INode(bits));
 872         break;
 873       }
 874       case T_DOUBLE: {
 875         elem = gvn().transform(new MoveL2DNode(bits));
 876         break;
 877       }
 878       case T_FLOAT: {
 879         bits = gvn().transform(new ConvL2INode(bits));
 880         elem = gvn().transform(new MoveI2FNode(bits));
 881         break;
 882       }
 883       case T_LONG: {
 884         // no conversion needed
 885         break;
 886       }
 887       default: fatal("%s", type2name(elem_bt));
 888     }
 889     broadcast = VectorNode::scalar2vector(elem, num_elem, Type::get_const_basic_type(elem_bt), is_mask);
 890     broadcast = gvn().transform(broadcast);
 891   }
 892 
 893   Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
 894   set_result(box);
 895   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 896   return true;
 897 }
 898 
 899 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type) {
 900   assert(arr_type != NULL, "unexpected");
 901   BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
 902   if (elem_bt == arr_elem_bt) {
 903     return true;
 904   } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
 905     // Load/store of short vector from/to char[] is supported
 906     return true;
 907   } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
 908     // Load/store of byte vector from/to boolean[] is supported
 909     return true;
 910   } else {
 911     return false;
 912   }
 913 }
 914 
 915 // public static
 916 // <C,
 917 //  VM,
 918 //  E,
 919 //  S extends VectorSpecies<E>>
 920 // VM load(Class<? extends VM> vmClass, Class<E> elementType, int length,
 921 //         Object base, long offset,    // Unsafe addressing
 922 //         C container, int index, S s,     // Arguments for default implementation
 923 //         LoadOperation<C, VM, E, S> defaultImpl)
 924 //
 925 // public static
 926 // <C,
 927 //  V extends Vector<?>>
 928 // void store(Class<?> vectorClass, Class<?> elementType, int length,
 929 //            Object base, long offset,    // Unsafe addressing
 930 //            V v,
 931 //            C container, int index,      // Arguments for default implementation
 932 //            StoreVectorOperation<C, V> defaultImpl)
 933 
 934 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
 935   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 936   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 937   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 938 
 939   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
 940       vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 941     if (C->print_intrinsics()) {
 942       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
 943                     NodeClassNames[argument(0)->Opcode()],
 944                     NodeClassNames[argument(1)->Opcode()],
 945                     NodeClassNames[argument(2)->Opcode()]);
 946     }
 947     return false; // not enough info for intrinsification
 948   }
 949   if (!is_klass_initialized(vector_klass)) {
 950     if (C->print_intrinsics()) {
 951       tty->print_cr("  ** klass argument not initialized");
 952     }
 953     return false;
 954   }
 955 
 956   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 957   if (!elem_type->is_primitive_type()) {
 958     if (C->print_intrinsics()) {
 959       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 960     }
 961     return false; // should be primitive type
 962   }
 963   BasicType elem_bt = elem_type->basic_type();
 964   int num_elem = vlen->get_con();
 965 
 966   // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
 967   if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
 968     if (C->print_intrinsics()) {
 969       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
 970                     is_store, is_store ? "store" : "load",
 971                     num_elem, type2name(elem_bt));
 972     }
 973     return false; // not supported
 974   }
 975 
 976   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 977   bool is_mask = is_vector_mask(vbox_klass);
 978 
 979   Node* base = argument(3);
 980   Node* offset = ConvL2X(argument(4));
 981 
 982   // Save state and restore on bailout
 983   uint old_sp = sp();
 984   SafePointNode* old_map = clone_map();
 985 
 986   Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
 987 
 988   // The memory barrier checks are based on ones for unsafe access.
 989   // This is not 1-1 implementation.
 990   const Type *const base_type = gvn().type(base);
 991 
 992   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
 993   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
 994 
 995   const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
 996   const bool in_heap   = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
 997 
 998   const bool is_mixed_access = !in_heap && !in_native;
 999 
1000   const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == NULL);
1001 
1002   const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
1003 
1004   // Now handle special case where load/store happens from/to byte array but element type is not byte.
1005   bool using_byte_array = arr_type != NULL && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE;
1006   // Handle loading masks.
1007   // If there is no consistency between array and vector element types, it must be special byte array case or loading masks
1008   if (arr_type != NULL && !using_byte_array && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type)) {
1009     if (C->print_intrinsics()) {
1010       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1011                     is_store, is_store ? "store" : "load",
1012                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1013     }
1014     set_map(old_map);
1015     set_sp(old_sp);
1016     return false;
1017   }
1018   // Since we are using byte array, we need to double check that the byte operations are supported by backend.
1019   if (using_byte_array) {
1020     int byte_num_elem = num_elem * type2aelembytes(elem_bt);
1021     if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, byte_num_elem, T_BYTE, VecMaskNotUsed)
1022         || !arch_supports_vector(Op_VectorReinterpret, byte_num_elem, T_BYTE, VecMaskNotUsed)) {
1023       if (C->print_intrinsics()) {
1024         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
1025                       is_store, is_store ? "store" : "load",
1026                       byte_num_elem, type2name(elem_bt));
1027       }
1028       set_map(old_map);
1029       set_sp(old_sp);
1030       return false; // not supported
1031     }
1032   }
1033   if (is_mask) {
1034     if (!arch_supports_vector(Op_LoadVector, num_elem, T_BOOLEAN, VecMaskNotUsed)) {
1035       if (C->print_intrinsics()) {
1036         tty->print_cr("  ** not supported: arity=%d op=%s/mask vlen=%d etype=bit ismask=no",
1037                       is_store, is_store ? "store" : "load",
1038                       num_elem);
1039       }
1040       set_map(old_map);
1041       set_sp(old_sp);
1042       return false; // not supported
1043     }
1044     if (!is_store) {
1045       if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1046         set_map(old_map);
1047         set_sp(old_sp);
1048         return false; // not supported
1049       }
1050     } else {
1051       if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
1052         set_map(old_map);
1053         set_sp(old_sp);
1054         return false; // not supported
1055       }
1056     }
1057   }
1058 
1059   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1060 
1061   if (needs_cpu_membar) {
1062     insert_mem_bar(Op_MemBarCPUOrder);
1063   }
1064 
1065   if (is_store) {
1066     Node* val = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1067     if (val == NULL) {
1068       set_map(old_map);
1069       set_sp(old_sp);
1070       return false; // operand unboxing failed
1071     }
1072     set_all_memory(reset_memory());
1073 
1074     // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
1075     int store_num_elem = num_elem;
1076     if (using_byte_array) {
1077       store_num_elem = num_elem * type2aelembytes(elem_bt);
1078       const TypeVect* to_vect_type = TypeVect::make(T_BYTE, store_num_elem);
1079       val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1080     }
1081 
1082     Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
1083     set_memory(vstore, addr_type);
1084   } else {
1085     // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
1086     Node* vload = NULL;
1087     if (using_byte_array) {
1088       int load_num_elem = num_elem * type2aelembytes(elem_bt);
1089       vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, load_num_elem, T_BYTE));
1090       const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1091       vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1092     } else {
1093       // Special handle for masks
1094       if (is_mask) {
1095         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
1096         vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
1097       } else {
1098         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
1099       }
1100     }
1101     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1102     set_result(box);
1103   }
1104 
1105   old_map->destruct(&_gvn);
1106 
1107   if (needs_cpu_membar) {
1108     insert_mem_bar(Op_MemBarCPUOrder);
1109   }
1110 
1111   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1112   return true;
1113 }
1114 
1115 // public static
1116 // <C,
1117 //  V extends Vector<?>,
1118 //  E,
1119 //  S extends VectorSpecies<E>,
1120 //  M extends VectorMask<E>>
1121 // V loadMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1122 //              int length, Object base, long offset, M m,
1123 //              C container, int index, S s,  // Arguments for default implementation
1124 //              LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1125 //
1126 // public static
1127 // <C,
1128 //  V extends Vector<E>,
1129 //  M extends VectorMask<E>,
1130 //  E>
1131 // void storeMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1132 //                  int length, Object base, long offset,
1133 //                  V v, M m,
1134 //                  C container, int index,  // Arguments for default implementation
1135 //                  StoreVectorMaskedOperation<C, V, M, E> defaultImpl) {
1136 //
1137 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1138   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1139   const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1140   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1141   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1142 
1143   if (vector_klass == NULL || mask_klass == NULL || elem_klass == NULL || vlen == NULL ||
1144       vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
1145       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1146     if (C->print_intrinsics()) {
1147       tty->print_cr("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1148                     NodeClassNames[argument(0)->Opcode()],
1149                     NodeClassNames[argument(1)->Opcode()],
1150                     NodeClassNames[argument(2)->Opcode()],
1151                     NodeClassNames[argument(3)->Opcode()]);
1152     }
1153     return false; // not enough info for intrinsification
1154   }
1155   if (!is_klass_initialized(vector_klass)) {
1156     if (C->print_intrinsics()) {
1157       tty->print_cr("  ** klass argument not initialized");
1158     }
1159     return false;
1160   }
1161 
1162   if (!is_klass_initialized(mask_klass)) {
1163     if (C->print_intrinsics()) {
1164       tty->print_cr("  ** mask klass argument not initialized");
1165     }
1166     return false;
1167   }
1168 
1169   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1170   if (!elem_type->is_primitive_type()) {
1171     if (C->print_intrinsics()) {
1172       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1173     }
1174     return false; // should be primitive type
1175   }
1176 
1177   BasicType elem_bt = elem_type->basic_type();
1178   int num_elem = vlen->get_con();
1179 
1180   Node* base = argument(4);
1181   Node* offset = ConvL2X(argument(5));
1182 
1183   // Save state and restore on bailout
1184   uint old_sp = sp();
1185   SafePointNode* old_map = clone_map();
1186 
1187   Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1188   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1189   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1190 
1191   // Now handle special case where load/store happens from/to byte array but element type is not byte.
1192   bool using_byte_array = arr_type != NULL && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE;
1193   // If there is no consistency between array and vector element types, it must be special byte array case
1194   if (arr_type != NULL && !using_byte_array && !elem_consistent_with_arr(elem_bt, arr_type)) {
1195     if (C->print_intrinsics()) {
1196       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
1197                     is_store, is_store ? "storeMasked" : "loadMasked",
1198                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1199     }
1200     set_map(old_map);
1201     set_sp(old_sp);
1202     return false;
1203   }
1204 
1205   int mem_num_elem = using_byte_array ? num_elem * type2aelembytes(elem_bt) : num_elem;
1206   BasicType mem_elem_bt = using_byte_array ? T_BYTE : elem_bt;
1207   bool use_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
1208                                             mem_num_elem, mem_elem_bt,
1209                                             (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred));
1210   // Masked vector store operation needs the architecture predicate feature. We need to check
1211   // whether the predicated vector operation is supported by backend.
1212   if (is_store && !use_predicate) {
1213     if (C->print_intrinsics()) {
1214       tty->print_cr("  ** not supported: op=storeMasked vlen=%d etype=%s using_byte_array=%d",
1215                     num_elem, type2name(elem_bt), using_byte_array ? 1 : 0);
1216     }
1217     set_map(old_map);
1218     set_sp(old_sp);
1219     return false;
1220   }
1221 
1222   // This only happens for masked vector load. If predicate is not supported, then check whether
1223   // the normal vector load and blend operations are supported by backend.
1224   if (!use_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1225       !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1226     if (C->print_intrinsics()) {
1227       tty->print_cr("  ** not supported: op=loadMasked vlen=%d etype=%s using_byte_array=%d",
1228                     num_elem, type2name(elem_bt), using_byte_array ? 1 : 0);
1229     }
1230     set_map(old_map);
1231     set_sp(old_sp);
1232     return false;
1233   }
1234 
1235   // Since we are using byte array, we need to double check that the vector reinterpret operation
1236   // with byte type is supported by backend.
1237   if (using_byte_array) {
1238     if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1239       if (C->print_intrinsics()) {
1240         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s using_byte_array=1",
1241                       is_store, is_store ? "storeMasked" : "loadMasked",
1242                       num_elem, type2name(elem_bt));
1243       }
1244       set_map(old_map);
1245       set_sp(old_sp);
1246       return false;
1247     }
1248   }
1249 
1250   // Since it needs to unbox the mask, we need to double check that the related load operations
1251   // for mask are supported by backend.
1252   if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1253     if (C->print_intrinsics()) {
1254       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s",
1255                       is_store, is_store ? "storeMasked" : "loadMasked",
1256                       num_elem, type2name(elem_bt));
1257     }
1258     set_map(old_map);
1259     set_sp(old_sp);
1260     return false;
1261   }
1262 
1263   // Can base be NULL? Otherwise, always on-heap access.
1264   bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1265   if (can_access_non_heap) {
1266     insert_mem_bar(Op_MemBarCPUOrder);
1267   }
1268 
1269   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1270   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1271   assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1272   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1273   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1274 
1275   Node* mask = unbox_vector(is_store ? argument(8) : argument(7), mbox_type, elem_bt, num_elem);
1276   if (mask == NULL) {
1277     if (C->print_intrinsics()) {
1278       tty->print_cr("  ** unbox failed mask=%s",
1279                     is_store ? NodeClassNames[argument(8)->Opcode()]
1280                              : NodeClassNames[argument(7)->Opcode()]);
1281     }
1282     set_map(old_map);
1283     set_sp(old_sp);
1284     return false;
1285   }
1286 
1287   if (is_store) {
1288     Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
1289     if (val == NULL) {
1290       if (C->print_intrinsics()) {
1291         tty->print_cr("  ** unbox failed vector=%s",
1292                       NodeClassNames[argument(7)->Opcode()]);
1293       }
1294       set_map(old_map);
1295       set_sp(old_sp);
1296       return false; // operand unboxing failed
1297     }
1298     set_all_memory(reset_memory());
1299 
1300     if (using_byte_array) {
1301       // Reinterpret the incoming vector to byte vector.
1302       const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1303       val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1304       // Reinterpret the vector mask to byte type.
1305       const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1306       const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1307       mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1308     }
1309     Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1310     set_memory(vstore, addr_type);
1311   } else {
1312     Node* vload = NULL;
1313 
1314     if (using_byte_array) {
1315       // Reinterpret the vector mask to byte type.
1316       const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1317       const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1318       mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1319     }
1320 
1321     if (use_predicate) {
1322       // Generate masked load vector node if predicate feature is supported.
1323       const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1324       vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1325     } else {
1326       // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1327       Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1328       zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, Type::get_const_basic_type(mem_elem_bt)));
1329       vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1330       vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1331     }
1332 
1333     if (using_byte_array) {
1334       const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1335       vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1336     }
1337 
1338     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1339     set_result(box);
1340   }
1341 
1342   old_map->destruct(&_gvn);
1343 
1344   if (can_access_non_heap) {
1345     insert_mem_bar(Op_MemBarCPUOrder);
1346   }
1347 
1348   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1349   return true;
1350 }
1351 
1352 // <C,
1353 //  V extends Vector<?>,
1354 //  W extends Vector<Integer>,
1355 //  S extends VectorSpecies<E>,
1356 //  M extends VectorMask<E>,
1357 //  E>
1358 // V loadWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int length,
1359 //               Class<? extends Vector<Integer>> vectorIndexClass,
1360 //               Object base, long offset, // Unsafe addressing
1361 //               W index_vector, M m,
1362 //               C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
1363 //               LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1364 //
1365 //  <C,
1366 //   V extends Vector<E>,
1367 //   W extends Vector<Integer>,
1368 //   M extends VectorMask<E>,
1369 //   E>
1370 //  void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1371 //                    int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset,    // Unsafe addressing
1372 //                    W index_vector, V v, M m,
1373 //                    C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1374 //                    StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1375 //
1376 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1377   const TypeInstPtr* vector_klass     = gvn().type(argument(0))->isa_instptr();
1378   const TypeInstPtr* mask_klass       = gvn().type(argument(1))->isa_instptr();
1379   const TypeInstPtr* elem_klass       = gvn().type(argument(2))->isa_instptr();
1380   const TypeInt*     vlen             = gvn().type(argument(3))->isa_int();
1381   const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1382 
1383   if (vector_klass == NULL || elem_klass == NULL || vector_idx_klass == NULL || vlen == NULL ||
1384       vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || vector_idx_klass->const_oop() == NULL || !vlen->is_con()) {
1385     if (C->print_intrinsics()) {
1386       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1387                     NodeClassNames[argument(0)->Opcode()],
1388                     NodeClassNames[argument(2)->Opcode()],
1389                     NodeClassNames[argument(3)->Opcode()],
1390                     NodeClassNames[argument(4)->Opcode()]);
1391     }
1392     return false; // not enough info for intrinsification
1393   }
1394 
1395   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1396     if (C->print_intrinsics()) {
1397       tty->print_cr("  ** klass argument not initialized");
1398     }
1399     return false;
1400   }
1401 
1402   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1403   if (!elem_type->is_primitive_type()) {
1404     if (C->print_intrinsics()) {
1405       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1406     }
1407     return false; // should be primitive type
1408   }
1409 
1410   BasicType elem_bt = elem_type->basic_type();
1411   int num_elem = vlen->get_con();
1412 
1413   const Type* vmask_type = gvn().type(is_scatter ? argument(10) : argument(9));
1414   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1415   if (is_masked_op) {
1416     if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
1417       if (C->print_intrinsics()) {
1418         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1419       }
1420       return false; // not enough info for intrinsification
1421     }
1422 
1423     if (!is_klass_initialized(mask_klass)) {
1424       if (C->print_intrinsics()) {
1425         tty->print_cr("  ** mask klass argument not initialized");
1426       }
1427       return false;
1428     }
1429 
1430     if (vmask_type->maybe_null()) {
1431       if (C->print_intrinsics()) {
1432         tty->print_cr("  ** null mask values are not allowed for masked op");
1433       }
1434       return false;
1435     }
1436 
1437     // Check whether the predicated gather/scatter node is supported by architecture.
1438     if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt,
1439                               (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred))) {
1440       if (C->print_intrinsics()) {
1441         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1442                       is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1443                       num_elem, type2name(elem_bt));
1444       }
1445       return false; // not supported
1446     }
1447   } else {
1448     // Check whether the normal gather/scatter node is supported for non-masked operation.
1449     if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1450       if (C->print_intrinsics()) {
1451         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1452                       is_scatter, is_scatter ? "scatter" : "gather",
1453                       num_elem, type2name(elem_bt));
1454       }
1455       return false; // not supported
1456     }
1457   }
1458 
1459   // Check that the vector holding indices is supported by architecture
1460   if (!arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
1461       if (C->print_intrinsics()) {
1462         tty->print_cr("  ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1463                       is_scatter, is_scatter ? "scatter" : "gather",
1464                       num_elem, is_masked_op ? 1 : 0);
1465       }
1466       return false; // not supported
1467   }
1468 
1469   Node* base = argument(5);
1470   Node* offset = ConvL2X(argument(6));
1471 
1472   // Save state and restore on bailout
1473   uint old_sp = sp();
1474   SafePointNode* old_map = clone_map();
1475 
1476   Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1477 
1478   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1479   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1480 
1481   // The array must be consistent with vector type
1482   if (arr_type == NULL || (arr_type != NULL && !elem_consistent_with_arr(elem_bt, arr_type))) {
1483     if (C->print_intrinsics()) {
1484       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1485                     is_scatter, is_scatter ? "scatter" : "gather",
1486                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1487     }
1488     set_map(old_map);
1489     set_sp(old_sp);
1490     return false;
1491   }
1492 
1493   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1494   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1495   ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1496   if (vbox_idx_klass == NULL) {
1497     set_map(old_map);
1498     set_sp(old_sp);
1499     return false;
1500   }
1501 
1502   const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1503   Node* index_vect = unbox_vector(argument(8), vbox_idx_type, T_INT, num_elem);
1504   if (index_vect == NULL) {
1505     set_map(old_map);
1506     set_sp(old_sp);
1507     return false;
1508   }
1509 
1510   Node* mask = NULL;
1511   if (is_masked_op) {
1512     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1513     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1514     mask = unbox_vector(is_scatter ? argument(10) : argument(9), mbox_type, elem_bt, num_elem);
1515     if (mask == NULL) {
1516       if (C->print_intrinsics()) {
1517         tty->print_cr("  ** unbox failed mask=%s",
1518                     is_scatter ? NodeClassNames[argument(10)->Opcode()]
1519                                : NodeClassNames[argument(9)->Opcode()]);
1520       }
1521       set_map(old_map);
1522       set_sp(old_sp);
1523       return false;
1524     }
1525   }
1526 
1527   const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1528   if (is_scatter) {
1529     Node* val = unbox_vector(argument(9), vbox_type, elem_bt, num_elem);
1530     if (val == NULL) {
1531       set_map(old_map);
1532       set_sp(old_sp);
1533       return false; // operand unboxing failed
1534     }
1535     set_all_memory(reset_memory());
1536 
1537     Node* vstore = NULL;
1538     if (mask != NULL) {
1539       vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, index_vect, mask));
1540     } else {
1541       vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1542     }
1543     set_memory(vstore, addr_type);
1544   } else {
1545     Node* vload = NULL;
1546     if (mask != NULL) {
1547       vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_vect, mask));
1548     } else {
1549       vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1550     }
1551     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1552     set_result(box);
1553   }
1554 
1555   old_map->destruct(&_gvn);
1556 
1557   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1558   return true;
1559 }
1560 
1561 // public static
1562 // <V extends Vector<E>,
1563 //  M extends VectorMask<E>,
1564 //  E>
1565 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1566 //                       Class<E> elementType, int length, V v, M m,
1567 //                       ReductionOperation<V, M> defaultImpl)
1568 bool LibraryCallKit::inline_vector_reduction() {
1569   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
1570   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1571   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
1572   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
1573   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
1574 
1575   if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
1576       !opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1577     if (C->print_intrinsics()) {
1578       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1579                     NodeClassNames[argument(0)->Opcode()],
1580                     NodeClassNames[argument(1)->Opcode()],
1581                     NodeClassNames[argument(3)->Opcode()],
1582                     NodeClassNames[argument(4)->Opcode()]);
1583     }
1584     return false; // not enough info for intrinsification
1585   }
1586   if (!is_klass_initialized(vector_klass)) {
1587     if (C->print_intrinsics()) {
1588       tty->print_cr("  ** klass argument not initialized");
1589     }
1590     return false;
1591   }
1592   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1593   if (!elem_type->is_primitive_type()) {
1594     if (C->print_intrinsics()) {
1595       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1596     }
1597     return false; // should be primitive type
1598   }
1599 
1600   const Type* vmask_type = gvn().type(argument(6));
1601   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1602   if (is_masked_op) {
1603     if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
1604       if (C->print_intrinsics()) {
1605         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1606       }
1607       return false; // not enough info for intrinsification
1608     }
1609 
1610     if (!is_klass_initialized(mask_klass)) {
1611       if (C->print_intrinsics()) {
1612         tty->print_cr("  ** mask klass argument not initialized");
1613       }
1614       return false;
1615     }
1616 
1617     if (vmask_type->maybe_null()) {
1618       if (C->print_intrinsics()) {
1619         tty->print_cr("  ** null mask values are not allowed for masked op");
1620       }
1621       return false;
1622     }
1623   }
1624 
1625   BasicType elem_bt = elem_type->basic_type();
1626   int num_elem = vlen->get_con();
1627   int opc  = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1628   int sopc = ReductionNode::opcode(opc, elem_bt);
1629 
1630   // When using mask, mask use type needs to be VecMaskUseLoad.
1631   if (!arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1632     if (C->print_intrinsics()) {
1633       tty->print_cr("  ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1634                     sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1635     }
1636     return false;
1637   }
1638 
1639   // Return true if current platform has implemented the masked operation with predicate feature.
1640   bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1641   if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1642     if (C->print_intrinsics()) {
1643       tty->print_cr("  ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1644                     sopc, num_elem, type2name(elem_bt));
1645     }
1646     return false;
1647   }
1648 
1649   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1650   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1651 
1652   Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1653   if (opd == NULL) {
1654     return false; // operand unboxing failed
1655   }
1656 
1657   Node* mask = NULL;
1658   if (is_masked_op) {
1659     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1660     assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1661     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1662     mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1663     if (mask == NULL) {
1664       if (C->print_intrinsics()) {
1665         tty->print_cr("  ** unbox failed mask=%s",
1666                       NodeClassNames[argument(6)->Opcode()]);
1667       }
1668       return false;
1669     }
1670   }
1671 
1672   Node* init = ReductionNode::make_reduction_input(gvn(), opc, elem_bt);
1673   Node* value = NULL;
1674   if (mask == NULL) {
1675     assert(!is_masked_op, "Masked op needs the mask value never null");
1676     value = ReductionNode::make(opc, NULL, init, opd, elem_bt);
1677   } else {
1678     if (use_predicate) {
1679       value = ReductionNode::make(opc, NULL, init, opd, elem_bt);
1680       value->add_req(mask);
1681       value->add_flag(Node::Flag_is_predicated_vector);
1682     } else {
1683       Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, Type::get_const_basic_type(elem_bt)));
1684       value = gvn().transform(new VectorBlendNode(reduce_identity, opd, mask));
1685       value = ReductionNode::make(opc, NULL, init, value, elem_bt);
1686     }
1687   }
1688   value = gvn().transform(value);
1689 
1690   Node* bits = NULL;
1691   switch (elem_bt) {
1692     case T_BYTE:
1693     case T_SHORT:
1694     case T_INT: {
1695       bits = gvn().transform(new ConvI2LNode(value));
1696       break;
1697     }
1698     case T_FLOAT: {
1699       value = gvn().transform(new MoveF2INode(value));
1700       bits  = gvn().transform(new ConvI2LNode(value));
1701       break;
1702     }
1703     case T_DOUBLE: {
1704       bits = gvn().transform(new MoveD2LNode(value));
1705       break;
1706     }
1707     case T_LONG: {
1708       bits = value; // no conversion needed
1709       break;
1710     }
1711     default: fatal("%s", type2name(elem_bt));
1712   }
1713   set_result(bits);
1714   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1715   return true;
1716 }
1717 
1718 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1719 //                                V v1, V v2,
1720 //                                BiFunction<V, V, Boolean> defaultImpl)
1721 //
1722 bool LibraryCallKit::inline_vector_test() {
1723   const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1724   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1725   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1726   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1727 
1728   if (cond == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
1729       !cond->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1730     if (C->print_intrinsics()) {
1731       tty->print_cr("  ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1732                     NodeClassNames[argument(0)->Opcode()],
1733                     NodeClassNames[argument(1)->Opcode()],
1734                     NodeClassNames[argument(2)->Opcode()],
1735                     NodeClassNames[argument(3)->Opcode()]);
1736     }
1737     return false; // not enough info for intrinsification
1738   }
1739   if (!is_klass_initialized(vector_klass)) {
1740     if (C->print_intrinsics()) {
1741       tty->print_cr("  ** klass argument not initialized");
1742     }
1743     return false;
1744   }
1745   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1746   if (!elem_type->is_primitive_type()) {
1747     if (C->print_intrinsics()) {
1748       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1749     }
1750     return false; // should be primitive type
1751   }
1752   BasicType elem_bt = elem_type->basic_type();
1753   int num_elem = vlen->get_con();
1754   BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1755   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1756   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1757 
1758   if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1759     if (C->print_intrinsics()) {
1760       tty->print_cr("  ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1761                     cond->get_con(), num_elem, type2name(elem_bt),
1762                     is_vector_mask(vbox_klass));
1763     }
1764     return false;
1765   }
1766 
1767   Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1768   Node* opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1769   if (opd1 == NULL || opd2 == NULL) {
1770     return false; // operand unboxing failed
1771   }
1772   Node* test = new VectorTestNode(opd1, opd2, booltest);
1773   test = gvn().transform(test);
1774 
1775   set_result(test);
1776   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1777   return true;
1778 }
1779 
1780 // public static
1781 // <V extends Vector<E>,
1782 //  M extends VectorMask<E>,
1783 //  E>
1784 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1785 //         V v1, V v2, M m,
1786 //         VectorBlendOp<V, M, E> defaultImpl)
1787 bool LibraryCallKit::inline_vector_blend() {
1788   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1789   const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1790   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1791   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1792 
1793   if (mask_klass == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL) {
1794     return false; // dead code
1795   }
1796   if (mask_klass->const_oop() == NULL || vector_klass->const_oop() == NULL ||
1797       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1798     if (C->print_intrinsics()) {
1799       tty->print_cr("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1800                     NodeClassNames[argument(0)->Opcode()],
1801                     NodeClassNames[argument(1)->Opcode()],
1802                     NodeClassNames[argument(2)->Opcode()],
1803                     NodeClassNames[argument(3)->Opcode()]);
1804     }
1805     return false; // not enough info for intrinsification
1806   }
1807   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1808     if (C->print_intrinsics()) {
1809       tty->print_cr("  ** klass argument not initialized");
1810     }
1811     return false;
1812   }
1813   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1814   if (!elem_type->is_primitive_type()) {
1815     if (C->print_intrinsics()) {
1816       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1817     }
1818     return false; // should be primitive type
1819   }
1820   BasicType elem_bt = elem_type->basic_type();
1821   BasicType mask_bt = elem_bt;
1822   int num_elem = vlen->get_con();
1823 
1824   if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1825     if (C->print_intrinsics()) {
1826       tty->print_cr("  ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1827                     num_elem, type2name(elem_bt));
1828     }
1829     return false; // not supported
1830   }
1831   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1832   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1833 
1834   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1835   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1836 
1837   Node* v1   = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1838   Node* v2   = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1839   Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1840 
1841   if (v1 == NULL || v2 == NULL || mask == NULL) {
1842     return false; // operand unboxing failed
1843   }
1844 
1845   Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1846 
1847   Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1848   set_result(box);
1849   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1850   return true;
1851 }
1852 
1853 //  public static
1854 //  <V extends Vector<E>,
1855 //   M extends VectorMask<E>,
1856 //   E>
1857 //  M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1858 //            V v1, V v2, M m,
1859 //            VectorCompareOp<V,M> defaultImpl)
1860 bool LibraryCallKit::inline_vector_compare() {
1861   const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1862   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1863   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
1864   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
1865   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
1866 
1867   if (cond == NULL || vector_klass == NULL || mask_klass == NULL || elem_klass == NULL || vlen == NULL) {
1868     return false; // dead code
1869   }
1870   if (!cond->is_con() || vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
1871       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1872     if (C->print_intrinsics()) {
1873       tty->print_cr("  ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1874                     NodeClassNames[argument(0)->Opcode()],
1875                     NodeClassNames[argument(1)->Opcode()],
1876                     NodeClassNames[argument(2)->Opcode()],
1877                     NodeClassNames[argument(3)->Opcode()],
1878                     NodeClassNames[argument(4)->Opcode()]);
1879     }
1880     return false; // not enough info for intrinsification
1881   }
1882   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1883     if (C->print_intrinsics()) {
1884       tty->print_cr("  ** klass argument not initialized");
1885     }
1886     return false;
1887   }
1888   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1889   if (!elem_type->is_primitive_type()) {
1890     if (C->print_intrinsics()) {
1891       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1892     }
1893     return false; // should be primitive type
1894   }
1895 
1896   int num_elem = vlen->get_con();
1897   BasicType elem_bt = elem_type->basic_type();
1898   BasicType mask_bt = elem_bt;
1899 
1900   if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1901     if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1902       if (C->print_intrinsics()) {
1903         tty->print_cr("  ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1904                       cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1905       }
1906       return false;
1907     }
1908   }
1909 
1910   if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1911     if (C->print_intrinsics()) {
1912       tty->print_cr("  ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1913                     cond->get_con(), num_elem, type2name(elem_bt));
1914     }
1915     return false;
1916   }
1917 
1918   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1919   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1920 
1921   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1922   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1923 
1924   Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1925   Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1926 
1927   bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1928   Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : NULL;
1929   if (is_masked_op && mask == NULL) {
1930     if (C->print_intrinsics()) {
1931       tty->print_cr("  ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1932                     cond->get_con(), num_elem, type2name(elem_bt));
1933     }
1934     return false;
1935   }
1936 
1937   bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1938   if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1939     if (C->print_intrinsics()) {
1940       tty->print_cr("  ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1941                     cond->get_con(), num_elem, type2name(elem_bt));
1942     }
1943     return false;
1944   }
1945 
1946   if (v1 == NULL || v2 == NULL) {
1947     return false; // operand unboxing failed
1948   }
1949   BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1950   ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1951 
1952   const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1953   Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1954 
1955   if (is_masked_op) {
1956     if (use_predicate) {
1957       operation->add_req(mask);
1958       operation->add_flag(Node::Flag_is_predicated_vector);
1959     } else {
1960       operation = gvn().transform(operation);
1961       operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1962     }
1963   }
1964 
1965   operation = gvn().transform(operation);
1966 
1967   Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1968   set_result(box);
1969   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1970   return true;
1971 }
1972 
1973 // public static
1974 // <V extends Vector<E>,
1975 //  Sh extends VectorShuffle<E>,
1976 //  M extends VectorMask<E>,
1977 //  E>
1978 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1979 //               V v1, Sh sh, M m,
1980 //               VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1981 bool LibraryCallKit::inline_vector_rearrange() {
1982   const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
1983   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1984   const TypeInstPtr* mask_klass    = gvn().type(argument(2))->isa_instptr();
1985   const TypeInstPtr* elem_klass    = gvn().type(argument(3))->isa_instptr();
1986   const TypeInt*     vlen          = gvn().type(argument(4))->isa_int();
1987 
1988   if (vector_klass == NULL  || shuffle_klass == NULL ||  elem_klass == NULL || vlen == NULL) {
1989     return false; // dead code
1990   }
1991   if (shuffle_klass->const_oop() == NULL ||
1992       vector_klass->const_oop()  == NULL ||
1993       elem_klass->const_oop()    == NULL ||
1994       !vlen->is_con()) {
1995     if (C->print_intrinsics()) {
1996       tty->print_cr("  ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1997                     NodeClassNames[argument(0)->Opcode()],
1998                     NodeClassNames[argument(1)->Opcode()],
1999                     NodeClassNames[argument(3)->Opcode()],
2000                     NodeClassNames[argument(4)->Opcode()]);
2001     }
2002     return false; // not enough info for intrinsification
2003   }
2004   if (!is_klass_initialized(vector_klass)  ||
2005       !is_klass_initialized(shuffle_klass)) {
2006     if (C->print_intrinsics()) {
2007       tty->print_cr("  ** klass argument not initialized");
2008     }
2009     return false;
2010   }
2011   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2012   if (!elem_type->is_primitive_type()) {
2013     if (C->print_intrinsics()) {
2014       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2015     }
2016     return false; // should be primitive type
2017   }
2018   BasicType elem_bt = elem_type->basic_type();
2019   BasicType shuffle_bt = elem_bt;
2020   int num_elem = vlen->get_con();
2021 
2022   if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
2023     if (C->print_intrinsics()) {
2024       tty->print_cr("  ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
2025                     num_elem, type2name(elem_bt));
2026     }
2027     return false; // not supported
2028   }
2029 
2030   bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
2031   bool use_predicate = is_masked_op;
2032   if (is_masked_op &&
2033       (mask_klass == NULL ||
2034        mask_klass->const_oop() == NULL ||
2035        !is_klass_initialized(mask_klass))) {
2036     if (C->print_intrinsics()) {
2037       tty->print_cr("  ** mask_klass argument not initialized");
2038     }
2039   }
2040   VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2041   if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
2042     use_predicate = false;
2043     if(!is_masked_op ||
2044        (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2045         !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)     ||
2046         !arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed))) {
2047       if (C->print_intrinsics()) {
2048         tty->print_cr("  ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
2049                       num_elem, type2name(elem_bt));
2050       }
2051       return false; // not supported
2052     }
2053   }
2054   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2055   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2056 
2057   ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
2058   const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
2059 
2060   Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2061   Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
2062 
2063   if (v1 == NULL || shuffle == NULL) {
2064     return false; // operand unboxing failed
2065   }
2066 
2067   Node* mask = NULL;
2068   if (is_masked_op) {
2069     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2070     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2071     mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2072     if (mask == NULL) {
2073       if (C->print_intrinsics()) {
2074         tty->print_cr("  ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
2075                       num_elem, type2name(elem_bt));
2076       }
2077       return false;
2078     }
2079   }
2080 
2081   Node* rearrange = new VectorRearrangeNode(v1, shuffle);
2082   if (is_masked_op) {
2083     if (use_predicate) {
2084       rearrange->add_req(mask);
2085       rearrange->add_flag(Node::Flag_is_predicated_vector);
2086     } else {
2087       const TypeVect* vt = v1->bottom_type()->is_vect();
2088       rearrange = gvn().transform(rearrange);
2089       Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2090       Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, Type::get_const_basic_type(elem_bt)));
2091       rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2092     }
2093   }
2094   rearrange = gvn().transform(rearrange);
2095 
2096   Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2097   set_result(box);
2098   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2099   return true;
2100 }
2101 
2102 static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
2103   address addr = NULL;
2104   assert(UseVectorStubs, "sanity");
2105   assert(name_ptr != NULL, "unexpected");
2106   assert((vop >= VectorSupport::VECTOR_OP_SVML_START) && (vop <= VectorSupport::VECTOR_OP_SVML_END), "unexpected");
2107   int op = vop - VectorSupport::VECTOR_OP_SVML_START;
2108 
2109   switch(bits) {
2110     case 64:  //fallthough
2111     case 128: //fallthough
2112     case 256: //fallthough
2113     case 512:
2114       if (bt == T_FLOAT) {
2115         snprintf(name_ptr, name_len, "vector_%s_float%d", VectorSupport::svmlname[op], bits);
2116         addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op];
2117       } else {
2118         assert(bt == T_DOUBLE, "must be FP type only");
2119         snprintf(name_ptr, name_len, "vector_%s_double%d", VectorSupport::svmlname[op], bits);
2120         addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op];
2121       }
2122       break;
2123     default:
2124       snprintf(name_ptr, name_len, "invalid");
2125       addr = NULL;
2126       Unimplemented();
2127       break;
2128   }
2129 
2130   return addr;
2131 }
2132 
2133 Node* LibraryCallKit::gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
2134   assert(UseVectorStubs, "sanity");
2135   assert(vector_api_op_id >= VectorSupport::VECTOR_OP_SVML_START && vector_api_op_id <= VectorSupport::VECTOR_OP_SVML_END, "need valid op id");
2136   assert(opd1 != NULL, "must not be null");
2137   const TypeVect* vt = TypeVect::make(bt, num_elem);
2138   const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != NULL ? 2 : 1, vt, vt);
2139   char name[100] = "";
2140 
2141   // Get address for svml method.
2142   address addr = get_svml_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
2143 
2144   if (addr == NULL) {
2145     return NULL;
2146   }
2147 
2148   assert(name != NULL, "name must not be null");
2149   Node* operation = make_runtime_call(RC_VECTOR,
2150                                       call_type,
2151                                       addr,
2152                                       name,
2153                                       TypePtr::BOTTOM,
2154                                       opd1,
2155                                       opd2);
2156   return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
2157 }
2158 
2159 //  public static
2160 //  <V extends Vector<E>,
2161 //   M extends VectorMask<E>,
2162 //   E>
2163 //  V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2164 //                 Class<E> elementType, int length,
2165 //                 V v, int n, M m,
2166 //                 VectorBroadcastIntOp<V, M> defaultImpl)
2167 bool LibraryCallKit::inline_vector_broadcast_int() {
2168   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
2169   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2170   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
2171   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
2172   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
2173 
2174   if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL) {
2175     return false; // dead code
2176   }
2177   if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
2178     if (C->print_intrinsics()) {
2179       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2180                     NodeClassNames[argument(0)->Opcode()],
2181                     NodeClassNames[argument(1)->Opcode()],
2182                     NodeClassNames[argument(3)->Opcode()],
2183                     NodeClassNames[argument(4)->Opcode()]);
2184     }
2185     return false; // not enough info for intrinsification
2186   }
2187   if (!is_klass_initialized(vector_klass)) {
2188     if (C->print_intrinsics()) {
2189       tty->print_cr("  ** klass argument not initialized");
2190     }
2191     return false;
2192   }
2193 
2194   const Type* vmask_type = gvn().type(argument(7));
2195   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2196   if (is_masked_op) {
2197     if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
2198       if (C->print_intrinsics()) {
2199         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2200       }
2201       return false; // not enough info for intrinsification
2202     }
2203 
2204     if (!is_klass_initialized(mask_klass)) {
2205       if (C->print_intrinsics()) {
2206         tty->print_cr("  ** mask klass argument not initialized");
2207       }
2208       return false;
2209     }
2210 
2211     if (vmask_type->maybe_null()) {
2212       if (C->print_intrinsics()) {
2213         tty->print_cr("  ** null mask values are not allowed for masked op");
2214       }
2215       return false;
2216     }
2217   }
2218 
2219   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2220   if (!elem_type->is_primitive_type()) {
2221     if (C->print_intrinsics()) {
2222       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2223     }
2224     return false; // should be primitive type
2225   }
2226 
2227   int num_elem = vlen->get_con();
2228   BasicType elem_bt = elem_type->basic_type();
2229   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2230 
2231   bool is_shift  = VectorNode::is_shift_opcode(opc);
2232   bool is_rotate = VectorNode::is_rotate_opcode(opc);
2233 
2234   if (opc == 0 || (!is_shift && !is_rotate)) {
2235     if (C->print_intrinsics()) {
2236       tty->print_cr("  ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2237     }
2238     return false; // operation not supported
2239   }
2240 
2241   int sopc = VectorNode::opcode(opc, elem_bt);
2242   if (sopc == 0) {
2243     if (C->print_intrinsics()) {
2244       tty->print_cr("  ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2245     }
2246     return false; // operation not supported
2247   }
2248 
2249   Node* cnt  = argument(6);
2250   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2251   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2252   const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2253 
2254   // If CPU supports vector constant rotate instructions pass it directly
2255   bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2256                          Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2257   bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2258 
2259   VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2260   bool use_predicate = is_masked_op;
2261 
2262   if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2263     use_predicate = false;
2264     if (!is_masked_op ||
2265         (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2266          !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2267 
2268       if (C->print_intrinsics()) {
2269         tty->print_cr("  ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2270                       sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2271       }
2272       return false; // not supported
2273     }
2274   }
2275 
2276   Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2277   Node* opd2 = NULL;
2278   if (is_shift) {
2279     opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2280   } else {
2281     assert(is_rotate, "unexpected operation");
2282     if (!is_const_rotate) {
2283       const Type * type_bt = Type::get_const_basic_type(elem_bt);
2284       cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2285       opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, type_bt));
2286     } else {
2287       // Constant shift value.
2288       opd2 = cnt;
2289     }
2290   }
2291 
2292   if (opd1 == NULL || opd2 == NULL) {
2293     return false;
2294   }
2295 
2296   Node* mask = NULL;
2297   if (is_masked_op) {
2298     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2299     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2300     mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2301     if (mask == NULL) {
2302       if (C->print_intrinsics()) {
2303         tty->print_cr("  ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2304       }
2305       return false;
2306     }
2307   }
2308 
2309   Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2310   if (is_masked_op && mask != NULL) {
2311     if (use_predicate) {
2312       operation->add_req(mask);
2313       operation->add_flag(Node::Flag_is_predicated_vector);
2314     } else {
2315       operation = gvn().transform(operation);
2316       operation = new VectorBlendNode(opd1, operation, mask);
2317     }
2318   }
2319   operation = gvn().transform(operation);
2320   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2321   set_result(vbox);
2322   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2323   return true;
2324 }
2325 
2326 // public static <VOUT extends VectorPayload,
2327 //                 VIN extends VectorPayload,
2328 //                   S extends VectorSpecies>
2329 // VOUT convert(int oprId,
2330 //           Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2331 //           Class<?>   toVectorClass, Class<?>   toElementType, int   toVLen,
2332 //           VIN v, S s,
2333 //           VectorConvertOp<VOUT, VIN, S> defaultImpl)
2334 //
2335 bool LibraryCallKit::inline_vector_convert() {
2336   const TypeInt*     opr               = gvn().type(argument(0))->isa_int();
2337 
2338   const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2339   const TypeInstPtr* elem_klass_from   = gvn().type(argument(2))->isa_instptr();
2340   const TypeInt*     vlen_from         = gvn().type(argument(3))->isa_int();
2341 
2342   const TypeInstPtr* vector_klass_to   = gvn().type(argument(4))->isa_instptr();
2343   const TypeInstPtr* elem_klass_to     = gvn().type(argument(5))->isa_instptr();
2344   const TypeInt*     vlen_to           = gvn().type(argument(6))->isa_int();
2345 
2346   if (opr == NULL ||
2347       vector_klass_from == NULL || elem_klass_from == NULL || vlen_from == NULL ||
2348       vector_klass_to   == NULL || elem_klass_to   == NULL || vlen_to   == NULL) {
2349     return false; // dead code
2350   }
2351   if (!opr->is_con() ||
2352       vector_klass_from->const_oop() == NULL || elem_klass_from->const_oop() == NULL || !vlen_from->is_con() ||
2353       vector_klass_to->const_oop() == NULL || elem_klass_to->const_oop() == NULL || !vlen_to->is_con()) {
2354     if (C->print_intrinsics()) {
2355       tty->print_cr("  ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2356                     NodeClassNames[argument(0)->Opcode()],
2357                     NodeClassNames[argument(1)->Opcode()],
2358                     NodeClassNames[argument(2)->Opcode()],
2359                     NodeClassNames[argument(3)->Opcode()],
2360                     NodeClassNames[argument(4)->Opcode()],
2361                     NodeClassNames[argument(5)->Opcode()],
2362                     NodeClassNames[argument(6)->Opcode()]);
2363     }
2364     return false; // not enough info for intrinsification
2365   }
2366   if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2367     if (C->print_intrinsics()) {
2368       tty->print_cr("  ** klass argument not initialized");
2369     }
2370     return false;
2371   }
2372 
2373   assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
2374          opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2375   bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST);
2376 
2377   ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2378   ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2379   if (is_vector_shuffle(vbox_klass_from)) {
2380     return false; // vector shuffles aren't supported
2381   }
2382   bool is_mask = is_vector_mask(vbox_klass_from);
2383 
2384   ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2385   if (!elem_type_from->is_primitive_type()) {
2386     return false; // should be primitive type
2387   }
2388   BasicType elem_bt_from = elem_type_from->basic_type();
2389   ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2390   if (!elem_type_to->is_primitive_type()) {
2391     return false; // should be primitive type
2392   }
2393   BasicType elem_bt_to = elem_type_to->basic_type();
2394 
2395   int num_elem_from = vlen_from->get_con();
2396   int num_elem_to = vlen_to->get_con();
2397 
2398   // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2399   // since we may need to change size.
2400   if (!arch_supports_vector(Op_VectorReinterpret,
2401                             num_elem_from,
2402                             elem_bt_from,
2403                             is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2404     if (C->print_intrinsics()) {
2405       tty->print_cr("  ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2406                     is_cast ? "cast" : "reinterpret",
2407                     num_elem_from, type2name(elem_bt_from), is_mask);
2408     }
2409     return false;
2410   }
2411 
2412   // Check whether we can support resizing/reinterpreting to the new size.
2413   if (!arch_supports_vector(Op_VectorReinterpret,
2414                             num_elem_to,
2415                             elem_bt_to,
2416                             is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2417     if (C->print_intrinsics()) {
2418       tty->print_cr("  ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2419                     is_cast ? "cast" : "reinterpret",
2420                     num_elem_to, type2name(elem_bt_to), is_mask);
2421     }
2422     return false;
2423   }
2424 
2425   // At this point, we know that both input and output vector registers are supported
2426   // by the architecture. Next check if the casted type is simply to same type - which means
2427   // that it is actually a resize and not a cast.
2428   if (is_cast && elem_bt_from == elem_bt_to) {
2429     is_cast = false;
2430   }
2431 
2432   const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2433 
2434   Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2435   if (opd1 == NULL) {
2436     return false;
2437   }
2438 
2439   const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2440   const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2441 
2442   // Safety check to prevent casting if source mask is of type vector
2443   // and destination mask of type predicate vector and vice-versa.
2444   // From X86 standpoint, this case will only arise over KNL target,
2445   // where certain masks (depending on the species) are either propagated
2446   // through a vector or predicate register.
2447   if (is_mask &&
2448       ((src_type->isa_vectmask() == NULL && dst_type->isa_vectmask()) ||
2449        (dst_type->isa_vectmask() == NULL && src_type->isa_vectmask()))) {
2450     return false;
2451   }
2452 
2453   Node* op = opd1;
2454   if (is_cast) {
2455     BasicType new_elem_bt_to = elem_bt_to;
2456     BasicType new_elem_bt_from = elem_bt_from;
2457     if (is_mask && is_floating_point_type(elem_bt_from)) {
2458       new_elem_bt_from = elem_bt_from == T_FLOAT ? T_INT : T_LONG;
2459     }
2460     int cast_vopc = VectorCastNode::opcode(new_elem_bt_from);
2461     // Make sure that cast is implemented to particular type/size combination.
2462     if (!arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2463       if (C->print_intrinsics()) {
2464         tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2465                       cast_vopc,
2466                       num_elem_to, type2name(elem_bt_to), is_mask);
2467       }
2468       return false;
2469     }
2470 
2471     if (num_elem_from < num_elem_to) {
2472       // Since input and output number of elements are not consistent, we need to make sure we
2473       // properly size. Thus, first make a cast that retains the number of elements from source.
2474       int num_elem_for_cast = num_elem_from;
2475 
2476       // It is possible that arch does not support this intermediate vector size
2477       // TODO More complex logic required here to handle this corner case for the sizes.
2478       if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2479         if (C->print_intrinsics()) {
2480           tty->print_cr("  ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2481                         cast_vopc,
2482                         num_elem_for_cast, type2name(elem_bt_to), is_mask);
2483         }
2484         return false;
2485       }
2486 
2487       op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2488       // Now ensure that the destination gets properly resized to needed size.
2489       op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2490     } else if (num_elem_from > num_elem_to) {
2491       // Since number elements from input is larger than output, simply reduce size of input (we are supposed to
2492       // drop top elements anyway).
2493       int num_elem_for_resize = num_elem_to;
2494 
2495       // It is possible that arch does not support this intermediate vector size
2496       // TODO More complex logic required here to handle this corner case for the sizes.
2497       if (!arch_supports_vector(Op_VectorReinterpret,
2498                                 num_elem_for_resize,
2499                                 elem_bt_from,
2500                                 VecMaskNotUsed)) {
2501         if (C->print_intrinsics()) {
2502           tty->print_cr("  ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2503                         num_elem_for_resize, type2name(elem_bt_from), is_mask);
2504         }
2505         return false;
2506       }
2507 
2508       op = gvn().transform(new VectorReinterpretNode(op,
2509                                                      src_type,
2510                                                      TypeVect::make(elem_bt_from,
2511                                                                     num_elem_for_resize)));
2512       op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2513     } else {
2514       if (is_mask) {
2515         if ((dst_type->isa_vectmask() && src_type->isa_vectmask()) ||
2516             (type2aelembytes(elem_bt_from) == type2aelembytes(elem_bt_to))) {
2517           op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2518         } else {
2519           op = VectorMaskCastNode::makeCastNode(&gvn(), op, dst_type);
2520         }
2521       } else {
2522         // Since input and output number of elements match, and since we know this vector size is
2523         // supported, simply do a cast with no resize needed.
2524         op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2525       }
2526     }
2527   } else if (Type::cmp(src_type, dst_type) != 0) {
2528     assert(!is_cast, "must be reinterpret");
2529     op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2530   }
2531 
2532   const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2533   Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2534   set_result(vbox);
2535   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2536   return true;
2537 }
2538 
2539 //  public static
2540 //  <V extends Vector<E>,
2541 //   E>
2542 //  V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2543 //           V vec, int ix, long val,
2544 //           VecInsertOp<V> defaultImpl)
2545 bool LibraryCallKit::inline_vector_insert() {
2546   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2547   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2548   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2549   const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
2550 
2551   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || idx == NULL) {
2552     return false; // dead code
2553   }
2554   if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
2555     if (C->print_intrinsics()) {
2556       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2557                     NodeClassNames[argument(0)->Opcode()],
2558                     NodeClassNames[argument(1)->Opcode()],
2559                     NodeClassNames[argument(2)->Opcode()],
2560                     NodeClassNames[argument(4)->Opcode()]);
2561     }
2562     return false; // not enough info for intrinsification
2563   }
2564   if (!is_klass_initialized(vector_klass)) {
2565     if (C->print_intrinsics()) {
2566       tty->print_cr("  ** klass argument not initialized");
2567     }
2568     return false;
2569   }
2570   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2571   if (!elem_type->is_primitive_type()) {
2572     if (C->print_intrinsics()) {
2573       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2574     }
2575     return false; // should be primitive type
2576   }
2577   BasicType elem_bt = elem_type->basic_type();
2578   int num_elem = vlen->get_con();
2579   if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2580     if (C->print_intrinsics()) {
2581       tty->print_cr("  ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2582                     num_elem, type2name(elem_bt));
2583     }
2584     return false; // not supported
2585   }
2586 
2587   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2588   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2589 
2590   Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2591   if (opd == NULL) {
2592     return false;
2593   }
2594 
2595   Node* insert_val = argument(5);
2596   assert(gvn().type(insert_val)->isa_long() != NULL, "expected to be long");
2597 
2598   // Convert insert value back to its appropriate type.
2599   switch (elem_bt) {
2600     case T_BYTE:
2601       insert_val = gvn().transform(new ConvL2INode(insert_val));
2602       insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::BYTE));
2603       break;
2604     case T_SHORT:
2605       insert_val = gvn().transform(new ConvL2INode(insert_val));
2606       insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::SHORT));
2607       break;
2608     case T_INT:
2609       insert_val = gvn().transform(new ConvL2INode(insert_val));
2610       break;
2611     case T_FLOAT:
2612       insert_val = gvn().transform(new ConvL2INode(insert_val));
2613       insert_val = gvn().transform(new MoveI2FNode(insert_val));
2614       break;
2615     case T_DOUBLE:
2616       insert_val = gvn().transform(new MoveL2DNode(insert_val));
2617       break;
2618     case T_LONG:
2619       // no conversion needed
2620       break;
2621     default: fatal("%s", type2name(elem_bt)); break;
2622   }
2623 
2624   Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con()));
2625 
2626   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2627   set_result(vbox);
2628   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2629   return true;
2630 }
2631 
2632 //  public static
2633 //  <V extends Vector<E>,
2634 //   E>
2635 //  long extract(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2636 //               V vec, int ix,
2637 //               VecExtractOp<V> defaultImpl)
2638 bool LibraryCallKit::inline_vector_extract() {
2639   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2640   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2641   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2642   const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
2643 
2644   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || idx == NULL) {
2645     return false; // dead code
2646   }
2647   if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
2648     if (C->print_intrinsics()) {
2649       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2650                     NodeClassNames[argument(0)->Opcode()],
2651                     NodeClassNames[argument(1)->Opcode()],
2652                     NodeClassNames[argument(2)->Opcode()],
2653                     NodeClassNames[argument(4)->Opcode()]);
2654     }
2655     return false; // not enough info for intrinsification
2656   }
2657   if (!is_klass_initialized(vector_klass)) {
2658     if (C->print_intrinsics()) {
2659       tty->print_cr("  ** klass argument not initialized");
2660     }
2661     return false;
2662   }
2663   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2664   if (!elem_type->is_primitive_type()) {
2665     if (C->print_intrinsics()) {
2666       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2667     }
2668     return false; // should be primitive type
2669   }
2670   BasicType elem_bt = elem_type->basic_type();
2671   int num_elem = vlen->get_con();
2672   int vopc = ExtractNode::opcode(elem_bt);
2673   if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2674     if (C->print_intrinsics()) {
2675       tty->print_cr("  ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2676                     num_elem, type2name(elem_bt));
2677     }
2678     return false; // not supported
2679   }
2680 
2681   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2682   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2683 
2684   Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2685   if (opd == NULL) {
2686     return false;
2687   }
2688 
2689   Node* operation = gvn().transform(ExtractNode::make(opd, idx->get_con(), elem_bt));
2690 
2691   Node* bits = NULL;
2692   switch (elem_bt) {
2693     case T_BYTE:
2694     case T_SHORT:
2695     case T_INT: {
2696       bits = gvn().transform(new ConvI2LNode(operation));
2697       break;
2698     }
2699     case T_FLOAT: {
2700       bits = gvn().transform(new MoveF2INode(operation));
2701       bits = gvn().transform(new ConvI2LNode(bits));
2702       break;
2703     }
2704     case T_DOUBLE: {
2705       bits = gvn().transform(new MoveD2LNode(operation));
2706       break;
2707     }
2708     case T_LONG: {
2709       bits = operation; // no conversion needed
2710       break;
2711     }
2712     default: fatal("%s", type2name(elem_bt));
2713   }
2714 
2715   set_result(bits);
2716   return true;
2717 }
2718