1 /*
   2  * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "ci/ciSymbols.hpp"
  27 #include "classfile/vmSymbols.hpp"
  28 #include "opto/library_call.hpp"
  29 #include "opto/runtime.hpp"
  30 #include "opto/vectornode.hpp"
  31 #include "prims/vectorSupport.hpp"
  32 #include "runtime/stubRoutines.hpp"
  33 
  34 #ifdef ASSERT
  35 static bool is_vector(ciKlass* klass) {
  36   return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
  37 }
  38 
  39 static bool check_vbox(const TypeInstPtr* vbox_type) {
  40   assert(vbox_type->klass_is_exact(), "");
  41 
  42   ciInstanceKlass* ik = vbox_type->klass()->as_instance_klass();
  43   assert(is_vector(ik), "not a vector");
  44 
  45   ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
  46   assert(fd1 != NULL, "element type info is missing");
  47 
  48   ciConstant val1 = fd1->constant_value();
  49   BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
  50   assert(is_java_primitive(elem_bt), "element type info is missing");
  51 
  52   ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
  53   assert(fd2 != NULL, "vector length info is missing");
  54 
  55   ciConstant val2 = fd2->constant_value();
  56   assert(val2.as_int() > 0, "vector length info is missing");
  57 
  58   return true;
  59 }
  60 #endif
  61 
  62 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt, bool has_scalar_args) {
  63     bool is_supported = true;
  64     // has_scalar_args flag is true only for non-constant scalar shift count,
  65     // since in this case shift needs to be broadcasted.
  66     if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
  67          (has_scalar_args &&
  68            !arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed))) {
  69       is_supported = false;
  70     }
  71 
  72     int lshiftopc, rshiftopc;
  73     switch(elem_bt) {
  74       case T_BYTE:
  75         lshiftopc = Op_LShiftI;
  76         rshiftopc = Op_URShiftB;
  77         break;
  78       case T_SHORT:
  79         lshiftopc = Op_LShiftI;
  80         rshiftopc = Op_URShiftS;
  81         break;
  82       case T_INT:
  83         lshiftopc = Op_LShiftI;
  84         rshiftopc = Op_URShiftI;
  85         break;
  86       case T_LONG:
  87         lshiftopc = Op_LShiftL;
  88         rshiftopc = Op_URShiftL;
  89         break;
  90       default:
  91         assert(false, "Unexpected type");
  92     }
  93     int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
  94     int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
  95     if (!is_supported &&
  96         arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
  97         arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
  98         arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
  99       is_supported = true;
 100     }
 101     return is_supported;
 102 }
 103 
 104 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
 105   assert(EnableVectorSupport, "");
 106 
 107   PreserveReexecuteState preexecs(this);
 108   jvms()->set_should_reexecute(true);
 109 
 110   VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
 111   set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
 112   make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
 113   set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
 114   set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
 115   Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
 116 
 117   assert(check_vbox(vbox_type), "");
 118   const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
 119   VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
 120   return gvn().transform(vbox);
 121 }
 122 
 123 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) {
 124   assert(EnableVectorSupport, "");
 125   const TypeInstPtr* vbox_type_v = gvn().type(v)->is_instptr();
 126   if (vbox_type->klass() != vbox_type_v->klass()) {
 127     return NULL; // arguments don't agree on vector shapes
 128   }
 129   if (vbox_type_v->maybe_null()) {
 130     return NULL; // no nulls are allowed
 131   }
 132   assert(check_vbox(vbox_type), "");
 133   const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
 134   Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector));
 135   return unbox;
 136 }
 137 
 138 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
 139   assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
 140   juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
 141   Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
 142   Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
 143   return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
 144 }
 145 
 146 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
 147   // Check that the operation is valid.
 148   if (sopc <= 0) {
 149 #ifndef PRODUCT
 150     if (C->print_intrinsics()) {
 151       tty->print_cr("  ** Rejected intrinsification because no valid vector op could be extracted");
 152     }
 153 #endif
 154     return false;
 155   }
 156 
 157   if (VectorNode::is_vector_rotate(sopc)) {
 158     if(!arch_supports_vector_rotate(sopc, num_elem, type, has_scalar_args)) {
 159 #ifndef PRODUCT
 160       if (C->print_intrinsics()) {
 161         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
 162                       NodeClassNames[sopc], type2name(type), num_elem);
 163       }
 164 #endif
 165       return false;
 166     }
 167   } else {
 168     // Check that architecture supports this op-size-type combination.
 169     if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
 170 #ifndef PRODUCT
 171       if (C->print_intrinsics()) {
 172         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support it",
 173                       NodeClassNames[sopc], type2name(type), num_elem);
 174       }
 175 #endif
 176       return false;
 177     } else {
 178       assert(Matcher::match_rule_supported(sopc), "must be supported");
 179     }
 180   }
 181 
 182   if (num_elem == 1) {
 183     if (mask_use_type != VecMaskNotUsed) {
 184 #ifndef PRODUCT
 185       if (C->print_intrinsics()) {
 186         tty->print_cr("  ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
 187                       NodeClassNames[sopc], type2name(type), num_elem);
 188       }
 189 #endif
 190       return false;
 191     }
 192 
 193     if (sopc != 0) {
 194       if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
 195 #ifndef PRODUCT
 196         if (C->print_intrinsics()) {
 197           tty->print_cr("  ** Not a svml call or load/store vector op (%s,%s,%d)",
 198                         NodeClassNames[sopc], type2name(type), num_elem);
 199         }
 200 #endif
 201         return false;
 202       }
 203     }
 204   }
 205 
 206   if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
 207       Matcher::supports_vector_variable_shifts() == false) {
 208     if (C->print_intrinsics()) {
 209       tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
 210                     NodeClassNames[sopc], type2name(type), num_elem);
 211     }
 212     return false;
 213   }
 214 
 215   // Check whether mask unboxing is supported.
 216   if (mask_use_type == VecMaskUseAll || mask_use_type == VecMaskUseLoad) {
 217     if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type)) {
 218     #ifndef PRODUCT
 219       if (C->print_intrinsics()) {
 220         tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
 221                       NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
 222       }
 223     #endif
 224       return false;
 225     }
 226   }
 227 
 228   // Check whether mask boxing is supported.
 229   if (mask_use_type == VecMaskUseAll || mask_use_type == VecMaskUseStore) {
 230     if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type)) {
 231     #ifndef PRODUCT
 232       if (C->print_intrinsics()) {
 233         tty->print_cr("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
 234                       NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
 235       }
 236     #endif
 237       return false;
 238     }
 239   }
 240 
 241   return true;
 242 }
 243 
 244 static bool is_vector_mask(ciKlass* klass) {
 245   return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
 246 }
 247 
 248 static bool is_vector_shuffle(ciKlass* klass) {
 249   return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
 250 }
 251 
 252 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
 253   if (vec_klass->const_oop() == NULL) {
 254     return false; // uninitialized or some kind of unsafe access
 255   }
 256   assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != NULL, "klass instance expected");
 257   ciInstanceKlass* klass =  vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
 258   return klass->is_initialized();
 259 }
 260 
 261 // public static
 262 // <VM>
 263 // VM unaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
 264 //            VM vm,
 265 //            Function<VM, VM> defaultImpl) {
 266 //
 267 // public static
 268 // <VM>
 269 // VM binaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
 270 //             VM vm1, VM vm2,
 271 //             BiFunction<VM, VM, VM> defaultImpl) {
 272 //
 273 // public static
 274 // <VM>
 275 // VM ternaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
 276 //              VM vm1, VM vm2, VM vm3,
 277 //              TernaryOperation<VM> defaultImpl) {
 278 //
 279 bool LibraryCallKit::inline_vector_nary_operation(int n) {
 280   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
 281   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
 282   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
 283   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
 284 
 285   if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
 286       !opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 287     if (C->print_intrinsics()) {
 288       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
 289                     NodeClassNames[argument(0)->Opcode()],
 290                     NodeClassNames[argument(1)->Opcode()],
 291                     NodeClassNames[argument(2)->Opcode()],
 292                     NodeClassNames[argument(3)->Opcode()]);
 293     }
 294     return false; // not enough info for intrinsification
 295   }
 296   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 297   if (!elem_type->is_primitive_type()) {
 298     if (C->print_intrinsics()) {
 299       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 300     }
 301     return false; // should be primitive type
 302   }
 303   if (!is_klass_initialized(vector_klass)) {
 304     if (C->print_intrinsics()) {
 305       tty->print_cr("  ** klass argument not initialized");
 306     }
 307     return false;
 308   }
 309   BasicType elem_bt = elem_type->basic_type();
 310   int num_elem = vlen->get_con();
 311   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
 312   int sopc = VectorNode::opcode(opc, elem_bt);
 313   if ((opc != Op_CallLeafVector) && (sopc == 0)) {
 314     if (C->print_intrinsics()) {
 315       tty->print_cr("  ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
 316     }
 317     return false; // operation not supported
 318   }
 319   if (num_elem == 1) {
 320     if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
 321       if (C->print_intrinsics()) {
 322         tty->print_cr("  ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
 323                       n, opc, num_elem, type2name(elem_bt));
 324       }
 325       return false;
 326     }
 327   }
 328   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 329   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 330 
 331   if (opc == Op_CallLeafVector) {
 332     if (!UseVectorStubs) {
 333       if (C->print_intrinsics()) {
 334         tty->print_cr("  ** vector stubs support is disabled");
 335       }
 336       return false;
 337     }
 338     if (!Matcher::supports_vector_calling_convention()) {
 339       if (C->print_intrinsics()) {
 340         tty->print_cr("  ** no vector calling conventions supported");
 341       }
 342       return false;
 343     }
 344     if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
 345       if (C->print_intrinsics()) {
 346         tty->print_cr("  ** vector size (vlen=%d, etype=%s) is not supported",
 347                       num_elem, type2name(elem_bt));
 348       }
 349       return false;
 350     }
 351   }
 352 
 353   // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
 354   if ((sopc != 0) &&
 355       !arch_supports_vector(sopc, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseAll : VecMaskNotUsed)) {
 356     if (C->print_intrinsics()) {
 357       tty->print_cr("  ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d",
 358                     n, sopc, num_elem, type2name(elem_bt),
 359                     is_vector_mask(vbox_klass) ? 1 : 0);
 360     }
 361     return false; // not supported
 362   }
 363 
 364   Node* opd1 = NULL; Node* opd2 = NULL; Node* opd3 = NULL;
 365   switch (n) {
 366     case 3: {
 367       opd3 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
 368       if (opd3 == NULL) {
 369         if (C->print_intrinsics()) {
 370           tty->print_cr("  ** unbox failed v3=%s",
 371                         NodeClassNames[argument(6)->Opcode()]);
 372         }
 373         return false;
 374       }
 375       // fall-through
 376     }
 377     case 2: {
 378       opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
 379       if (opd2 == NULL) {
 380         if (C->print_intrinsics()) {
 381           tty->print_cr("  ** unbox failed v2=%s",
 382                         NodeClassNames[argument(5)->Opcode()]);
 383         }
 384         return false;
 385       }
 386       // fall-through
 387     }
 388     case 1: {
 389       opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
 390       if (opd1 == NULL) {
 391         if (C->print_intrinsics()) {
 392           tty->print_cr("  ** unbox failed v1=%s",
 393                         NodeClassNames[argument(4)->Opcode()]);
 394         }
 395         return false;
 396       }
 397       break;
 398     }
 399     default: fatal("unsupported arity: %d", n);
 400   }
 401 
 402   Node* operation = NULL;
 403   if (opc == Op_CallLeafVector) {
 404     assert(UseVectorStubs, "sanity");
 405     operation = gen_call_to_svml(opr->get_con(), elem_bt, num_elem, opd1, opd2);
 406     if (operation == NULL) {
 407       if (C->print_intrinsics()) {
 408         tty->print_cr("  ** svml call failed for %s_%s_%d",
 409                          (elem_bt == T_FLOAT)?"float":"double",
 410                          VectorSupport::svmlname[opr->get_con() - VectorSupport::VECTOR_OP_SVML_START],
 411                          num_elem * type2aelembytes(elem_bt));
 412       }
 413       return false;
 414      }
 415   } else {
 416     const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
 417     switch (n) {
 418       case 1:
 419       case 2: {
 420         operation = gvn().transform(VectorNode::make(sopc, opd1, opd2, vt));
 421         break;
 422       }
 423       case 3: {
 424         operation = gvn().transform(VectorNode::make(sopc, opd1, opd2, opd3, vt));
 425         break;
 426       }
 427       default: fatal("unsupported arity: %d", n);
 428     }
 429   }
 430   // Wrap it up in VectorBox to keep object type information.
 431   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
 432   set_result(vbox);
 433   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 434   return true;
 435 }
 436 
 437 // <Sh extends VectorShuffle<E>,  E>
 438 //  Sh ShuffleIota(Class<?> E, Class<?> ShuffleClass, Vector.Species<E> s, int length,
 439 //                  int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
 440 bool LibraryCallKit::inline_vector_shuffle_iota() {
 441   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
 442   const TypeInt*     vlen          = gvn().type(argument(3))->isa_int();
 443   const TypeInt*     start_val     = gvn().type(argument(4))->isa_int();
 444   const TypeInt*     step_val      = gvn().type(argument(5))->isa_int();
 445   const TypeInt*     wrap          = gvn().type(argument(6))->isa_int();
 446 
 447   Node* start = argument(4);
 448   Node* step  = argument(5);
 449 
 450   if (shuffle_klass == NULL || vlen == NULL || start_val == NULL || step_val == NULL || wrap == NULL) {
 451     return false; // dead code
 452   }
 453   if (!vlen->is_con() || !is_power_of_2(vlen->get_con()) ||
 454       shuffle_klass->const_oop() == NULL || !wrap->is_con()) {
 455     return false; // not enough info for intrinsification
 456   }
 457   if (!is_klass_initialized(shuffle_klass)) {
 458     if (C->print_intrinsics()) {
 459       tty->print_cr("  ** klass argument not initialized");
 460     }
 461     return false;
 462   }
 463 
 464   int do_wrap = wrap->get_con();
 465   int num_elem = vlen->get_con();
 466   BasicType elem_bt = T_BYTE;
 467 
 468   if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed)) {
 469     return false;
 470   }
 471   if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed)) {
 472     return false;
 473   }
 474   if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) {
 475     return false;
 476   }
 477   if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
 478     return false;
 479   }
 480   if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
 481     return false;
 482   }
 483   if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
 484     return false;
 485   }
 486 
 487   const Type * type_bt = Type::get_const_basic_type(elem_bt);
 488   const TypeVect * vt  = TypeVect::make(type_bt, num_elem);
 489 
 490   Node* res =  gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
 491 
 492   if(!step_val->is_con() || !is_power_of_2(step_val->get_con())) {
 493     Node* bcast_step     = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt));
 494     res = gvn().transform(VectorNode::make(Op_MulI, res, bcast_step, num_elem, elem_bt));
 495   } else if (step_val->get_con() > 1) {
 496     Node* cnt = gvn().makecon(TypeInt::make(log2i_exact(step_val->get_con())));
 497     Node* shift_cnt = vector_shift_count(cnt, Op_LShiftI, elem_bt, num_elem);
 498     res = gvn().transform(VectorNode::make(Op_LShiftVB, res, shift_cnt, vt));
 499   }
 500 
 501   if (!start_val->is_con() || start_val->get_con() != 0) {
 502     Node* bcast_start    = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt));
 503     res = gvn().transform(VectorNode::make(Op_AddI, res, bcast_start, num_elem, elem_bt));
 504   }
 505 
 506   Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1));
 507   Node * bcast_mod  = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt));
 508   if(do_wrap)  {
 509     // Wrap the indices greater than lane count.
 510     res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
 511   } else {
 512     ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(1));
 513     Node * lane_cnt  = gvn().makecon(TypeInt::make(num_elem));
 514     Node * bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt));
 515     Node* mask = gvn().transform(new VectorMaskCmpNode(BoolTest::ge, bcast_lane_cnt, res, pred_node, vt));
 516 
 517     // Make the indices greater than lane count as -ve values. This matches the java side implementation.
 518     res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
 519     Node * biased_val = gvn().transform(VectorNode::make(Op_SubI, res, bcast_lane_cnt, num_elem, elem_bt));
 520     res = gvn().transform(new VectorBlendNode(biased_val, res, mask));
 521   }
 522 
 523   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 524   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 525 
 526   // Wrap it up in VectorBox to keep object type information.
 527   res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
 528   set_result(res);
 529   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 530   return true;
 531 }
 532 
 533 // <E, M>
 534 // int maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
 535 //                          int length, M m, VectorMaskOp<M> defaultImpl)
 536 bool LibraryCallKit::inline_vector_mask_operation() {
 537   const TypeInt*     oper       = gvn().type(argument(0))->isa_int();
 538   const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
 539   const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
 540   const TypeInt*     vlen       = gvn().type(argument(3))->isa_int();
 541   Node*              mask       = argument(4);
 542 
 543   if (mask_klass == NULL || elem_klass == NULL || mask->is_top() || vlen == NULL) {
 544     return false; // dead code
 545   }
 546 
 547   if (!is_klass_initialized(mask_klass)) {
 548     if (C->print_intrinsics()) {
 549       tty->print_cr("  ** klass argument not initialized");
 550     }
 551     return false;
 552   }
 553 
 554   int num_elem = vlen->get_con();
 555   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 556   BasicType elem_bt = elem_type->basic_type();
 557 
 558   if (!arch_supports_vector(Op_LoadVector, num_elem, T_BOOLEAN, VecMaskNotUsed)) {
 559     if (C->print_intrinsics()) {
 560       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 561                     Op_LoadVector, num_elem, type2name(T_BOOLEAN));
 562     }
 563     return false; // not supported
 564   }
 565 
 566   int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
 567   if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskNotUsed)) {
 568     if (C->print_intrinsics()) {
 569       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 570                     mopc, num_elem, type2name(elem_bt));
 571     }
 572     return false; // not supported
 573   }
 574 
 575   const Type* elem_ty = Type::get_const_basic_type(elem_bt);
 576   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
 577   const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
 578   Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
 579   Node* store_mask = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
 580   Node* maskoper = gvn().transform(VectorMaskOpNode::make(store_mask, TypeInt::INT, mopc));
 581   set_result(maskoper);
 582 
 583   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 584   return true;
 585 }
 586 
 587 // <VM ,Sh extends VectorShuffle<E>, E>
 588 // VM shuffleToVector(Class<VM> VecClass, Class<?>E , Class<?> ShuffleClass, Sh s, int length,
 589 //                    ShuffleToVectorOperation<VM,Sh,E> defaultImpl)
 590 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
 591   const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
 592   const TypeInstPtr* elem_klass    = gvn().type(argument(1))->isa_instptr();
 593   const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
 594   Node*              shuffle       = argument(3);
 595   const TypeInt*     vlen          = gvn().type(argument(4))->isa_int();
 596 
 597   if (vector_klass == NULL || elem_klass == NULL || shuffle_klass == NULL || shuffle->is_top() || vlen == NULL) {
 598     return false; // dead code
 599   }
 600   if (!vlen->is_con() || vector_klass->const_oop() == NULL || shuffle_klass->const_oop() == NULL) {
 601     return false; // not enough info for intrinsification
 602   }
 603   if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
 604     if (C->print_intrinsics()) {
 605       tty->print_cr("  ** klass argument not initialized");
 606     }
 607     return false;
 608   }
 609 
 610   int num_elem = vlen->get_con();
 611   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 612   BasicType elem_bt = elem_type->basic_type();
 613 
 614   if (num_elem < 4) {
 615     return false;
 616   }
 617 
 618   int cast_vopc = VectorCastNode::opcode(T_BYTE); // from shuffle of type T_BYTE
 619   // Make sure that cast is implemented to particular type/size combination.
 620   if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
 621     if (C->print_intrinsics()) {
 622       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 623         cast_vopc, num_elem, type2name(elem_bt));
 624     }
 625     return false;
 626   }
 627 
 628   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 629   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 630 
 631   // Unbox shuffle with true flag to indicate its load shuffle to vector
 632   // shuffle is a byte array
 633   Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, T_BYTE, num_elem, true);
 634 
 635   // cast byte to target element type
 636   shuffle_vec = gvn().transform(VectorCastNode::make(cast_vopc, shuffle_vec, elem_bt, num_elem));
 637 
 638   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 639   const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 640 
 641   // Box vector
 642   Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem);
 643   set_result(res);
 644   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 645   return true;
 646 }
 647 
 648 // <V extends Vector<?,?>>
 649 // V broadcastCoerced(Class<?> vectorClass, Class<?> elementType, int vlen,
 650 //                    long bits,
 651 //                    LongFunction<V> defaultImpl)
 652 bool LibraryCallKit::inline_vector_broadcast_coerced() {
 653   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 654   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 655   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 656 
 657   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
 658       vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 659     if (C->print_intrinsics()) {
 660       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
 661                     NodeClassNames[argument(0)->Opcode()],
 662                     NodeClassNames[argument(1)->Opcode()],
 663                     NodeClassNames[argument(2)->Opcode()]);
 664     }
 665     return false; // not enough info for intrinsification
 666   }
 667 
 668   if (!is_klass_initialized(vector_klass)) {
 669     if (C->print_intrinsics()) {
 670       tty->print_cr("  ** klass argument not initialized");
 671     }
 672     return false;
 673   }
 674   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 675   if (!elem_type->is_primitive_type()) {
 676     if (C->print_intrinsics()) {
 677       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 678     }
 679     return false; // should be primitive type
 680   }
 681   BasicType elem_bt = elem_type->basic_type();
 682   int num_elem = vlen->get_con();
 683   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 684   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 685 
 686   // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
 687   if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt,
 688                             (is_vector_mask(vbox_klass) ? VecMaskUseStore : VecMaskNotUsed), true /*has_scalar_args*/)) {
 689     if (C->print_intrinsics()) {
 690       tty->print_cr("  ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d",
 691                     num_elem, type2name(elem_bt),
 692                     is_vector_mask(vbox_klass) ? 1 : 0);
 693     }
 694     return false; // not supported
 695   }
 696 
 697   Node* bits = argument(3); // long
 698 
 699   Node* elem = NULL;
 700   switch (elem_bt) {
 701     case T_BOOLEAN: // fall-through
 702     case T_BYTE:    // fall-through
 703     case T_SHORT:   // fall-through
 704     case T_CHAR:    // fall-through
 705     case T_INT: {
 706       elem = gvn().transform(new ConvL2INode(bits));
 707       break;
 708     }
 709     case T_DOUBLE: {
 710       elem = gvn().transform(new MoveL2DNode(bits));
 711       break;
 712     }
 713     case T_FLOAT: {
 714       bits = gvn().transform(new ConvL2INode(bits));
 715       elem = gvn().transform(new MoveI2FNode(bits));
 716       break;
 717     }
 718     case T_LONG: {
 719       elem = bits; // no conversion needed
 720       break;
 721     }
 722     default: fatal("%s", type2name(elem_bt));
 723   }
 724 
 725   Node* broadcast = VectorNode::scalar2vector(elem, num_elem, Type::get_const_basic_type(elem_bt));
 726   broadcast = gvn().transform(broadcast);
 727 
 728   Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
 729   set_result(box);
 730   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 731   return true;
 732 }
 733 
 734 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type) {
 735   assert(arr_type != NULL, "unexpected");
 736   BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
 737   if (elem_bt == arr_elem_bt) {
 738     return true;
 739   } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
 740     // Load/store of short vector from/to char[] is supported
 741     return true;
 742   } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
 743     // Load/store of byte vector from/to boolean[] is supported
 744     return true;
 745   } else {
 746     return false;
 747   }
 748 }
 749 
 750 //    <C, V extends Vector<?,?>>
 751 //    V load(Class<?> vectorClass, Class<?> elementType, int vlen,
 752 //           Object base, long offset,
 753 //           /* Vector.Mask<E,S> m*/
 754 //           Object container, int index,
 755 //           LoadOperation<C, VM> defaultImpl) {
 756 //
 757 //    <C, V extends Vector<?,?>>
 758 //    void store(Class<?> vectorClass, Class<?> elementType, int vlen,
 759 //               Object base, long offset,
 760 //               V v, /*Vector.Mask<E,S> m*/
 761 //               Object container, int index,
 762 //               StoreVectorOperation<C, V> defaultImpl) {
 763 
 764 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
 765   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 766   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 767   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 768 
 769   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
 770       vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 771     if (C->print_intrinsics()) {
 772       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
 773                     NodeClassNames[argument(0)->Opcode()],
 774                     NodeClassNames[argument(1)->Opcode()],
 775                     NodeClassNames[argument(2)->Opcode()]);
 776     }
 777     return false; // not enough info for intrinsification
 778   }
 779   if (!is_klass_initialized(vector_klass)) {
 780     if (C->print_intrinsics()) {
 781       tty->print_cr("  ** klass argument not initialized");
 782     }
 783     return false;
 784   }
 785 
 786   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 787   if (!elem_type->is_primitive_type()) {
 788     if (C->print_intrinsics()) {
 789       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 790     }
 791     return false; // should be primitive type
 792   }
 793   BasicType elem_bt = elem_type->basic_type();
 794   int num_elem = vlen->get_con();
 795 
 796   // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
 797   if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
 798     if (C->print_intrinsics()) {
 799       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
 800                     is_store, is_store ? "store" : "load",
 801                     num_elem, type2name(elem_bt));
 802     }
 803     return false; // not supported
 804   }
 805 
 806   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 807   bool is_mask = is_vector_mask(vbox_klass);
 808 
 809   Node* base = argument(3);
 810   Node* offset = ConvL2X(argument(4));
 811 
 812   // Save state and restore on bailout
 813   uint old_sp = sp();
 814   SafePointNode* old_map = clone_map();
 815 
 816   Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
 817   // Can base be NULL? Otherwise, always on-heap access.
 818   bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
 819 
 820   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
 821   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
 822 
 823   // Now handle special case where load/store happens from/to byte array but element type is not byte.
 824   bool using_byte_array = arr_type != NULL && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE;
 825   // Handle loading masks.
 826   // If there is no consistency between array and vector element types, it must be special byte array case or loading masks
 827   if (arr_type != NULL && !using_byte_array && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type)) {
 828     if (C->print_intrinsics()) {
 829       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
 830                     is_store, is_store ? "store" : "load",
 831                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
 832     }
 833     set_map(old_map);
 834     set_sp(old_sp);
 835     return false;
 836   }
 837   // Since we are using byte array, we need to double check that the byte operations are supported by backend.
 838   if (using_byte_array) {
 839     int byte_num_elem = num_elem * type2aelembytes(elem_bt);
 840     if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, byte_num_elem, T_BYTE, VecMaskNotUsed)
 841         || !arch_supports_vector(Op_VectorReinterpret, byte_num_elem, T_BYTE, VecMaskNotUsed)) {
 842       if (C->print_intrinsics()) {
 843         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
 844                       is_store, is_store ? "store" : "load",
 845                       byte_num_elem, type2name(elem_bt));
 846       }
 847       set_map(old_map);
 848       set_sp(old_sp);
 849       return false; // not supported
 850     }
 851   }
 852   if (is_mask) {
 853     if (!arch_supports_vector(Op_LoadVector, num_elem, T_BOOLEAN, VecMaskNotUsed)) {
 854       if (C->print_intrinsics()) {
 855         tty->print_cr("  ** not supported: arity=%d op=%s/mask vlen=%d etype=bit ismask=no",
 856                       is_store, is_store ? "store" : "load",
 857                       num_elem);
 858       }
 859       set_map(old_map);
 860       set_sp(old_sp);
 861       return false; // not supported
 862     }
 863     if (!is_store) {
 864       if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
 865         set_map(old_map);
 866         set_sp(old_sp);
 867         return false; // not supported
 868       }
 869     } else {
 870       if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
 871         set_map(old_map);
 872         set_sp(old_sp);
 873         return false; // not supported
 874       }
 875     }
 876   }
 877 
 878   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 879 
 880   if (can_access_non_heap) {
 881     insert_mem_bar(Op_MemBarCPUOrder);
 882   }
 883 
 884   if (is_store) {
 885     Node* val = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
 886     if (val == NULL) {
 887       set_map(old_map);
 888       set_sp(old_sp);
 889       return false; // operand unboxing failed
 890     }
 891     set_all_memory(reset_memory());
 892 
 893     // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
 894     int store_num_elem = num_elem;
 895     if (using_byte_array) {
 896       store_num_elem = num_elem * type2aelembytes(elem_bt);
 897       const TypeVect* to_vect_type = TypeVect::make(T_BYTE, store_num_elem);
 898       val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
 899     }
 900 
 901     Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
 902     set_memory(vstore, addr_type);
 903   } else {
 904     // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
 905     Node* vload = NULL;
 906     if (using_byte_array) {
 907       int load_num_elem = num_elem * type2aelembytes(elem_bt);
 908       vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, load_num_elem, T_BYTE));
 909       const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
 910       vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
 911     } else {
 912       // Special handle for masks
 913       if (is_mask) {
 914         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
 915         const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
 916         vload = gvn().transform(new VectorLoadMaskNode(vload, to_vect_type));
 917       } else {
 918         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
 919       }
 920     }
 921     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
 922     set_result(box);
 923   }
 924 
 925   old_map->destruct(&_gvn);
 926 
 927   if (can_access_non_heap) {
 928     insert_mem_bar(Op_MemBarCPUOrder);
 929   }
 930 
 931   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 932   return true;
 933 }
 934 
 935 //   <C, V extends Vector<?>, W extends IntVector, E, S extends VectorSpecies<E>>
 936 //   void loadWithMap(Class<?> vectorClass, Class<E> E, int length, Class<?> vectorIndexClass,
 937 //                    Object base, long offset, // Unsafe addressing
 938 //                    W index_vector,
 939 //                    C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
 940 //                    LoadVectorOperationWithMap<C, V, E, S> defaultImpl)
 941 //
 942 //    <C, V extends Vector<?>, W extends IntVector>
 943 //    void storeWithMap(Class<?> vectorClass, Class<?> elementType, int length, Class<?> vectorIndexClass,
 944 //                      Object base, long offset,    // Unsafe addressing
 945 //                      W index_vector, V v,
 946 //                      C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
 947 //                      StoreVectorOperationWithMap<C, V> defaultImpl) {
 948 //
 949 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
 950   const TypeInstPtr* vector_klass     = gvn().type(argument(0))->isa_instptr();
 951   const TypeInstPtr* elem_klass       = gvn().type(argument(1))->isa_instptr();
 952   const TypeInt*     vlen             = gvn().type(argument(2))->isa_int();
 953   const TypeInstPtr* vector_idx_klass = gvn().type(argument(3))->isa_instptr();
 954 
 955   if (vector_klass == NULL || elem_klass == NULL || vector_idx_klass == NULL || vlen == NULL ||
 956       vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || vector_idx_klass->const_oop() == NULL || !vlen->is_con()) {
 957     if (C->print_intrinsics()) {
 958       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
 959                     NodeClassNames[argument(0)->Opcode()],
 960                     NodeClassNames[argument(1)->Opcode()],
 961                     NodeClassNames[argument(2)->Opcode()],
 962                     NodeClassNames[argument(3)->Opcode()]);
 963     }
 964     return false; // not enough info for intrinsification
 965   }
 966 
 967   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
 968     if (C->print_intrinsics()) {
 969       tty->print_cr("  ** klass argument not initialized");
 970     }
 971     return false;
 972   }
 973   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 974   if (!elem_type->is_primitive_type()) {
 975     if (C->print_intrinsics()) {
 976       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 977     }
 978     return false; // should be primitive type
 979   }
 980   BasicType elem_bt = elem_type->basic_type();
 981   int num_elem = vlen->get_con();
 982 
 983   if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
 984     if (C->print_intrinsics()) {
 985       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
 986                     is_scatter, is_scatter ? "scatter" : "gather",
 987                     num_elem, type2name(elem_bt));
 988     }
 989     return false; // not supported
 990   }
 991 
 992   // Check that the vector holding indices is supported by architecture
 993   if (!arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
 994       if (C->print_intrinsics()) {
 995         tty->print_cr("  ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int ismask=no",
 996                       is_scatter, is_scatter ? "scatter" : "gather",
 997                       num_elem);
 998       }
 999       return false; // not supported
1000     }
1001 
1002   Node* base = argument(4);
1003   Node* offset = ConvL2X(argument(5));
1004 
1005   // Save state and restore on bailout
1006   uint old_sp = sp();
1007   SafePointNode* old_map = clone_map();
1008 
1009   Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1010 
1011   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1012   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1013 
1014   // The array must be consistent with vector type
1015   if (arr_type == NULL || (arr_type != NULL && !elem_consistent_with_arr(elem_bt, arr_type))) {
1016     if (C->print_intrinsics()) {
1017       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1018                     is_scatter, is_scatter ? "scatter" : "gather",
1019                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1020     }
1021     set_map(old_map);
1022     set_sp(old_sp);
1023     return false;
1024   }
1025   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1026   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1027 
1028   ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1029 
1030   if (vbox_idx_klass == NULL) {
1031     set_map(old_map);
1032     set_sp(old_sp);
1033     return false;
1034   }
1035 
1036   const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1037 
1038   Node* index_vect = unbox_vector(argument(7), vbox_idx_type, T_INT, num_elem);
1039   if (index_vect == NULL) {
1040     set_map(old_map);
1041     set_sp(old_sp);
1042     return false;
1043   }
1044   const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1045   if (is_scatter) {
1046     Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
1047     if (val == NULL) {
1048       set_map(old_map);
1049       set_sp(old_sp);
1050       return false; // operand unboxing failed
1051     }
1052     set_all_memory(reset_memory());
1053 
1054     Node* vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1055     set_memory(vstore, addr_type);
1056   } else {
1057     Node* vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1058 
1059     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1060     set_result(box);
1061   }
1062 
1063   old_map->destruct(&_gvn);
1064 
1065   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1066   return true;
1067 }
1068 
1069 // <V extends Vector<?,?>>
1070 // long reductionCoerced(int oprId, Class<?> vectorClass, Class<?> elementType, int vlen,
1071 //                       V v,
1072 //                       Function<V,Long> defaultImpl)
1073 
1074 bool LibraryCallKit::inline_vector_reduction() {
1075   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
1076   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1077   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1078   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1079 
1080   if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
1081       !opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1082     if (C->print_intrinsics()) {
1083       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1084                     NodeClassNames[argument(0)->Opcode()],
1085                     NodeClassNames[argument(1)->Opcode()],
1086                     NodeClassNames[argument(2)->Opcode()],
1087                     NodeClassNames[argument(3)->Opcode()]);
1088     }
1089     return false; // not enough info for intrinsification
1090   }
1091   if (!is_klass_initialized(vector_klass)) {
1092     if (C->print_intrinsics()) {
1093       tty->print_cr("  ** klass argument not initialized");
1094     }
1095     return false;
1096   }
1097   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1098   if (!elem_type->is_primitive_type()) {
1099     if (C->print_intrinsics()) {
1100       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1101     }
1102     return false; // should be primitive type
1103   }
1104   BasicType elem_bt = elem_type->basic_type();
1105   int num_elem = vlen->get_con();
1106 
1107   int opc  = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1108   int sopc = ReductionNode::opcode(opc, elem_bt);
1109 
1110   // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
1111   if (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed)) {
1112     if (C->print_intrinsics()) {
1113       tty->print_cr("  ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s ismask=no",
1114                     sopc, num_elem, type2name(elem_bt));
1115     }
1116     return false;
1117   }
1118 
1119   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1120   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1121 
1122   Node* opd = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1123   if (opd == NULL) {
1124     return false; // operand unboxing failed
1125   }
1126 
1127   Node* init = ReductionNode::make_reduction_input(gvn(), opc, elem_bt);
1128   Node* rn = gvn().transform(ReductionNode::make(opc, NULL, init, opd, elem_bt));
1129 
1130   Node* bits = NULL;
1131   switch (elem_bt) {
1132     case T_BYTE:
1133     case T_SHORT:
1134     case T_INT: {
1135       bits = gvn().transform(new ConvI2LNode(rn));
1136       break;
1137     }
1138     case T_FLOAT: {
1139       rn   = gvn().transform(new MoveF2INode(rn));
1140       bits = gvn().transform(new ConvI2LNode(rn));
1141       break;
1142     }
1143     case T_DOUBLE: {
1144       bits = gvn().transform(new MoveD2LNode(rn));
1145       break;
1146     }
1147     case T_LONG: {
1148       bits = rn; // no conversion needed
1149       break;
1150     }
1151     default: fatal("%s", type2name(elem_bt));
1152   }
1153   set_result(bits);
1154   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1155   return true;
1156 }
1157 
1158 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1159 //                                V v1, V v2,
1160 //                                BiFunction<V, V, Boolean> defaultImpl) {
1161 //
1162 bool LibraryCallKit::inline_vector_test() {
1163   const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1164   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1165   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1166   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1167 
1168   if (cond == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
1169       !cond->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1170     if (C->print_intrinsics()) {
1171       tty->print_cr("  ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1172                     NodeClassNames[argument(0)->Opcode()],
1173                     NodeClassNames[argument(1)->Opcode()],
1174                     NodeClassNames[argument(2)->Opcode()],
1175                     NodeClassNames[argument(3)->Opcode()]);
1176     }
1177     return false; // not enough info for intrinsification
1178   }
1179   if (!is_klass_initialized(vector_klass)) {
1180     if (C->print_intrinsics()) {
1181       tty->print_cr("  ** klass argument not initialized");
1182     }
1183     return false;
1184   }
1185   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1186   if (!elem_type->is_primitive_type()) {
1187     if (C->print_intrinsics()) {
1188       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1189     }
1190     return false; // should be primitive type
1191   }
1192   BasicType elem_bt = elem_type->basic_type();
1193   int num_elem = vlen->get_con();
1194   BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1195   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1196   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1197 
1198   if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1199     if (C->print_intrinsics()) {
1200       tty->print_cr("  ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1201                     cond->get_con(), num_elem, type2name(elem_bt),
1202                     is_vector_mask(vbox_klass));
1203     }
1204     return false;
1205   }
1206 
1207   Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1208   Node* opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1209   if (opd1 == NULL || opd2 == NULL) {
1210     return false; // operand unboxing failed
1211   }
1212   Node* test = new VectorTestNode(opd1, opd2, booltest);
1213   test = gvn().transform(test);
1214 
1215   set_result(test);
1216   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1217   return true;
1218 }
1219 
1220 // public static
1221 // <V extends Vector, M extends Mask>
1222 // V blend(Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int vlen,
1223 //         V v1, V v2, M m,
1224 //         VectorBlendOp<V,M> defaultImpl) { ...
1225 //
1226 bool LibraryCallKit::inline_vector_blend() {
1227   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1228   const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1229   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1230   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1231 
1232   if (mask_klass == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL) {
1233     return false; // dead code
1234   }
1235   if (mask_klass->const_oop() == NULL || vector_klass->const_oop() == NULL ||
1236       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1237     if (C->print_intrinsics()) {
1238       tty->print_cr("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1239                     NodeClassNames[argument(0)->Opcode()],
1240                     NodeClassNames[argument(1)->Opcode()],
1241                     NodeClassNames[argument(2)->Opcode()],
1242                     NodeClassNames[argument(3)->Opcode()]);
1243     }
1244     return false; // not enough info for intrinsification
1245   }
1246   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1247     if (C->print_intrinsics()) {
1248       tty->print_cr("  ** klass argument not initialized");
1249     }
1250     return false;
1251   }
1252   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1253   if (!elem_type->is_primitive_type()) {
1254     if (C->print_intrinsics()) {
1255       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1256     }
1257     return false; // should be primitive type
1258   }
1259   BasicType elem_bt = elem_type->basic_type();
1260   BasicType mask_bt = elem_bt;
1261   int num_elem = vlen->get_con();
1262 
1263   if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1264     if (C->print_intrinsics()) {
1265       tty->print_cr("  ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1266                     num_elem, type2name(elem_bt));
1267     }
1268     return false; // not supported
1269   }
1270   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1271   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1272 
1273   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1274   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1275 
1276   Node* v1   = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1277   Node* v2   = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1278   Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1279 
1280   if (v1 == NULL || v2 == NULL || mask == NULL) {
1281     return false; // operand unboxing failed
1282   }
1283 
1284   Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1285 
1286   Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1287   set_result(box);
1288   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1289   return true;
1290 }
1291 
1292 //  public static <V extends Vector<E,S>,
1293 //          M extends Vector.Mask<E,S>,
1294 //          S extends Vector.Shape, E>
1295 //  M compare(int cond, Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int vlen,
1296 //            V v1, V v2,
1297 //            VectorCompareOp<V,M> defaultImpl) { ...
1298 //
1299 bool LibraryCallKit::inline_vector_compare() {
1300   const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1301   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1302   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
1303   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
1304   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
1305 
1306   if (cond == NULL || vector_klass == NULL || mask_klass == NULL || elem_klass == NULL || vlen == NULL) {
1307     return false; // dead code
1308   }
1309   if (!cond->is_con() || vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
1310       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1311     if (C->print_intrinsics()) {
1312       tty->print_cr("  ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1313                     NodeClassNames[argument(0)->Opcode()],
1314                     NodeClassNames[argument(1)->Opcode()],
1315                     NodeClassNames[argument(2)->Opcode()],
1316                     NodeClassNames[argument(3)->Opcode()],
1317                     NodeClassNames[argument(4)->Opcode()]);
1318     }
1319     return false; // not enough info for intrinsification
1320   }
1321   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1322     if (C->print_intrinsics()) {
1323       tty->print_cr("  ** klass argument not initialized");
1324     }
1325     return false;
1326   }
1327   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1328   if (!elem_type->is_primitive_type()) {
1329     if (C->print_intrinsics()) {
1330       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1331     }
1332     return false; // should be primitive type
1333   }
1334 
1335   int num_elem = vlen->get_con();
1336   BasicType elem_bt = elem_type->basic_type();
1337   BasicType mask_bt = elem_bt;
1338 
1339   if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1340     if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1341       if (C->print_intrinsics()) {
1342         tty->print_cr("  ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1343                       cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1344       }
1345       return false;
1346     }
1347   }
1348 
1349   if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1350     if (C->print_intrinsics()) {
1351       tty->print_cr("  ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1352                     cond->get_con(), num_elem, type2name(elem_bt));
1353     }
1354     return false;
1355   }
1356 
1357   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1358   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1359 
1360   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1361   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1362 
1363   Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1364   Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1365 
1366   if (v1 == NULL || v2 == NULL) {
1367     return false; // operand unboxing failed
1368   }
1369   BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1370   ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1371 
1372   const TypeVect* vt = TypeVect::make(mask_bt, num_elem);
1373   Node* operation = gvn().transform(new VectorMaskCmpNode(pred, v1, v2, pred_node, vt));
1374 
1375   Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1376   set_result(box);
1377   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1378   return true;
1379 }
1380 
1381 // public static
1382 // <V extends Vector, Sh extends Shuffle>
1383 //  V rearrangeOp(Class<V> vectorClass, Class<Sh> shuffleClass, Class< ? > elementType, int vlen,
1384 //    V v1, Sh sh,
1385 //    VectorSwizzleOp<V, Sh, S, E> defaultImpl) { ...
1386 
1387 bool LibraryCallKit::inline_vector_rearrange() {
1388   const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
1389   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1390   const TypeInstPtr* elem_klass    = gvn().type(argument(2))->isa_instptr();
1391   const TypeInt*     vlen          = gvn().type(argument(3))->isa_int();
1392 
1393   if (vector_klass == NULL || shuffle_klass == NULL || elem_klass == NULL || vlen == NULL) {
1394     return false; // dead code
1395   }
1396   if (shuffle_klass->const_oop() == NULL || vector_klass->const_oop() == NULL ||
1397     elem_klass->const_oop() == NULL || !vlen->is_con()) {
1398     if (C->print_intrinsics()) {
1399       tty->print_cr("  ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1400                     NodeClassNames[argument(0)->Opcode()],
1401                     NodeClassNames[argument(1)->Opcode()],
1402                     NodeClassNames[argument(2)->Opcode()],
1403                     NodeClassNames[argument(3)->Opcode()]);
1404     }
1405     return false; // not enough info for intrinsification
1406   }
1407   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(shuffle_klass)) {
1408     if (C->print_intrinsics()) {
1409       tty->print_cr("  ** klass argument not initialized");
1410     }
1411     return false;
1412   }
1413   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1414   if (!elem_type->is_primitive_type()) {
1415     if (C->print_intrinsics()) {
1416       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1417     }
1418     return false; // should be primitive type
1419   }
1420   BasicType elem_bt = elem_type->basic_type();
1421   BasicType shuffle_bt = elem_bt;
1422   int num_elem = vlen->get_con();
1423 
1424   if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
1425     if (C->print_intrinsics()) {
1426       tty->print_cr("  ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1427                     num_elem, type2name(elem_bt));
1428     }
1429     return false; // not supported
1430   }
1431   if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed)) {
1432     if (C->print_intrinsics()) {
1433       tty->print_cr("  ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
1434                     num_elem, type2name(elem_bt));
1435     }
1436     return false; // not supported
1437   }
1438   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1439   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1440 
1441   ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
1442   const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
1443 
1444   Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1445   Node* shuffle = unbox_vector(argument(5), shbox_type, shuffle_bt, num_elem);
1446 
1447   if (v1 == NULL || shuffle == NULL) {
1448     return false; // operand unboxing failed
1449   }
1450 
1451   Node* rearrange = gvn().transform(new VectorRearrangeNode(v1, shuffle));
1452 
1453   Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
1454   set_result(box);
1455   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1456   return true;
1457 }
1458 
1459 static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
1460   address addr = NULL;
1461   assert(UseVectorStubs, "sanity");
1462   assert(name_ptr != NULL, "unexpected");
1463   assert((vop >= VectorSupport::VECTOR_OP_SVML_START) && (vop <= VectorSupport::VECTOR_OP_SVML_END), "unexpected");
1464   int op = vop - VectorSupport::VECTOR_OP_SVML_START;
1465 
1466   switch(bits) {
1467     case 64:  //fallthough
1468     case 128: //fallthough
1469     case 256: //fallthough
1470     case 512:
1471       if (bt == T_FLOAT) {
1472         snprintf(name_ptr, name_len, "vector_%s_float%d", VectorSupport::svmlname[op], bits);
1473         addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op];
1474       } else {
1475         assert(bt == T_DOUBLE, "must be FP type only");
1476         snprintf(name_ptr, name_len, "vector_%s_double%d", VectorSupport::svmlname[op], bits);
1477         addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op];
1478       }
1479       break;
1480     default:
1481       snprintf(name_ptr, name_len, "invalid");
1482       addr = NULL;
1483       Unimplemented();
1484       break;
1485   }
1486 
1487   return addr;
1488 }
1489 
1490 Node* LibraryCallKit::gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
1491   assert(UseVectorStubs, "sanity");
1492   assert(vector_api_op_id >= VectorSupport::VECTOR_OP_SVML_START && vector_api_op_id <= VectorSupport::VECTOR_OP_SVML_END, "need valid op id");
1493   assert(opd1 != NULL, "must not be null");
1494   const TypeVect* vt = TypeVect::make(bt, num_elem);
1495   const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != NULL ? 2 : 1, vt, vt);
1496   char name[100] = "";
1497 
1498   // Get address for svml method.
1499   address addr = get_svml_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
1500 
1501   if (addr == NULL) {
1502     return NULL;
1503   }
1504 
1505   assert(name != NULL, "name must not be null");
1506   Node* operation = make_runtime_call(RC_VECTOR,
1507                                       call_type,
1508                                       addr,
1509                                       name,
1510                                       TypePtr::BOTTOM,
1511                                       opd1,
1512                                       opd2);
1513   return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
1514 }
1515 
1516 //  public static
1517 //  <V extends Vector<?,?>>
1518 //  V broadcastInt(int opr, Class<V> vectorClass, Class<?> elementType, int vlen,
1519 //                 V v, int i,
1520 //                 VectorBroadcastIntOp<V> defaultImpl) {
1521 //
1522 bool LibraryCallKit::inline_vector_broadcast_int() {
1523   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
1524   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1525   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1526   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1527 
1528   if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL) {
1529     return false; // dead code
1530   }
1531   if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1532     if (C->print_intrinsics()) {
1533       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1534                     NodeClassNames[argument(0)->Opcode()],
1535                     NodeClassNames[argument(1)->Opcode()],
1536                     NodeClassNames[argument(2)->Opcode()],
1537                     NodeClassNames[argument(3)->Opcode()]);
1538     }
1539     return false; // not enough info for intrinsification
1540   }
1541   if (!is_klass_initialized(vector_klass)) {
1542     if (C->print_intrinsics()) {
1543       tty->print_cr("  ** klass argument not initialized");
1544     }
1545     return false;
1546   }
1547   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1548   if (!elem_type->is_primitive_type()) {
1549     if (C->print_intrinsics()) {
1550       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1551     }
1552     return false; // should be primitive type
1553   }
1554   BasicType elem_bt = elem_type->basic_type();
1555   int num_elem = vlen->get_con();
1556   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1557   bool is_shift  = VectorNode::is_shift_opcode(opc);
1558   bool is_rotate = VectorNode::is_rotate_opcode(opc);
1559   if (opc == 0 || (!is_shift && !is_rotate)) {
1560     if (C->print_intrinsics()) {
1561       tty->print_cr("  ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
1562     }
1563     return false; // operation not supported
1564   }
1565   int sopc = VectorNode::opcode(opc, elem_bt);
1566   if (sopc == 0) {
1567     if (C->print_intrinsics()) {
1568       tty->print_cr("  ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
1569     }
1570     return false; // operation not supported
1571   }
1572   Node* cnt  = argument(5);
1573   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1574   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1575   const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
1576 
1577   // If CPU supports vector constant rotate instructions pass it directly
1578   bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
1579                          Matcher::supports_vector_constant_rotates(cnt_type->get_con());
1580   bool has_scalar_args = is_rotate ? !is_const_rotate : true;
1581   if (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args)) {
1582     if (C->print_intrinsics()) {
1583       tty->print_cr("  ** not supported: arity=0 op=int/%d vlen=%d etype=%s ismask=no",
1584                     sopc, num_elem, type2name(elem_bt));
1585     }
1586     return false; // not supported
1587   }
1588   Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1589   Node* opd2 = NULL;
1590   if (is_shift) {
1591     opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
1592   } else {
1593     assert(is_rotate, "unexpected operation");
1594     if (!is_const_rotate) {
1595       const Type * type_bt = Type::get_const_basic_type(elem_bt);
1596       cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
1597       opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, type_bt));
1598     } else {
1599       // Constant shift value.
1600       opd2 = cnt;
1601     }
1602   }
1603   if (opd1 == NULL || opd2 == NULL) {
1604     return false;
1605   }
1606   Node* operation = gvn().transform(VectorNode::make(opc, opd1, opd2, num_elem, elem_bt));
1607 
1608   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
1609   set_result(vbox);
1610   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1611   return true;
1612 }
1613 
1614 // public static <VOUT extends VectorPayload,
1615 //                 VIN extends VectorPayload,
1616 //                   S extends VectorSpecies>
1617 // VOUT convert(int oprId,
1618 //           Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
1619 //           Class<?>   toVectorClass, Class<?>   toElementType, int   toVLen,
1620 //           VIN v, S s,
1621 //           VectorConvertOp<VOUT, VIN, S> defaultImpl) {
1622 //
1623 bool LibraryCallKit::inline_vector_convert() {
1624   const TypeInt*     opr               = gvn().type(argument(0))->isa_int();
1625 
1626   const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
1627   const TypeInstPtr* elem_klass_from   = gvn().type(argument(2))->isa_instptr();
1628   const TypeInt*     vlen_from         = gvn().type(argument(3))->isa_int();
1629 
1630   const TypeInstPtr* vector_klass_to   = gvn().type(argument(4))->isa_instptr();
1631   const TypeInstPtr* elem_klass_to     = gvn().type(argument(5))->isa_instptr();
1632   const TypeInt*     vlen_to           = gvn().type(argument(6))->isa_int();
1633 
1634   if (opr == NULL ||
1635       vector_klass_from == NULL || elem_klass_from == NULL || vlen_from == NULL ||
1636       vector_klass_to   == NULL || elem_klass_to   == NULL || vlen_to   == NULL) {
1637     return false; // dead code
1638   }
1639   if (!opr->is_con() ||
1640       vector_klass_from->const_oop() == NULL || elem_klass_from->const_oop() == NULL || !vlen_from->is_con() ||
1641       vector_klass_to->const_oop() == NULL || elem_klass_to->const_oop() == NULL || !vlen_to->is_con()) {
1642     if (C->print_intrinsics()) {
1643       tty->print_cr("  ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
1644                     NodeClassNames[argument(0)->Opcode()],
1645                     NodeClassNames[argument(1)->Opcode()],
1646                     NodeClassNames[argument(2)->Opcode()],
1647                     NodeClassNames[argument(3)->Opcode()],
1648                     NodeClassNames[argument(4)->Opcode()],
1649                     NodeClassNames[argument(5)->Opcode()],
1650                     NodeClassNames[argument(6)->Opcode()]);
1651     }
1652     return false; // not enough info for intrinsification
1653   }
1654   if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
1655     if (C->print_intrinsics()) {
1656       tty->print_cr("  ** klass argument not initialized");
1657     }
1658     return false;
1659   }
1660 
1661   assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
1662          opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
1663   bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST);
1664 
1665   ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
1666   ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
1667   if (is_vector_shuffle(vbox_klass_from)) {
1668     return false; // vector shuffles aren't supported
1669   }
1670   bool is_mask = is_vector_mask(vbox_klass_from);
1671 
1672   ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
1673   if (!elem_type_from->is_primitive_type()) {
1674     return false; // should be primitive type
1675   }
1676   BasicType elem_bt_from = elem_type_from->basic_type();
1677   ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
1678   if (!elem_type_to->is_primitive_type()) {
1679     return false; // should be primitive type
1680   }
1681   BasicType elem_bt_to = elem_type_to->basic_type();
1682   if (is_mask && (type2aelembytes(elem_bt_from) != type2aelembytes(elem_bt_to))) {
1683     return false; // elem size mismatch
1684   }
1685 
1686   int num_elem_from = vlen_from->get_con();
1687   int num_elem_to = vlen_to->get_con();
1688 
1689   // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
1690   // since we may need to change size.
1691   if (!arch_supports_vector(Op_VectorReinterpret,
1692                             num_elem_from,
1693                             elem_bt_from,
1694                             is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
1695     if (C->print_intrinsics()) {
1696       tty->print_cr("  ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
1697                     is_cast ? "cast" : "reinterpret",
1698                     num_elem_from, type2name(elem_bt_from), is_mask);
1699     }
1700     return false;
1701   }
1702 
1703   // Check whether we can support resizing/reinterpreting to the new size.
1704   if (!arch_supports_vector(Op_VectorReinterpret,
1705                             num_elem_to,
1706                             elem_bt_to,
1707                             is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
1708     if (C->print_intrinsics()) {
1709       tty->print_cr("  ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
1710                     is_cast ? "cast" : "reinterpret",
1711                     num_elem_to, type2name(elem_bt_to), is_mask);
1712     }
1713     return false;
1714   }
1715 
1716   // At this point, we know that both input and output vector registers are supported
1717   // by the architecture. Next check if the casted type is simply to same type - which means
1718   // that it is actually a resize and not a cast.
1719   if (is_cast && elem_bt_from == elem_bt_to) {
1720     is_cast = false;
1721   }
1722 
1723   const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
1724 
1725   Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
1726   if (opd1 == NULL) {
1727     return false;
1728   }
1729 
1730   const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from);
1731   const TypeVect* dst_type = TypeVect::make(elem_bt_to,   num_elem_to);
1732 
1733   Node* op = opd1;
1734   if (is_cast) {
1735     assert(!is_mask, "masks cannot be casted");
1736     int cast_vopc = VectorCastNode::opcode(elem_bt_from);
1737     // Make sure that cast is implemented to particular type/size combination.
1738     if (!arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
1739       if (C->print_intrinsics()) {
1740         tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
1741                       cast_vopc,
1742                       num_elem_to, type2name(elem_bt_to), is_mask);
1743       }
1744       return false;
1745     }
1746 
1747     if (num_elem_from < num_elem_to) {
1748       // Since input and output number of elements are not consistent, we need to make sure we
1749       // properly size. Thus, first make a cast that retains the number of elements from source.
1750       int num_elem_for_cast = num_elem_from;
1751 
1752       // It is possible that arch does not support this intermediate vector size
1753       // TODO More complex logic required here to handle this corner case for the sizes.
1754       if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
1755         if (C->print_intrinsics()) {
1756           tty->print_cr("  ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
1757                         cast_vopc,
1758                         num_elem_for_cast, type2name(elem_bt_to), is_mask);
1759         }
1760         return false;
1761       }
1762 
1763       op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
1764       // Now ensure that the destination gets properly resized to needed size.
1765       op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
1766     } else if (num_elem_from > num_elem_to) {
1767       // Since number elements from input is larger than output, simply reduce size of input (we are supposed to
1768       // drop top elements anyway).
1769       int num_elem_for_resize = num_elem_to;
1770 
1771       // It is possible that arch does not support this intermediate vector size
1772       // TODO More complex logic required here to handle this corner case for the sizes.
1773       if (!arch_supports_vector(Op_VectorReinterpret,
1774                                 num_elem_for_resize,
1775                                 elem_bt_from,
1776                                 VecMaskNotUsed)) {
1777         if (C->print_intrinsics()) {
1778           tty->print_cr("  ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
1779                         num_elem_for_resize, type2name(elem_bt_from), is_mask);
1780         }
1781         return false;
1782       }
1783 
1784       op = gvn().transform(new VectorReinterpretNode(op,
1785                                                      src_type,
1786                                                      TypeVect::make(elem_bt_from,
1787                                                                     num_elem_for_resize)));
1788       op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
1789     } else {
1790       // Since input and output number of elements match, and since we know this vector size is
1791       // supported, simply do a cast with no resize needed.
1792       op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
1793     }
1794   } else if (Type::cmp(src_type, dst_type) != 0) {
1795     assert(!is_cast, "must be reinterpret");
1796     op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
1797   }
1798 
1799   const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
1800   Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
1801   set_result(vbox);
1802   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
1803   return true;
1804 }
1805 
1806 //  public static
1807 //  <V extends Vector<?>>
1808 //  V insert(Class<? extends V> vectorClass, Class<?> elementType, int vlen,
1809 //           V vec, int ix, long val,
1810 //           VecInsertOp<V> defaultImpl) {
1811 //
1812 bool LibraryCallKit::inline_vector_insert() {
1813   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1814   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
1815   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
1816   const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
1817 
1818   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || idx == NULL) {
1819     return false; // dead code
1820   }
1821   if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
1822     if (C->print_intrinsics()) {
1823       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
1824                     NodeClassNames[argument(0)->Opcode()],
1825                     NodeClassNames[argument(1)->Opcode()],
1826                     NodeClassNames[argument(2)->Opcode()],
1827                     NodeClassNames[argument(4)->Opcode()]);
1828     }
1829     return false; // not enough info for intrinsification
1830   }
1831   if (!is_klass_initialized(vector_klass)) {
1832     if (C->print_intrinsics()) {
1833       tty->print_cr("  ** klass argument not initialized");
1834     }
1835     return false;
1836   }
1837   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1838   if (!elem_type->is_primitive_type()) {
1839     if (C->print_intrinsics()) {
1840       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1841     }
1842     return false; // should be primitive type
1843   }
1844   BasicType elem_bt = elem_type->basic_type();
1845   int num_elem = vlen->get_con();
1846   if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
1847     if (C->print_intrinsics()) {
1848       tty->print_cr("  ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
1849                     num_elem, type2name(elem_bt));
1850     }
1851     return false; // not supported
1852   }
1853 
1854   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1855   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1856 
1857   Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
1858   if (opd == NULL) {
1859     return false;
1860   }
1861 
1862   Node* insert_val = argument(5);
1863   assert(gvn().type(insert_val)->isa_long() != NULL, "expected to be long");
1864 
1865   // Convert insert value back to its appropriate type.
1866   switch (elem_bt) {
1867     case T_BYTE:
1868       insert_val = gvn().transform(new ConvL2INode(insert_val));
1869       insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::BYTE));
1870       break;
1871     case T_SHORT:
1872       insert_val = gvn().transform(new ConvL2INode(insert_val));
1873       insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::SHORT));
1874       break;
1875     case T_INT:
1876       insert_val = gvn().transform(new ConvL2INode(insert_val));
1877       break;
1878     case T_FLOAT:
1879       insert_val = gvn().transform(new ConvL2INode(insert_val));
1880       insert_val = gvn().transform(new MoveI2FNode(insert_val));
1881       break;
1882     case T_DOUBLE:
1883       insert_val = gvn().transform(new MoveL2DNode(insert_val));
1884       break;
1885     case T_LONG:
1886       // no conversion needed
1887       break;
1888     default: fatal("%s", type2name(elem_bt)); break;
1889   }
1890 
1891   Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con()));
1892 
1893   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
1894   set_result(vbox);
1895   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1896   return true;
1897 }
1898 
1899 //  public static
1900 //  <V extends Vector<?>>
1901 //  long extract(Class<?> vectorClass, Class<?> elementType, int vlen,
1902 //               V vec, int ix,
1903 //               VecExtractOp<V> defaultImpl) {
1904 //
1905 bool LibraryCallKit::inline_vector_extract() {
1906   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1907   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
1908   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
1909   const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
1910 
1911   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || idx == NULL) {
1912     return false; // dead code
1913   }
1914   if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
1915     if (C->print_intrinsics()) {
1916       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
1917                     NodeClassNames[argument(0)->Opcode()],
1918                     NodeClassNames[argument(1)->Opcode()],
1919                     NodeClassNames[argument(2)->Opcode()],
1920                     NodeClassNames[argument(4)->Opcode()]);
1921     }
1922     return false; // not enough info for intrinsification
1923   }
1924   if (!is_klass_initialized(vector_klass)) {
1925     if (C->print_intrinsics()) {
1926       tty->print_cr("  ** klass argument not initialized");
1927     }
1928     return false;
1929   }
1930   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1931   if (!elem_type->is_primitive_type()) {
1932     if (C->print_intrinsics()) {
1933       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1934     }
1935     return false; // should be primitive type
1936   }
1937   BasicType elem_bt = elem_type->basic_type();
1938   int num_elem = vlen->get_con();
1939   int vopc = ExtractNode::opcode(elem_bt);
1940   if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
1941     if (C->print_intrinsics()) {
1942       tty->print_cr("  ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
1943                     num_elem, type2name(elem_bt));
1944     }
1945     return false; // not supported
1946   }
1947 
1948   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1949   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1950 
1951   Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
1952   if (opd == NULL) {
1953     return false;
1954   }
1955 
1956   Node* operation = gvn().transform(ExtractNode::make(opd, idx->get_con(), elem_bt));
1957 
1958   Node* bits = NULL;
1959   switch (elem_bt) {
1960     case T_BYTE:
1961     case T_SHORT:
1962     case T_INT: {
1963       bits = gvn().transform(new ConvI2LNode(operation));
1964       break;
1965     }
1966     case T_FLOAT: {
1967       bits = gvn().transform(new MoveF2INode(operation));
1968       bits = gvn().transform(new ConvI2LNode(bits));
1969       break;
1970     }
1971     case T_DOUBLE: {
1972       bits = gvn().transform(new MoveD2LNode(operation));
1973       break;
1974     }
1975     case T_LONG: {
1976       bits = operation; // no conversion needed
1977       break;
1978     }
1979     default: fatal("%s", type2name(elem_bt));
1980   }
1981 
1982   set_result(bits);
1983   return true;
1984 }
1985