< prev index next >

src/hotspot/share/opto/vectorIntrinsics.cpp

Print this page

  42   ciInstanceKlass* ik = vbox_type->klass()->as_instance_klass();
  43   assert(is_vector(ik), "not a vector");
  44 
  45   ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
  46   assert(fd1 != NULL, "element type info is missing");
  47 
  48   ciConstant val1 = fd1->constant_value();
  49   BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
  50   assert(is_java_primitive(elem_bt), "element type info is missing");
  51 
  52   ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
  53   assert(fd2 != NULL, "vector length info is missing");
  54 
  55   ciConstant val2 = fd2->constant_value();
  56   assert(val2.as_int() > 0, "vector length info is missing");
  57 
  58   return true;
  59 }
  60 #endif
  61 
  62 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt, bool has_scalar_args) {
  63     bool is_supported = true;
  64     // has_scalar_args flag is true only for non-constant scalar shift count,
  65     // since in this case shift needs to be broadcasted.
  66     if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
  67          (has_scalar_args &&
  68            !arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed))) {
  69       is_supported = false;
  70     }
  71 
  72     int lshiftopc, rshiftopc;
  73     switch(elem_bt) {
  74       case T_BYTE:
  75         lshiftopc = Op_LShiftI;
  76         rshiftopc = Op_URShiftB;
  77         break;
  78       case T_SHORT:
  79         lshiftopc = Op_LShiftI;
  80         rshiftopc = Op_URShiftS;
  81         break;
  82       case T_INT:
  83         lshiftopc = Op_LShiftI;
  84         rshiftopc = Op_URShiftI;
  85         break;
  86       case T_LONG:
  87         lshiftopc = Op_LShiftL;
  88         rshiftopc = Op_URShiftL;
  89         break;
  90       default:
  91         assert(false, "Unexpected type");
  92     }
  93     int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
  94     int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
  95     if (!is_supported &&
  96         arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
  97         arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
  98         arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
  99       is_supported = true;
 100     }
 101     return is_supported;






































 102 }
 103 
 104 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
 105   assert(EnableVectorSupport, "");
 106 
 107   PreserveReexecuteState preexecs(this);
 108   jvms()->set_should_reexecute(true);
 109 
 110   VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
 111   set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
 112   make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
 113   set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
 114   set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
 115   Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
 116 
 117   assert(check_vbox(vbox_type), "");
 118   const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
 119   VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
 120   return gvn().transform(vbox);
 121 }
 122 
 123 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) {
 124   assert(EnableVectorSupport, "");
 125   const TypeInstPtr* vbox_type_v = gvn().type(v)->is_instptr();
 126   if (vbox_type->klass() != vbox_type_v->klass()) {
 127     return NULL; // arguments don't agree on vector shapes
 128   }
 129   if (vbox_type_v->maybe_null()) {
 130     return NULL; // no nulls are allowed
 131   }
 132   assert(check_vbox(vbox_type), "");
 133   const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
 134   Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector));
 135   return unbox;
 136 }
 137 
 138 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
 139   assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
 140   juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
 141   Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
 142   Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
 143   return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
 144 }
 145 
 146 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
 147   // Check that the operation is valid.
 148   if (sopc <= 0) {
 149 #ifndef PRODUCT
 150     if (C->print_intrinsics()) {
 151       tty->print_cr("  ** Rejected intrinsification because no valid vector op could be extracted");
 152     }
 153 #endif
 154     return false;
 155   }
 156 
 157   if (VectorNode::is_vector_rotate(sopc)) {
 158     if(!arch_supports_vector_rotate(sopc, num_elem, type, has_scalar_args)) {
 159 #ifndef PRODUCT
 160       if (C->print_intrinsics()) {
 161         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
 162                       NodeClassNames[sopc], type2name(type), num_elem);
 163       }
 164 #endif
 165       return false;
 166     }
 167   } else {
 168     // Check that architecture supports this op-size-type combination.
 169     if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
 170 #ifndef PRODUCT
 171       if (C->print_intrinsics()) {
 172         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support it",
 173                       NodeClassNames[sopc], type2name(type), num_elem);
 174       }
 175 #endif
 176       return false;
 177     } else {
 178       assert(Matcher::match_rule_supported(sopc), "must be supported");

 196         if (C->print_intrinsics()) {
 197           tty->print_cr("  ** Not a svml call or load/store vector op (%s,%s,%d)",
 198                         NodeClassNames[sopc], type2name(type), num_elem);
 199         }
 200 #endif
 201         return false;
 202       }
 203     }
 204   }
 205 
 206   if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
 207       Matcher::supports_vector_variable_shifts() == false) {
 208     if (C->print_intrinsics()) {
 209       tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
 210                     NodeClassNames[sopc], type2name(type), num_elem);
 211     }
 212     return false;
 213   }
 214 
 215   // Check whether mask unboxing is supported.
 216   if (mask_use_type == VecMaskUseAll || mask_use_type == VecMaskUseLoad) {
 217     if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type)) {
 218     #ifndef PRODUCT
 219       if (C->print_intrinsics()) {
 220         tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
 221                       NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
 222       }
 223     #endif
 224       return false;
 225     }
 226   }
 227 
 228   // Check whether mask boxing is supported.
 229   if (mask_use_type == VecMaskUseAll || mask_use_type == VecMaskUseStore) {
 230     if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type)) {
 231     #ifndef PRODUCT
 232       if (C->print_intrinsics()) {
 233         tty->print_cr("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
 234                       NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
 235       }
 236     #endif
 237       return false;
 238     }
 239   }
 240 
 241   return true;
 242 }
 243 
 244 static bool is_vector_mask(ciKlass* klass) {
 245   return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
 246 }






 247 
 248 static bool is_vector_shuffle(ciKlass* klass) {
 249   return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
 250 }
 251 
 252 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
 253   if (vec_klass->const_oop() == NULL) {
 254     return false; // uninitialized or some kind of unsafe access
 255   }
 256   assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != NULL, "klass instance expected");
 257   ciInstanceKlass* klass =  vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
 258   return klass->is_initialized();
 259 }
 260 
 261 // public static
 262 // <VM>
 263 // VM unaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
 264 //            VM vm,
 265 //            Function<VM, VM> defaultImpl) {


 266 //
 267 // public static
 268 // <VM>
 269 // VM binaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
 270 //             VM vm1, VM vm2,
 271 //             BiFunction<VM, VM, VM> defaultImpl) {


 272 //
 273 // public static
 274 // <VM>
 275 // VM ternaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
 276 //              VM vm1, VM vm2, VM vm3,
 277 //              TernaryOperation<VM> defaultImpl) {


 278 //
 279 bool LibraryCallKit::inline_vector_nary_operation(int n) {
 280   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
 281   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
 282   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
 283   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();

 284 
 285   if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
 286       !opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 287     if (C->print_intrinsics()) {
 288       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
 289                     NodeClassNames[argument(0)->Opcode()],
 290                     NodeClassNames[argument(1)->Opcode()],
 291                     NodeClassNames[argument(2)->Opcode()],
 292                     NodeClassNames[argument(3)->Opcode()]);
 293     }
 294     return false; // not enough info for intrinsification
 295   }

 296   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 297   if (!elem_type->is_primitive_type()) {
 298     if (C->print_intrinsics()) {
 299       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 300     }
 301     return false; // should be primitive type
 302   }
 303   if (!is_klass_initialized(vector_klass)) {
 304     if (C->print_intrinsics()) {
 305       tty->print_cr("  ** klass argument not initialized");
 306     }
 307     return false;
 308   }




























 309   BasicType elem_bt = elem_type->basic_type();
 310   int num_elem = vlen->get_con();
 311   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
 312   int sopc = VectorNode::opcode(opc, elem_bt);
 313   if ((opc != Op_CallLeafVector) && (sopc == 0)) {
 314     if (C->print_intrinsics()) {
 315       tty->print_cr("  ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
 316     }
 317     return false; // operation not supported
 318   }
 319   if (num_elem == 1) {
 320     if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
 321       if (C->print_intrinsics()) {
 322         tty->print_cr("  ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
 323                       n, opc, num_elem, type2name(elem_bt));
 324       }
 325       return false;
 326     }
 327   }
 328   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 329   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 330 




 331   if (opc == Op_CallLeafVector) {
 332     if (!UseVectorStubs) {
 333       if (C->print_intrinsics()) {
 334         tty->print_cr("  ** vector stubs support is disabled");
 335       }
 336       return false;
 337     }
 338     if (!Matcher::supports_vector_calling_convention()) {
 339       if (C->print_intrinsics()) {
 340         tty->print_cr("  ** no vector calling conventions supported");
 341       }
 342       return false;
 343     }
 344     if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
 345       if (C->print_intrinsics()) {
 346         tty->print_cr("  ** vector size (vlen=%d, etype=%s) is not supported",
 347                       num_elem, type2name(elem_bt));
 348       }
 349       return false;
 350     }
 351   }
 352 
 353   // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
 354   if ((sopc != 0) &&
 355       !arch_supports_vector(sopc, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseAll : VecMaskNotUsed)) {

 356     if (C->print_intrinsics()) {
 357       tty->print_cr("  ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d",
 358                     n, sopc, num_elem, type2name(elem_bt),
 359                     is_vector_mask(vbox_klass) ? 1 : 0);
 360     }
 361     return false; // not supported
 362   }
 363 










 364   Node* opd1 = NULL; Node* opd2 = NULL; Node* opd3 = NULL;
 365   switch (n) {
 366     case 3: {
 367       opd3 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
 368       if (opd3 == NULL) {
 369         if (C->print_intrinsics()) {
 370           tty->print_cr("  ** unbox failed v3=%s",
 371                         NodeClassNames[argument(6)->Opcode()]);
 372         }
 373         return false;
 374       }
 375       // fall-through
 376     }
 377     case 2: {
 378       opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
 379       if (opd2 == NULL) {
 380         if (C->print_intrinsics()) {
 381           tty->print_cr("  ** unbox failed v2=%s",
 382                         NodeClassNames[argument(5)->Opcode()]);
 383         }
 384         return false;
 385       }
 386       // fall-through
 387     }
 388     case 1: {
 389       opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
 390       if (opd1 == NULL) {
 391         if (C->print_intrinsics()) {
 392           tty->print_cr("  ** unbox failed v1=%s",
 393                         NodeClassNames[argument(4)->Opcode()]);
 394         }
 395         return false;
 396       }
 397       break;
 398     }
 399     default: fatal("unsupported arity: %d", n);
 400   }
 401 















 402   Node* operation = NULL;
 403   if (opc == Op_CallLeafVector) {
 404     assert(UseVectorStubs, "sanity");
 405     operation = gen_call_to_svml(opr->get_con(), elem_bt, num_elem, opd1, opd2);
 406     if (operation == NULL) {
 407       if (C->print_intrinsics()) {
 408         tty->print_cr("  ** svml call failed for %s_%s_%d",
 409                          (elem_bt == T_FLOAT)?"float":"double",
 410                          VectorSupport::svmlname[opr->get_con() - VectorSupport::VECTOR_OP_SVML_START],
 411                          num_elem * type2aelembytes(elem_bt));
 412       }
 413       return false;
 414      }
 415   } else {
 416     const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
 417     switch (n) {
 418       case 1:
 419       case 2: {
 420         operation = gvn().transform(VectorNode::make(sopc, opd1, opd2, vt));
 421         break;
 422       }
 423       case 3: {
 424         operation = gvn().transform(VectorNode::make(sopc, opd1, opd2, opd3, vt));
 425         break;
 426       }
 427       default: fatal("unsupported arity: %d", n);
 428     }
 429   }












 430   // Wrap it up in VectorBox to keep object type information.
 431   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
 432   set_result(vbox);
 433   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 434   return true;
 435 }
 436 
 437 // <Sh extends VectorShuffle<E>,  E>
 438 //  Sh ShuffleIota(Class<?> E, Class<?> ShuffleClass, Vector.Species<E> s, int length,
 439 //                  int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
 440 bool LibraryCallKit::inline_vector_shuffle_iota() {
 441   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
 442   const TypeInt*     vlen          = gvn().type(argument(3))->isa_int();
 443   const TypeInt*     start_val     = gvn().type(argument(4))->isa_int();
 444   const TypeInt*     step_val      = gvn().type(argument(5))->isa_int();
 445   const TypeInt*     wrap          = gvn().type(argument(6))->isa_int();
 446 
 447   Node* start = argument(4);
 448   Node* step  = argument(5);
 449 
 450   if (shuffle_klass == NULL || vlen == NULL || start_val == NULL || step_val == NULL || wrap == NULL) {
 451     return false; // dead code
 452   }
 453   if (!vlen->is_con() || !is_power_of_2(vlen->get_con()) ||
 454       shuffle_klass->const_oop() == NULL || !wrap->is_con()) {
 455     return false; // not enough info for intrinsification
 456   }
 457   if (!is_klass_initialized(shuffle_klass)) {
 458     if (C->print_intrinsics()) {

 492   if(!step_val->is_con() || !is_power_of_2(step_val->get_con())) {
 493     Node* bcast_step     = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt));
 494     res = gvn().transform(VectorNode::make(Op_MulI, res, bcast_step, num_elem, elem_bt));
 495   } else if (step_val->get_con() > 1) {
 496     Node* cnt = gvn().makecon(TypeInt::make(log2i_exact(step_val->get_con())));
 497     Node* shift_cnt = vector_shift_count(cnt, Op_LShiftI, elem_bt, num_elem);
 498     res = gvn().transform(VectorNode::make(Op_LShiftVB, res, shift_cnt, vt));
 499   }
 500 
 501   if (!start_val->is_con() || start_val->get_con() != 0) {
 502     Node* bcast_start    = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt));
 503     res = gvn().transform(VectorNode::make(Op_AddI, res, bcast_start, num_elem, elem_bt));
 504   }
 505 
 506   Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1));
 507   Node * bcast_mod  = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt));
 508   if(do_wrap)  {
 509     // Wrap the indices greater than lane count.
 510     res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
 511   } else {
 512     ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(1));
 513     Node * lane_cnt  = gvn().makecon(TypeInt::make(num_elem));
 514     Node * bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt));
 515     Node* mask = gvn().transform(new VectorMaskCmpNode(BoolTest::ge, bcast_lane_cnt, res, pred_node, vt));

 516 
 517     // Make the indices greater than lane count as -ve values. This matches the java side implementation.
 518     res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
 519     Node * biased_val = gvn().transform(VectorNode::make(Op_SubI, res, bcast_lane_cnt, num_elem, elem_bt));
 520     res = gvn().transform(new VectorBlendNode(biased_val, res, mask));
 521   }
 522 
 523   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 524   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 525 
 526   // Wrap it up in VectorBox to keep object type information.
 527   res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
 528   set_result(res);
 529   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 530   return true;
 531 }
 532 
 533 // <E, M>
 534 // int maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
 535 //                          int length, M m, VectorMaskOp<M> defaultImpl)
 536 bool LibraryCallKit::inline_vector_mask_operation() {
 537   const TypeInt*     oper       = gvn().type(argument(0))->isa_int();
 538   const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
 539   const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
 540   const TypeInt*     vlen       = gvn().type(argument(3))->isa_int();
 541   Node*              mask       = argument(4);
 542 
 543   if (mask_klass == NULL || elem_klass == NULL || mask->is_top() || vlen == NULL) {
 544     return false; // dead code
 545   }
 546 
 547   if (!is_klass_initialized(mask_klass)) {
 548     if (C->print_intrinsics()) {
 549       tty->print_cr("  ** klass argument not initialized");
 550     }
 551     return false;
 552   }
 553 
 554   int num_elem = vlen->get_con();

 559     if (C->print_intrinsics()) {
 560       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 561                     Op_LoadVector, num_elem, type2name(T_BOOLEAN));
 562     }
 563     return false; // not supported
 564   }
 565 
 566   int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
 567   if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskNotUsed)) {
 568     if (C->print_intrinsics()) {
 569       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 570                     mopc, num_elem, type2name(elem_bt));
 571     }
 572     return false; // not supported
 573   }
 574 
 575   const Type* elem_ty = Type::get_const_basic_type(elem_bt);
 576   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
 577   const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
 578   Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
 579   Node* store_mask = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
 580   Node* maskoper = gvn().transform(VectorMaskOpNode::make(store_mask, TypeInt::INT, mopc));






 581   set_result(maskoper);
 582 
 583   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 584   return true;
 585 }
 586 
 587 // <VM ,Sh extends VectorShuffle<E>, E>
 588 // VM shuffleToVector(Class<VM> VecClass, Class<?>E , Class<?> ShuffleClass, Sh s, int length,
 589 //                    ShuffleToVectorOperation<VM,Sh,E> defaultImpl)




 590 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
 591   const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
 592   const TypeInstPtr* elem_klass    = gvn().type(argument(1))->isa_instptr();
 593   const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
 594   Node*              shuffle       = argument(3);
 595   const TypeInt*     vlen          = gvn().type(argument(4))->isa_int();
 596 
 597   if (vector_klass == NULL || elem_klass == NULL || shuffle_klass == NULL || shuffle->is_top() || vlen == NULL) {
 598     return false; // dead code
 599   }
 600   if (!vlen->is_con() || vector_klass->const_oop() == NULL || shuffle_klass->const_oop() == NULL) {
 601     return false; // not enough info for intrinsification
 602   }
 603   if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
 604     if (C->print_intrinsics()) {
 605       tty->print_cr("  ** klass argument not initialized");
 606     }
 607     return false;
 608   }
 609 

 628   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 629   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 630 
 631   // Unbox shuffle with true flag to indicate its load shuffle to vector
 632   // shuffle is a byte array
 633   Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, T_BYTE, num_elem, true);
 634 
 635   // cast byte to target element type
 636   shuffle_vec = gvn().transform(VectorCastNode::make(cast_vopc, shuffle_vec, elem_bt, num_elem));
 637 
 638   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 639   const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 640 
 641   // Box vector
 642   Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem);
 643   set_result(res);
 644   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 645   return true;
 646 }
 647 
 648 // <V extends Vector<?,?>>
 649 // V broadcastCoerced(Class<?> vectorClass, Class<?> elementType, int vlen,
 650 //                    long bits,
 651 //                    LongFunction<V> defaultImpl)



 652 bool LibraryCallKit::inline_vector_broadcast_coerced() {
 653   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 654   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 655   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 656 
 657   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
 658       vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 659     if (C->print_intrinsics()) {
 660       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
 661                     NodeClassNames[argument(0)->Opcode()],
 662                     NodeClassNames[argument(1)->Opcode()],
 663                     NodeClassNames[argument(2)->Opcode()]);
 664     }
 665     return false; // not enough info for intrinsification
 666   }
 667 
 668   if (!is_klass_initialized(vector_klass)) {
 669     if (C->print_intrinsics()) {
 670       tty->print_cr("  ** klass argument not initialized");
 671     }

 678     }
 679     return false; // should be primitive type
 680   }
 681   BasicType elem_bt = elem_type->basic_type();
 682   int num_elem = vlen->get_con();
 683   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 684   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 685 
 686   // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
 687   if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt,
 688                             (is_vector_mask(vbox_klass) ? VecMaskUseStore : VecMaskNotUsed), true /*has_scalar_args*/)) {
 689     if (C->print_intrinsics()) {
 690       tty->print_cr("  ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d",
 691                     num_elem, type2name(elem_bt),
 692                     is_vector_mask(vbox_klass) ? 1 : 0);
 693     }
 694     return false; // not supported
 695   }
 696 
 697   Node* bits = argument(3); // long
 698 
 699   Node* elem = NULL;
 700   switch (elem_bt) {
 701     case T_BOOLEAN: // fall-through
 702     case T_BYTE:    // fall-through
 703     case T_SHORT:   // fall-through
 704     case T_CHAR:    // fall-through
 705     case T_INT: {
 706       elem = gvn().transform(new ConvL2INode(bits));
 707       break;
 708     }
 709     case T_DOUBLE: {
 710       elem = gvn().transform(new MoveL2DNode(bits));
 711       break;
 712     }
 713     case T_FLOAT: {
 714       bits = gvn().transform(new ConvL2INode(bits));
 715       elem = gvn().transform(new MoveI2FNode(bits));
 716       break;
 717     }
 718     case T_LONG: {
 719       elem = bits; // no conversion needed
 720       break;
 721     }
 722     default: fatal("%s", type2name(elem_bt));
 723   }
 724 
 725   Node* broadcast = VectorNode::scalar2vector(elem, num_elem, Type::get_const_basic_type(elem_bt));
 726   broadcast = gvn().transform(broadcast);
 727 
 728   Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
 729   set_result(box);
 730   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 731   return true;
 732 }
 733 
 734 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type) {
 735   assert(arr_type != NULL, "unexpected");
 736   BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
 737   if (elem_bt == arr_elem_bt) {
 738     return true;
 739   } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
 740     // Load/store of short vector from/to char[] is supported
 741     return true;
 742   } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
 743     // Load/store of byte vector from/to boolean[] is supported
 744     return true;
 745   } else {
 746     return false;
 747   }
 748 }
 749 
 750 //    <C, V extends Vector<?,?>>
 751 //    V load(Class<?> vectorClass, Class<?> elementType, int vlen,
 752 //           Object base, long offset,
 753 //           /* Vector.Mask<E,S> m*/
 754 //           Object container, int index,
 755 //           LoadOperation<C, VM> defaultImpl) {



 756 //
 757 //    <C, V extends Vector<?,?>>
 758 //    void store(Class<?> vectorClass, Class<?> elementType, int vlen,
 759 //               Object base, long offset,
 760 //               V v, /*Vector.Mask<E,S> m*/
 761 //               Object container, int index,
 762 //               StoreVectorOperation<C, V> defaultImpl) {


 763 
 764 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
 765   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 766   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 767   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 768 
 769   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
 770       vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 771     if (C->print_intrinsics()) {
 772       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
 773                     NodeClassNames[argument(0)->Opcode()],
 774                     NodeClassNames[argument(1)->Opcode()],
 775                     NodeClassNames[argument(2)->Opcode()]);
 776     }
 777     return false; // not enough info for intrinsification
 778   }
 779   if (!is_klass_initialized(vector_klass)) {
 780     if (C->print_intrinsics()) {
 781       tty->print_cr("  ** klass argument not initialized");
 782     }

 797   if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
 798     if (C->print_intrinsics()) {
 799       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
 800                     is_store, is_store ? "store" : "load",
 801                     num_elem, type2name(elem_bt));
 802     }
 803     return false; // not supported
 804   }
 805 
 806   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 807   bool is_mask = is_vector_mask(vbox_klass);
 808 
 809   Node* base = argument(3);
 810   Node* offset = ConvL2X(argument(4));
 811 
 812   // Save state and restore on bailout
 813   uint old_sp = sp();
 814   SafePointNode* old_map = clone_map();
 815 
 816   Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
 817   // Can base be NULL? Otherwise, always on-heap access.
 818   bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));


 819 
 820   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
 821   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
 822 









 823   // Now handle special case where load/store happens from/to byte array but element type is not byte.
 824   bool using_byte_array = arr_type != NULL && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE;
 825   // Handle loading masks.
 826   // If there is no consistency between array and vector element types, it must be special byte array case or loading masks
 827   if (arr_type != NULL && !using_byte_array && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type)) {
 828     if (C->print_intrinsics()) {
 829       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
 830                     is_store, is_store ? "store" : "load",
 831                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
 832     }
 833     set_map(old_map);
 834     set_sp(old_sp);
 835     return false;
 836   }
 837   // Since we are using byte array, we need to double check that the byte operations are supported by backend.
 838   if (using_byte_array) {
 839     int byte_num_elem = num_elem * type2aelembytes(elem_bt);
 840     if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, byte_num_elem, T_BYTE, VecMaskNotUsed)
 841         || !arch_supports_vector(Op_VectorReinterpret, byte_num_elem, T_BYTE, VecMaskNotUsed)) {
 842       if (C->print_intrinsics()) {

 860       set_sp(old_sp);
 861       return false; // not supported
 862     }
 863     if (!is_store) {
 864       if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
 865         set_map(old_map);
 866         set_sp(old_sp);
 867         return false; // not supported
 868       }
 869     } else {
 870       if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
 871         set_map(old_map);
 872         set_sp(old_sp);
 873         return false; // not supported
 874       }
 875     }
 876   }
 877 
 878   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 879 
 880   if (can_access_non_heap) {
 881     insert_mem_bar(Op_MemBarCPUOrder);
 882   }
 883 
 884   if (is_store) {
 885     Node* val = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
 886     if (val == NULL) {
 887       set_map(old_map);
 888       set_sp(old_sp);
 889       return false; // operand unboxing failed
 890     }
 891     set_all_memory(reset_memory());
 892 
 893     // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
 894     int store_num_elem = num_elem;
 895     if (using_byte_array) {
 896       store_num_elem = num_elem * type2aelembytes(elem_bt);
 897       const TypeVect* to_vect_type = TypeVect::make(T_BYTE, store_num_elem);
 898       val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
 899     }
 900 
 901     Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
 902     set_memory(vstore, addr_type);
 903   } else {
 904     // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
 905     Node* vload = NULL;
 906     if (using_byte_array) {
 907       int load_num_elem = num_elem * type2aelembytes(elem_bt);
 908       vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, load_num_elem, T_BYTE));
 909       const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
 910       vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
 911     } else {
 912       // Special handle for masks
 913       if (is_mask) {
 914         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
 915         const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
 916         vload = gvn().transform(new VectorLoadMaskNode(vload, to_vect_type));
 917       } else {
 918         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
 919       }
 920     }
 921     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
 922     set_result(box);
 923   }
 924 
 925   old_map->destruct(&_gvn);
 926 













































































































































































































































 927   if (can_access_non_heap) {
 928     insert_mem_bar(Op_MemBarCPUOrder);
 929   }
 930 
 931   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 932   return true;
 933 }
 934 
 935 //   <C, V extends Vector<?>, W extends IntVector, E, S extends VectorSpecies<E>>
 936 //   void loadWithMap(Class<?> vectorClass, Class<E> E, int length, Class<?> vectorIndexClass,
 937 //                    Object base, long offset, // Unsafe addressing
 938 //                    W index_vector,
 939 //                    C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
 940 //                    LoadVectorOperationWithMap<C, V, E, S> defaultImpl)






 941 //
 942 //    <C, V extends Vector<?>, W extends IntVector>
 943 //    void storeWithMap(Class<?> vectorClass, Class<?> elementType, int length, Class<?> vectorIndexClass,
 944 //                      Object base, long offset,    // Unsafe addressing
 945 //                      W index_vector, V v,
 946 //                      C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
 947 //                      StoreVectorOperationWithMap<C, V> defaultImpl) {




 948 //
 949 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
 950   const TypeInstPtr* vector_klass     = gvn().type(argument(0))->isa_instptr();
 951   const TypeInstPtr* elem_klass       = gvn().type(argument(1))->isa_instptr();
 952   const TypeInt*     vlen             = gvn().type(argument(2))->isa_int();
 953   const TypeInstPtr* vector_idx_klass = gvn().type(argument(3))->isa_instptr();

 954 
 955   if (vector_klass == NULL || elem_klass == NULL || vector_idx_klass == NULL || vlen == NULL ||
 956       vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || vector_idx_klass->const_oop() == NULL || !vlen->is_con()) {
 957     if (C->print_intrinsics()) {
 958       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
 959                     NodeClassNames[argument(0)->Opcode()],
 960                     NodeClassNames[argument(1)->Opcode()],
 961                     NodeClassNames[argument(2)->Opcode()],
 962                     NodeClassNames[argument(3)->Opcode()]);

 963     }
 964     return false; // not enough info for intrinsification
 965   }
 966 
 967   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
 968     if (C->print_intrinsics()) {
 969       tty->print_cr("  ** klass argument not initialized");
 970     }
 971     return false;
 972   }

 973   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 974   if (!elem_type->is_primitive_type()) {
 975     if (C->print_intrinsics()) {
 976       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 977     }
 978     return false; // should be primitive type
 979   }

 980   BasicType elem_bt = elem_type->basic_type();
 981   int num_elem = vlen->get_con();
 982 
 983   if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
 984     if (C->print_intrinsics()) {
 985       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
 986                     is_scatter, is_scatter ? "scatter" : "gather",
 987                     num_elem, type2name(elem_bt));






































 988     }
 989     return false; // not supported
 990   }
 991 
 992   // Check that the vector holding indices is supported by architecture
 993   if (!arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
 994       if (C->print_intrinsics()) {
 995         tty->print_cr("  ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int ismask=no",
 996                       is_scatter, is_scatter ? "scatter" : "gather",
 997                       num_elem);
 998       }
 999       return false; // not supported
1000     }
1001 
1002   Node* base = argument(4);
1003   Node* offset = ConvL2X(argument(5));
1004 
1005   // Save state and restore on bailout
1006   uint old_sp = sp();
1007   SafePointNode* old_map = clone_map();
1008 
1009   Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1010 
1011   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1012   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1013 
1014   // The array must be consistent with vector type
1015   if (arr_type == NULL || (arr_type != NULL && !elem_consistent_with_arr(elem_bt, arr_type))) {
1016     if (C->print_intrinsics()) {
1017       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1018                     is_scatter, is_scatter ? "scatter" : "gather",
1019                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1020     }
1021     set_map(old_map);
1022     set_sp(old_sp);
1023     return false;
1024   }

1025   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1026   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1027 
1028   ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1029 
1030   if (vbox_idx_klass == NULL) {
1031     set_map(old_map);
1032     set_sp(old_sp);
1033     return false;
1034   }
1035 
1036   const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1037 
1038   Node* index_vect = unbox_vector(argument(7), vbox_idx_type, T_INT, num_elem);
1039   if (index_vect == NULL) {
1040     set_map(old_map);
1041     set_sp(old_sp);
1042     return false;
1043   }


















1044   const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1045   if (is_scatter) {
1046     Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
1047     if (val == NULL) {
1048       set_map(old_map);
1049       set_sp(old_sp);
1050       return false; // operand unboxing failed
1051     }
1052     set_all_memory(reset_memory());
1053 
1054     Node* vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));





1055     set_memory(vstore, addr_type);
1056   } else {
1057     Node* vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1058 




1059     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1060     set_result(box);
1061   }
1062 
1063   old_map->destruct(&_gvn);
1064 
1065   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1066   return true;
1067 }
1068 
1069 // <V extends Vector<?,?>>
1070 // long reductionCoerced(int oprId, Class<?> vectorClass, Class<?> elementType, int vlen,
1071 //                       V v,
1072 //                       Function<V,Long> defaultImpl)
1073 


1074 bool LibraryCallKit::inline_vector_reduction() {
1075   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
1076   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1077   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1078   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();

1079 
1080   if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
1081       !opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1082     if (C->print_intrinsics()) {
1083       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1084                     NodeClassNames[argument(0)->Opcode()],
1085                     NodeClassNames[argument(1)->Opcode()],
1086                     NodeClassNames[argument(2)->Opcode()],
1087                     NodeClassNames[argument(3)->Opcode()]);
1088     }
1089     return false; // not enough info for intrinsification
1090   }
1091   if (!is_klass_initialized(vector_klass)) {
1092     if (C->print_intrinsics()) {
1093       tty->print_cr("  ** klass argument not initialized");
1094     }
1095     return false;
1096   }
1097   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1098   if (!elem_type->is_primitive_type()) {
1099     if (C->print_intrinsics()) {
1100       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1101     }
1102     return false; // should be primitive type
1103   }


























1104   BasicType elem_bt = elem_type->basic_type();
1105   int num_elem = vlen->get_con();
1106 
1107   int opc  = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1108   int sopc = ReductionNode::opcode(opc, elem_bt);
1109 
1110   // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
1111   if (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed)) {










1112     if (C->print_intrinsics()) {
1113       tty->print_cr("  ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s ismask=no",
1114                     sopc, num_elem, type2name(elem_bt));
1115     }
1116     return false;
1117   }
1118 
1119   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1120   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1121 
1122   Node* opd = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1123   if (opd == NULL) {
1124     return false; // operand unboxing failed
1125   }
1126 















1127   Node* init = ReductionNode::make_reduction_input(gvn(), opc, elem_bt);
1128   Node* rn = gvn().transform(ReductionNode::make(opc, NULL, init, opd, elem_bt));















1129 
1130   Node* bits = NULL;
1131   switch (elem_bt) {
1132     case T_BYTE:
1133     case T_SHORT:
1134     case T_INT: {
1135       bits = gvn().transform(new ConvI2LNode(rn));
1136       break;
1137     }
1138     case T_FLOAT: {
1139       rn   = gvn().transform(new MoveF2INode(rn));
1140       bits = gvn().transform(new ConvI2LNode(rn));
1141       break;
1142     }
1143     case T_DOUBLE: {
1144       bits = gvn().transform(new MoveD2LNode(rn));
1145       break;
1146     }
1147     case T_LONG: {
1148       bits = rn; // no conversion needed
1149       break;
1150     }
1151     default: fatal("%s", type2name(elem_bt));
1152   }
1153   set_result(bits);
1154   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1155   return true;
1156 }
1157 
1158 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1159 //                                V v1, V v2,
1160 //                                BiFunction<V, V, Boolean> defaultImpl) {
1161 //
1162 bool LibraryCallKit::inline_vector_test() {
1163   const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1164   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1165   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1166   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1167 
1168   if (cond == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
1169       !cond->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1170     if (C->print_intrinsics()) {
1171       tty->print_cr("  ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1172                     NodeClassNames[argument(0)->Opcode()],
1173                     NodeClassNames[argument(1)->Opcode()],
1174                     NodeClassNames[argument(2)->Opcode()],
1175                     NodeClassNames[argument(3)->Opcode()]);
1176     }
1177     return false; // not enough info for intrinsification
1178   }
1179   if (!is_klass_initialized(vector_klass)) {
1180     if (C->print_intrinsics()) {

1201                     cond->get_con(), num_elem, type2name(elem_bt),
1202                     is_vector_mask(vbox_klass));
1203     }
1204     return false;
1205   }
1206 
1207   Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1208   Node* opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1209   if (opd1 == NULL || opd2 == NULL) {
1210     return false; // operand unboxing failed
1211   }
1212   Node* test = new VectorTestNode(opd1, opd2, booltest);
1213   test = gvn().transform(test);
1214 
1215   set_result(test);
1216   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1217   return true;
1218 }
1219 
1220 // public static
1221 // <V extends Vector, M extends Mask>
1222 // V blend(Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int vlen,


1223 //         V v1, V v2, M m,
1224 //         VectorBlendOp<V,M> defaultImpl) { ...
1225 //
1226 bool LibraryCallKit::inline_vector_blend() {
1227   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1228   const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1229   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1230   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1231 
1232   if (mask_klass == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL) {
1233     return false; // dead code
1234   }
1235   if (mask_klass->const_oop() == NULL || vector_klass->const_oop() == NULL ||
1236       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1237     if (C->print_intrinsics()) {
1238       tty->print_cr("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1239                     NodeClassNames[argument(0)->Opcode()],
1240                     NodeClassNames[argument(1)->Opcode()],
1241                     NodeClassNames[argument(2)->Opcode()],
1242                     NodeClassNames[argument(3)->Opcode()]);
1243     }
1244     return false; // not enough info for intrinsification
1245   }

1272 
1273   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1274   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1275 
1276   Node* v1   = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1277   Node* v2   = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1278   Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1279 
1280   if (v1 == NULL || v2 == NULL || mask == NULL) {
1281     return false; // operand unboxing failed
1282   }
1283 
1284   Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1285 
1286   Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1287   set_result(box);
1288   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1289   return true;
1290 }
1291 
1292 //  public static <V extends Vector<E,S>,
1293 //          M extends Vector.Mask<E,S>,
1294 //          S extends Vector.Shape, E>
1295 //  M compare(int cond, Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int vlen,
1296 //            V v1, V v2,
1297 //            VectorCompareOp<V,M> defaultImpl) { ...
1298 //
1299 bool LibraryCallKit::inline_vector_compare() {
1300   const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1301   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1302   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
1303   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
1304   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
1305 
1306   if (cond == NULL || vector_klass == NULL || mask_klass == NULL || elem_klass == NULL || vlen == NULL) {
1307     return false; // dead code
1308   }
1309   if (!cond->is_con() || vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
1310       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1311     if (C->print_intrinsics()) {
1312       tty->print_cr("  ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1313                     NodeClassNames[argument(0)->Opcode()],
1314                     NodeClassNames[argument(1)->Opcode()],
1315                     NodeClassNames[argument(2)->Opcode()],
1316                     NodeClassNames[argument(3)->Opcode()],
1317                     NodeClassNames[argument(4)->Opcode()]);
1318     }

1346     }
1347   }
1348 
1349   if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1350     if (C->print_intrinsics()) {
1351       tty->print_cr("  ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1352                     cond->get_con(), num_elem, type2name(elem_bt));
1353     }
1354     return false;
1355   }
1356 
1357   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1358   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1359 
1360   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1361   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1362 
1363   Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1364   Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1365 



















1366   if (v1 == NULL || v2 == NULL) {
1367     return false; // operand unboxing failed
1368   }
1369   BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1370   ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1371 
1372   const TypeVect* vt = TypeVect::make(mask_bt, num_elem);
1373   Node* operation = gvn().transform(new VectorMaskCmpNode(pred, v1, v2, pred_node, vt));












1374 
1375   Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1376   set_result(box);
1377   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1378   return true;
1379 }
1380 
1381 // public static
1382 // <V extends Vector, Sh extends Shuffle>
1383 //  V rearrangeOp(Class<V> vectorClass, Class<Sh> shuffleClass, Class< ? > elementType, int vlen,
1384 //    V v1, Sh sh,
1385 //    VectorSwizzleOp<V, Sh, S, E> defaultImpl) { ...
1386 


1387 bool LibraryCallKit::inline_vector_rearrange() {
1388   const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
1389   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1390   const TypeInstPtr* elem_klass    = gvn().type(argument(2))->isa_instptr();
1391   const TypeInt*     vlen          = gvn().type(argument(3))->isa_int();

1392 
1393   if (vector_klass == NULL || shuffle_klass == NULL || elem_klass == NULL || vlen == NULL) {
1394     return false; // dead code
1395   }
1396   if (shuffle_klass->const_oop() == NULL || vector_klass->const_oop() == NULL ||
1397     elem_klass->const_oop() == NULL || !vlen->is_con()) {


1398     if (C->print_intrinsics()) {
1399       tty->print_cr("  ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1400                     NodeClassNames[argument(0)->Opcode()],
1401                     NodeClassNames[argument(1)->Opcode()],
1402                     NodeClassNames[argument(2)->Opcode()],
1403                     NodeClassNames[argument(3)->Opcode()]);
1404     }
1405     return false; // not enough info for intrinsification
1406   }
1407   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(shuffle_klass)) {

1408     if (C->print_intrinsics()) {
1409       tty->print_cr("  ** klass argument not initialized");
1410     }
1411     return false;
1412   }
1413   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1414   if (!elem_type->is_primitive_type()) {
1415     if (C->print_intrinsics()) {
1416       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1417     }
1418     return false; // should be primitive type
1419   }
1420   BasicType elem_bt = elem_type->basic_type();
1421   BasicType shuffle_bt = elem_bt;
1422   int num_elem = vlen->get_con();
1423 
1424   if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
1425     if (C->print_intrinsics()) {
1426       tty->print_cr("  ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1427                     num_elem, type2name(elem_bt));
1428     }
1429     return false; // not supported
1430   }
1431   if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed)) {






1432     if (C->print_intrinsics()) {
1433       tty->print_cr("  ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
1434                     num_elem, type2name(elem_bt));













1435     }
1436     return false; // not supported
1437   }
1438   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1439   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1440 
1441   ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
1442   const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
1443 
1444   Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1445   Node* shuffle = unbox_vector(argument(5), shbox_type, shuffle_bt, num_elem);
1446 
1447   if (v1 == NULL || shuffle == NULL) {
1448     return false; // operand unboxing failed
1449   }
1450 
1451   Node* rearrange = gvn().transform(new VectorRearrangeNode(v1, shuffle));



























1452 
1453   Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
1454   set_result(box);
1455   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1456   return true;
1457 }
1458 
1459 static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
1460   address addr = NULL;
1461   assert(UseVectorStubs, "sanity");
1462   assert(name_ptr != NULL, "unexpected");
1463   assert((vop >= VectorSupport::VECTOR_OP_SVML_START) && (vop <= VectorSupport::VECTOR_OP_SVML_END), "unexpected");
1464   int op = vop - VectorSupport::VECTOR_OP_SVML_START;
1465 
1466   switch(bits) {
1467     case 64:  //fallthough
1468     case 128: //fallthough
1469     case 256: //fallthough
1470     case 512:
1471       if (bt == T_FLOAT) {

1497 
1498   // Get address for svml method.
1499   address addr = get_svml_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
1500 
1501   if (addr == NULL) {
1502     return NULL;
1503   }
1504 
1505   assert(name != NULL, "name must not be null");
1506   Node* operation = make_runtime_call(RC_VECTOR,
1507                                       call_type,
1508                                       addr,
1509                                       name,
1510                                       TypePtr::BOTTOM,
1511                                       opd1,
1512                                       opd2);
1513   return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
1514 }
1515 
1516 //  public static
1517 //  <V extends Vector<?,?>>
1518 //  V broadcastInt(int opr, Class<V> vectorClass, Class<?> elementType, int vlen,
1519 //                 V v, int i,
1520 //                 VectorBroadcastIntOp<V> defaultImpl) {
1521 //


1522 bool LibraryCallKit::inline_vector_broadcast_int() {
1523   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
1524   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1525   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1526   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();

1527 
1528   if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL) {
1529     return false; // dead code
1530   }
1531   if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1532     if (C->print_intrinsics()) {
1533       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1534                     NodeClassNames[argument(0)->Opcode()],
1535                     NodeClassNames[argument(1)->Opcode()],
1536                     NodeClassNames[argument(2)->Opcode()],
1537                     NodeClassNames[argument(3)->Opcode()]);
1538     }
1539     return false; // not enough info for intrinsification
1540   }
1541   if (!is_klass_initialized(vector_klass)) {
1542     if (C->print_intrinsics()) {
1543       tty->print_cr("  ** klass argument not initialized");
1544     }
1545     return false;
1546   }


























1547   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1548   if (!elem_type->is_primitive_type()) {
1549     if (C->print_intrinsics()) {
1550       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1551     }
1552     return false; // should be primitive type
1553   }
1554   BasicType elem_bt = elem_type->basic_type();
1555   int num_elem = vlen->get_con();

1556   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);

1557   bool is_shift  = VectorNode::is_shift_opcode(opc);
1558   bool is_rotate = VectorNode::is_rotate_opcode(opc);

1559   if (opc == 0 || (!is_shift && !is_rotate)) {
1560     if (C->print_intrinsics()) {
1561       tty->print_cr("  ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
1562     }
1563     return false; // operation not supported
1564   }

1565   int sopc = VectorNode::opcode(opc, elem_bt);
1566   if (sopc == 0) {
1567     if (C->print_intrinsics()) {
1568       tty->print_cr("  ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
1569     }
1570     return false; // operation not supported
1571   }
1572   Node* cnt  = argument(5);

1573   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1574   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1575   const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
1576 
1577   // If CPU supports vector constant rotate instructions pass it directly
1578   bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
1579                          Matcher::supports_vector_constant_rotates(cnt_type->get_con());
1580   bool has_scalar_args = is_rotate ? !is_const_rotate : true;
1581   if (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args)) {
1582     if (C->print_intrinsics()) {
1583       tty->print_cr("  ** not supported: arity=0 op=int/%d vlen=%d etype=%s ismask=no",
1584                     sopc, num_elem, type2name(elem_bt));











1585     }
1586     return false; // not supported
1587   }
1588   Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);

1589   Node* opd2 = NULL;
1590   if (is_shift) {
1591     opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
1592   } else {
1593     assert(is_rotate, "unexpected operation");
1594     if (!is_const_rotate) {
1595       const Type * type_bt = Type::get_const_basic_type(elem_bt);
1596       cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
1597       opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, type_bt));
1598     } else {
1599       // Constant shift value.
1600       opd2 = cnt;
1601     }
1602   }

1603   if (opd1 == NULL || opd2 == NULL) {
1604     return false;
1605   }
1606   Node* operation = gvn().transform(VectorNode::make(opc, opd1, opd2, num_elem, elem_bt));
1607 
























1608   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
1609   set_result(vbox);
1610   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1611   return true;
1612 }
1613 
1614 // public static <VOUT extends VectorPayload,
1615 //                 VIN extends VectorPayload,
1616 //                   S extends VectorSpecies>
1617 // VOUT convert(int oprId,
1618 //           Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
1619 //           Class<?>   toVectorClass, Class<?>   toElementType, int   toVLen,
1620 //           VIN v, S s,
1621 //           VectorConvertOp<VOUT, VIN, S> defaultImpl) {
1622 //
1623 bool LibraryCallKit::inline_vector_convert() {
1624   const TypeInt*     opr               = gvn().type(argument(0))->isa_int();
1625 
1626   const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
1627   const TypeInstPtr* elem_klass_from   = gvn().type(argument(2))->isa_instptr();
1628   const TypeInt*     vlen_from         = gvn().type(argument(3))->isa_int();
1629 
1630   const TypeInstPtr* vector_klass_to   = gvn().type(argument(4))->isa_instptr();
1631   const TypeInstPtr* elem_klass_to     = gvn().type(argument(5))->isa_instptr();
1632   const TypeInt*     vlen_to           = gvn().type(argument(6))->isa_int();
1633 
1634   if (opr == NULL ||
1635       vector_klass_from == NULL || elem_klass_from == NULL || vlen_from == NULL ||
1636       vector_klass_to   == NULL || elem_klass_to   == NULL || vlen_to   == NULL) {
1637     return false; // dead code
1638   }
1639   if (!opr->is_con() ||
1640       vector_klass_from->const_oop() == NULL || elem_klass_from->const_oop() == NULL || !vlen_from->is_con() ||
1641       vector_klass_to->const_oop() == NULL || elem_klass_to->const_oop() == NULL || !vlen_to->is_con()) {

1662          opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
1663   bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST);
1664 
1665   ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
1666   ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
1667   if (is_vector_shuffle(vbox_klass_from)) {
1668     return false; // vector shuffles aren't supported
1669   }
1670   bool is_mask = is_vector_mask(vbox_klass_from);
1671 
1672   ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
1673   if (!elem_type_from->is_primitive_type()) {
1674     return false; // should be primitive type
1675   }
1676   BasicType elem_bt_from = elem_type_from->basic_type();
1677   ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
1678   if (!elem_type_to->is_primitive_type()) {
1679     return false; // should be primitive type
1680   }
1681   BasicType elem_bt_to = elem_type_to->basic_type();
1682   if (is_mask && (type2aelembytes(elem_bt_from) != type2aelembytes(elem_bt_to))) {
1683     return false; // elem size mismatch
1684   }
1685 
1686   int num_elem_from = vlen_from->get_con();
1687   int num_elem_to = vlen_to->get_con();
1688 
1689   // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
1690   // since we may need to change size.
1691   if (!arch_supports_vector(Op_VectorReinterpret,
1692                             num_elem_from,
1693                             elem_bt_from,
1694                             is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
1695     if (C->print_intrinsics()) {
1696       tty->print_cr("  ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
1697                     is_cast ? "cast" : "reinterpret",
1698                     num_elem_from, type2name(elem_bt_from), is_mask);
1699     }
1700     return false;
1701   }
1702 
1703   // Check whether we can support resizing/reinterpreting to the new size.
1704   if (!arch_supports_vector(Op_VectorReinterpret,

1710                     is_cast ? "cast" : "reinterpret",
1711                     num_elem_to, type2name(elem_bt_to), is_mask);
1712     }
1713     return false;
1714   }
1715 
1716   // At this point, we know that both input and output vector registers are supported
1717   // by the architecture. Next check if the casted type is simply to same type - which means
1718   // that it is actually a resize and not a cast.
1719   if (is_cast && elem_bt_from == elem_bt_to) {
1720     is_cast = false;
1721   }
1722 
1723   const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
1724 
1725   Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
1726   if (opd1 == NULL) {
1727     return false;
1728   }
1729 
1730   const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from);
1731   const TypeVect* dst_type = TypeVect::make(elem_bt_to,   num_elem_to);











1732 
1733   Node* op = opd1;
1734   if (is_cast) {
1735     assert(!is_mask, "masks cannot be casted");
1736     int cast_vopc = VectorCastNode::opcode(elem_bt_from);




1737     // Make sure that cast is implemented to particular type/size combination.
1738     if (!arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
1739       if (C->print_intrinsics()) {
1740         tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
1741                       cast_vopc,
1742                       num_elem_to, type2name(elem_bt_to), is_mask);
1743       }
1744       return false;
1745     }
1746 
1747     if (num_elem_from < num_elem_to) {
1748       // Since input and output number of elements are not consistent, we need to make sure we
1749       // properly size. Thus, first make a cast that retains the number of elements from source.
1750       int num_elem_for_cast = num_elem_from;
1751 
1752       // It is possible that arch does not support this intermediate vector size
1753       // TODO More complex logic required here to handle this corner case for the sizes.
1754       if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
1755         if (C->print_intrinsics()) {
1756           tty->print_cr("  ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",

1770 
1771       // It is possible that arch does not support this intermediate vector size
1772       // TODO More complex logic required here to handle this corner case for the sizes.
1773       if (!arch_supports_vector(Op_VectorReinterpret,
1774                                 num_elem_for_resize,
1775                                 elem_bt_from,
1776                                 VecMaskNotUsed)) {
1777         if (C->print_intrinsics()) {
1778           tty->print_cr("  ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
1779                         num_elem_for_resize, type2name(elem_bt_from), is_mask);
1780         }
1781         return false;
1782       }
1783 
1784       op = gvn().transform(new VectorReinterpretNode(op,
1785                                                      src_type,
1786                                                      TypeVect::make(elem_bt_from,
1787                                                                     num_elem_for_resize)));
1788       op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
1789     } else {
1790       // Since input and output number of elements match, and since we know this vector size is
1791       // supported, simply do a cast with no resize needed.
1792       op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));























1793     }
1794   } else if (Type::cmp(src_type, dst_type) != 0) {
1795     assert(!is_cast, "must be reinterpret");
1796     op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
1797   }
1798 
1799   const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
1800   Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
1801   set_result(vbox);
1802   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
1803   return true;
1804 }
1805 
1806 //  public static
1807 //  <V extends Vector<?>>
1808 //  V insert(Class<? extends V> vectorClass, Class<?> elementType, int vlen,

1809 //           V vec, int ix, long val,
1810 //           VecInsertOp<V> defaultImpl) {
1811 //
1812 bool LibraryCallKit::inline_vector_insert() {
1813   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1814   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
1815   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
1816   const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
1817 
1818   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || idx == NULL) {
1819     return false; // dead code
1820   }
1821   if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
1822     if (C->print_intrinsics()) {
1823       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
1824                     NodeClassNames[argument(0)->Opcode()],
1825                     NodeClassNames[argument(1)->Opcode()],
1826                     NodeClassNames[argument(2)->Opcode()],
1827                     NodeClassNames[argument(4)->Opcode()]);
1828     }
1829     return false; // not enough info for intrinsification
1830   }
1831   if (!is_klass_initialized(vector_klass)) {

1880       insert_val = gvn().transform(new MoveI2FNode(insert_val));
1881       break;
1882     case T_DOUBLE:
1883       insert_val = gvn().transform(new MoveL2DNode(insert_val));
1884       break;
1885     case T_LONG:
1886       // no conversion needed
1887       break;
1888     default: fatal("%s", type2name(elem_bt)); break;
1889   }
1890 
1891   Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con()));
1892 
1893   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
1894   set_result(vbox);
1895   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1896   return true;
1897 }
1898 
1899 //  public static
1900 //  <V extends Vector<?>>
1901 //  long extract(Class<?> vectorClass, Class<?> elementType, int vlen,

1902 //               V vec, int ix,
1903 //               VecExtractOp<V> defaultImpl) {
1904 //
1905 bool LibraryCallKit::inline_vector_extract() {
1906   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1907   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
1908   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
1909   const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
1910 
1911   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || idx == NULL) {
1912     return false; // dead code
1913   }
1914   if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
1915     if (C->print_intrinsics()) {
1916       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
1917                     NodeClassNames[argument(0)->Opcode()],
1918                     NodeClassNames[argument(1)->Opcode()],
1919                     NodeClassNames[argument(2)->Opcode()],
1920                     NodeClassNames[argument(4)->Opcode()]);
1921     }
1922     return false; // not enough info for intrinsification
1923   }
1924   if (!is_klass_initialized(vector_klass)) {

  42   ciInstanceKlass* ik = vbox_type->klass()->as_instance_klass();
  43   assert(is_vector(ik), "not a vector");
  44 
  45   ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
  46   assert(fd1 != NULL, "element type info is missing");
  47 
  48   ciConstant val1 = fd1->constant_value();
  49   BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
  50   assert(is_java_primitive(elem_bt), "element type info is missing");
  51 
  52   ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
  53   assert(fd2 != NULL, "vector length info is missing");
  54 
  55   ciConstant val2 = fd2->constant_value();
  56   assert(val2.as_int() > 0, "vector length info is missing");
  57 
  58   return true;
  59 }
  60 #endif
  61 
  62 static bool is_vector_mask(ciKlass* klass) {
  63   return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
  64 }
  65 
  66 static bool is_vector_shuffle(ciKlass* klass) {
  67   return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
  68 }
  69 
  70 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
  71                                                  VectorMaskUseType mask_use_type, bool has_scalar_args) {
  72   bool is_supported = true;
  73 
  74   // has_scalar_args flag is true only for non-constant scalar shift count,
  75   // since in this case shift needs to be broadcasted.
  76   if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
  77        (has_scalar_args &&
  78          !arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed))) {
  79     is_supported = false;
  80   }
  81 
  82   if (is_supported) {
  83     // Check whether mask unboxing is supported.
  84     if ((mask_use_type & VecMaskUseLoad) != 0) {
  85       if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt)) {
  86       #ifndef PRODUCT
  87         if (C->print_intrinsics()) {
  88           tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
  89                         NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
  90         }
  91       #endif
  92         return false;
  93       }
  94     }
  95 
  96     if ((mask_use_type & VecMaskUsePred) != 0) {
  97       if (!Matcher::has_predicated_vectors() ||
  98           !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
  99       #ifndef PRODUCT
 100         if (C->print_intrinsics()) {
 101           tty->print_cr("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
 102                         NodeClassNames[opc], type2name(elem_bt), num_elem);
 103         }
 104       #endif
 105         return false;
 106       }
 107     }
 108   }
 109 
 110   int lshiftopc, rshiftopc;
 111   switch(elem_bt) {
 112     case T_BYTE:
 113       lshiftopc = Op_LShiftI;
 114       rshiftopc = Op_URShiftB;
 115       break;
 116     case T_SHORT:
 117       lshiftopc = Op_LShiftI;
 118       rshiftopc = Op_URShiftS;
 119       break;
 120     case T_INT:
 121       lshiftopc = Op_LShiftI;
 122       rshiftopc = Op_URShiftI;
 123       break;
 124     case T_LONG:
 125       lshiftopc = Op_LShiftL;
 126       rshiftopc = Op_URShiftL;
 127       break;
 128     default:
 129       assert(false, "Unexpected type");
 130   }
 131   int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
 132   int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
 133   if (!is_supported &&
 134       arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
 135       arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
 136       arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
 137     is_supported = true;
 138   }
 139   return is_supported;
 140 }
 141 
 142 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
 143   assert(EnableVectorSupport, "");
 144 
 145   PreserveReexecuteState preexecs(this);
 146   jvms()->set_should_reexecute(true);
 147 
 148   VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
 149   set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
 150   make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
 151   set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
 152   set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
 153   Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
 154 
 155   assert(check_vbox(vbox_type), "");
 156   const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->klass()));
 157   VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
 158   return gvn().transform(vbox);
 159 }
 160 
 161 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) {
 162   assert(EnableVectorSupport, "");
 163   const TypeInstPtr* vbox_type_v = gvn().type(v)->is_instptr();
 164   if (vbox_type->klass() != vbox_type_v->klass()) {
 165     return NULL; // arguments don't agree on vector shapes
 166   }
 167   if (vbox_type_v->maybe_null()) {
 168     return NULL; // no nulls are allowed
 169   }
 170   assert(check_vbox(vbox_type), "");
 171   const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->klass()));
 172   Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector));
 173   return unbox;
 174 }
 175 
 176 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
 177   assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
 178   juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
 179   Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
 180   Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
 181   return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
 182 }
 183 
 184 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
 185   // Check that the operation is valid.
 186   if (sopc <= 0) {
 187 #ifndef PRODUCT
 188     if (C->print_intrinsics()) {
 189       tty->print_cr("  ** Rejected intrinsification because no valid vector op could be extracted");
 190     }
 191 #endif
 192     return false;
 193   }
 194 
 195   if (VectorNode::is_vector_rotate(sopc)) {
 196     if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
 197 #ifndef PRODUCT
 198       if (C->print_intrinsics()) {
 199         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
 200                       NodeClassNames[sopc], type2name(type), num_elem);
 201       }
 202 #endif
 203       return false;
 204     }
 205   } else {
 206     // Check that architecture supports this op-size-type combination.
 207     if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
 208 #ifndef PRODUCT
 209       if (C->print_intrinsics()) {
 210         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support it",
 211                       NodeClassNames[sopc], type2name(type), num_elem);
 212       }
 213 #endif
 214       return false;
 215     } else {
 216       assert(Matcher::match_rule_supported(sopc), "must be supported");

 234         if (C->print_intrinsics()) {
 235           tty->print_cr("  ** Not a svml call or load/store vector op (%s,%s,%d)",
 236                         NodeClassNames[sopc], type2name(type), num_elem);
 237         }
 238 #endif
 239         return false;
 240       }
 241     }
 242   }
 243 
 244   if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
 245       Matcher::supports_vector_variable_shifts() == false) {
 246     if (C->print_intrinsics()) {
 247       tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
 248                     NodeClassNames[sopc], type2name(type), num_elem);
 249     }
 250     return false;
 251   }
 252 
 253   // Check whether mask unboxing is supported.
 254   if ((mask_use_type & VecMaskUseLoad) != 0) {
 255     if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type)) {
 256     #ifndef PRODUCT
 257       if (C->print_intrinsics()) {
 258         tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
 259                       NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
 260       }
 261     #endif
 262       return false;
 263     }
 264   }
 265 
 266   // Check whether mask boxing is supported.
 267   if ((mask_use_type & VecMaskUseStore) != 0) {
 268     if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type)) {
 269     #ifndef PRODUCT
 270       if (C->print_intrinsics()) {
 271         tty->print_cr("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
 272                       NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
 273       }
 274     #endif
 275       return false;
 276     }
 277   }
 278 
 279   if ((mask_use_type & VecMaskUsePred) != 0) {
 280     if (!Matcher::has_predicated_vectors() ||
 281         !Matcher::match_rule_supported_vector_masked(sopc, num_elem, type)) {
 282     #ifndef PRODUCT
 283       if (C->print_intrinsics()) {
 284         tty->print_cr("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
 285                       NodeClassNames[sopc], type2name(type), num_elem);
 286       }
 287     #endif
 288       return false;
 289     }
 290   }
 291 
 292   return true;

 293 }
 294 
 295 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
 296   if (vec_klass->const_oop() == NULL) {
 297     return false; // uninitialized or some kind of unsafe access
 298   }
 299   assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != NULL, "klass instance expected");
 300   ciInstanceKlass* klass =  vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
 301   return klass->is_initialized();
 302 }
 303 
 304 // public static
 305 // <V extends Vector<E>,
 306 //  M extends VectorMask<E>,
 307 //  E>
 308 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 309 //           int length, V v, M m,
 310 //           UnaryOperation<V, M> defaultImpl)
 311 //
 312 // public static
 313 // <V,
 314 //  M extends VectorMask<E>,
 315 //  E>
 316 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 317 //            int length, V v1, V v2, M m,
 318 //            BinaryOperation<V, M> defaultImpl)
 319 //
 320 // public static
 321 // <V extends Vector<E>,
 322 //  M extends VectorMask<E>,
 323 //  E>
 324 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 325 //             int length, V v1, V v2, V v3, M m,
 326 //             TernaryOperation<V, M> defaultImpl)
 327 //
 328 bool LibraryCallKit::inline_vector_nary_operation(int n) {
 329   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
 330   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
 331   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
 332   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
 333   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
 334 
 335   if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
 336       !opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 337     if (C->print_intrinsics()) {
 338       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
 339                     NodeClassNames[argument(0)->Opcode()],
 340                     NodeClassNames[argument(1)->Opcode()],
 341                     NodeClassNames[argument(3)->Opcode()],
 342                     NodeClassNames[argument(4)->Opcode()]);
 343     }
 344     return false; // not enough info for intrinsification
 345   }
 346 
 347   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 348   if (!elem_type->is_primitive_type()) {
 349     if (C->print_intrinsics()) {
 350       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 351     }
 352     return false; // should be primitive type
 353   }
 354   if (!is_klass_initialized(vector_klass)) {
 355     if (C->print_intrinsics()) {
 356       tty->print_cr("  ** klass argument not initialized");
 357     }
 358     return false;
 359   }
 360 
 361   // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
 362   // is used to control this operation.
 363   const Type* vmask_type = gvn().type(argument(n + 5));
 364   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
 365   if (is_masked_op) {
 366     if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
 367       if (C->print_intrinsics()) {
 368         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
 369       }
 370       return false; // not enough info for intrinsification
 371     }
 372 
 373     if (!is_klass_initialized(mask_klass)) {
 374       if (C->print_intrinsics()) {
 375         tty->print_cr("  ** mask klass argument not initialized");
 376       }
 377       return false;
 378     }
 379 
 380     if (vmask_type->maybe_null()) {
 381       if (C->print_intrinsics()) {
 382         tty->print_cr("  ** null mask values are not allowed for masked op");
 383       }
 384       return false;
 385     }
 386   }
 387 
 388   BasicType elem_bt = elem_type->basic_type();
 389   int num_elem = vlen->get_con();
 390   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
 391   int sopc = VectorNode::opcode(opc, elem_bt);
 392   if ((opc != Op_CallLeafVector) && (sopc == 0)) {
 393     if (C->print_intrinsics()) {
 394       tty->print_cr("  ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
 395     }
 396     return false; // operation not supported
 397   }
 398   if (num_elem == 1) {
 399     if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
 400       if (C->print_intrinsics()) {
 401         tty->print_cr("  ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
 402                       n, opc, num_elem, type2name(elem_bt));
 403       }
 404       return false;
 405     }
 406   }
 407   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 408   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 409 
 410   if (is_vector_mask(vbox_klass)) {
 411     assert(!is_masked_op, "mask operations do not need mask to control");
 412   }
 413 
 414   if (opc == Op_CallLeafVector) {
 415     if (!UseVectorStubs) {
 416       if (C->print_intrinsics()) {
 417         tty->print_cr("  ** vector stubs support is disabled");
 418       }
 419       return false;
 420     }
 421     if (!Matcher::supports_vector_calling_convention()) {
 422       if (C->print_intrinsics()) {
 423         tty->print_cr("  ** no vector calling conventions supported");
 424       }
 425       return false;
 426     }
 427     if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
 428       if (C->print_intrinsics()) {
 429         tty->print_cr("  ** vector size (vlen=%d, etype=%s) is not supported",
 430                       num_elem, type2name(elem_bt));
 431       }
 432       return false;
 433     }
 434   }
 435 
 436   // When using mask, mask use type needs to be VecMaskUseLoad.
 437   VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
 438                                       : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
 439   if ((sopc != 0) && !arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
 440     if (C->print_intrinsics()) {
 441       tty->print_cr("  ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
 442                     n, sopc, num_elem, type2name(elem_bt),
 443                     is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
 444     }
 445     return false; // not supported
 446   }
 447 
 448   // Return true if current platform has implemented the masked operation with predicate feature.
 449   bool use_predicate = is_masked_op && sopc != 0 && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
 450   if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
 451     if (C->print_intrinsics()) {
 452       tty->print_cr("  ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
 453                     n, sopc, num_elem, type2name(elem_bt));
 454     }
 455     return false;
 456   }
 457 
 458   Node* opd1 = NULL; Node* opd2 = NULL; Node* opd3 = NULL;
 459   switch (n) {
 460     case 3: {
 461       opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
 462       if (opd3 == NULL) {
 463         if (C->print_intrinsics()) {
 464           tty->print_cr("  ** unbox failed v3=%s",
 465                         NodeClassNames[argument(7)->Opcode()]);
 466         }
 467         return false;
 468       }
 469       // fall-through
 470     }
 471     case 2: {
 472       opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
 473       if (opd2 == NULL) {
 474         if (C->print_intrinsics()) {
 475           tty->print_cr("  ** unbox failed v2=%s",
 476                         NodeClassNames[argument(6)->Opcode()]);
 477         }
 478         return false;
 479       }
 480       // fall-through
 481     }
 482     case 1: {
 483       opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
 484       if (opd1 == NULL) {
 485         if (C->print_intrinsics()) {
 486           tty->print_cr("  ** unbox failed v1=%s",
 487                         NodeClassNames[argument(5)->Opcode()]);
 488         }
 489         return false;
 490       }
 491       break;
 492     }
 493     default: fatal("unsupported arity: %d", n);
 494   }
 495 
 496   Node* mask = NULL;
 497   if (is_masked_op) {
 498     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
 499     assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
 500     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
 501     mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
 502     if (mask == NULL) {
 503       if (C->print_intrinsics()) {
 504         tty->print_cr("  ** unbox failed mask=%s",
 505                       NodeClassNames[argument(n + 5)->Opcode()]);
 506       }
 507       return false;
 508     }
 509   }
 510 
 511   Node* operation = NULL;
 512   if (opc == Op_CallLeafVector) {
 513     assert(UseVectorStubs, "sanity");
 514     operation = gen_call_to_svml(opr->get_con(), elem_bt, num_elem, opd1, opd2);
 515     if (operation == NULL) {
 516       if (C->print_intrinsics()) {
 517         tty->print_cr("  ** svml call failed for %s_%s_%d",
 518                          (elem_bt == T_FLOAT)?"float":"double",
 519                          VectorSupport::svmlname[opr->get_con() - VectorSupport::VECTOR_OP_SVML_START],
 520                          num_elem * type2aelembytes(elem_bt));
 521       }
 522       return false;
 523      }
 524   } else {
 525     const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
 526     switch (n) {
 527       case 1:
 528       case 2: {
 529         operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass));
 530         break;
 531       }
 532       case 3: {
 533         operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
 534         break;
 535       }
 536       default: fatal("unsupported arity: %d", n);
 537     }
 538   }
 539 
 540   if (is_masked_op && mask != NULL) {
 541     if (use_predicate) {
 542       operation->add_req(mask);
 543       operation->add_flag(Node::Flag_is_predicated_vector);
 544     } else {
 545       operation = gvn().transform(operation);
 546       operation = new VectorBlendNode(opd1, operation, mask);
 547     }
 548   }
 549   operation = gvn().transform(operation);
 550 
 551   // Wrap it up in VectorBox to keep object type information.
 552   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
 553   set_result(vbox);
 554   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 555   return true;
 556 }
 557 
 558 // <Sh extends VectorShuffle<E>,  E>
 559 //  Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
 560 //                  int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
 561 bool LibraryCallKit::inline_vector_shuffle_iota() {
 562   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
 563   const TypeInt*     vlen          = gvn().type(argument(3))->isa_int();
 564   const TypeInt*     start_val     = gvn().type(argument(4))->isa_int();
 565   const TypeInt*     step_val      = gvn().type(argument(5))->isa_int();
 566   const TypeInt*     wrap          = gvn().type(argument(6))->isa_int();
 567 
 568   Node* start = argument(4);
 569   Node* step  = argument(5);
 570 
 571   if (shuffle_klass == NULL || vlen == NULL || start_val == NULL || step_val == NULL || wrap == NULL) {
 572     return false; // dead code
 573   }
 574   if (!vlen->is_con() || !is_power_of_2(vlen->get_con()) ||
 575       shuffle_klass->const_oop() == NULL || !wrap->is_con()) {
 576     return false; // not enough info for intrinsification
 577   }
 578   if (!is_klass_initialized(shuffle_klass)) {
 579     if (C->print_intrinsics()) {

 613   if(!step_val->is_con() || !is_power_of_2(step_val->get_con())) {
 614     Node* bcast_step     = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt));
 615     res = gvn().transform(VectorNode::make(Op_MulI, res, bcast_step, num_elem, elem_bt));
 616   } else if (step_val->get_con() > 1) {
 617     Node* cnt = gvn().makecon(TypeInt::make(log2i_exact(step_val->get_con())));
 618     Node* shift_cnt = vector_shift_count(cnt, Op_LShiftI, elem_bt, num_elem);
 619     res = gvn().transform(VectorNode::make(Op_LShiftVB, res, shift_cnt, vt));
 620   }
 621 
 622   if (!start_val->is_con() || start_val->get_con() != 0) {
 623     Node* bcast_start    = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt));
 624     res = gvn().transform(VectorNode::make(Op_AddI, res, bcast_start, num_elem, elem_bt));
 625   }
 626 
 627   Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1));
 628   Node * bcast_mod  = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt));
 629   if(do_wrap)  {
 630     // Wrap the indices greater than lane count.
 631     res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
 632   } else {
 633     ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::ge));
 634     Node * lane_cnt  = gvn().makecon(TypeInt::make(num_elem));
 635     Node * bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt));
 636     const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
 637     Node* mask = gvn().transform(new VectorMaskCmpNode(BoolTest::ge, bcast_lane_cnt, res, pred_node, vmask_type));
 638 
 639     // Make the indices greater than lane count as -ve values. This matches the java side implementation.
 640     res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
 641     Node * biased_val = gvn().transform(VectorNode::make(Op_SubI, res, bcast_lane_cnt, num_elem, elem_bt));
 642     res = gvn().transform(new VectorBlendNode(biased_val, res, mask));
 643   }
 644 
 645   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 646   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 647 
 648   // Wrap it up in VectorBox to keep object type information.
 649   res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
 650   set_result(res);
 651   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 652   return true;
 653 }
 654 
 655 // <E, M>
 656 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
 657 //                          int length, M m, VectorMaskOp<M> defaultImpl)
 658 bool LibraryCallKit::inline_vector_mask_operation() {
 659   const TypeInt*     oper       = gvn().type(argument(0))->isa_int();
 660   const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
 661   const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
 662   const TypeInt*     vlen       = gvn().type(argument(3))->isa_int();
 663   Node*              mask       = argument(4);
 664 
 665   if (mask_klass == NULL || elem_klass == NULL || mask->is_top() || vlen == NULL) {
 666     return false; // dead code
 667   }
 668 
 669   if (!is_klass_initialized(mask_klass)) {
 670     if (C->print_intrinsics()) {
 671       tty->print_cr("  ** klass argument not initialized");
 672     }
 673     return false;
 674   }
 675 
 676   int num_elem = vlen->get_con();

 681     if (C->print_intrinsics()) {
 682       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 683                     Op_LoadVector, num_elem, type2name(T_BOOLEAN));
 684     }
 685     return false; // not supported
 686   }
 687 
 688   int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
 689   if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskNotUsed)) {
 690     if (C->print_intrinsics()) {
 691       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 692                     mopc, num_elem, type2name(elem_bt));
 693     }
 694     return false; // not supported
 695   }
 696 
 697   const Type* elem_ty = Type::get_const_basic_type(elem_bt);
 698   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
 699   const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
 700   Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
 701   if (mask_vec->bottom_type()->isa_vectmask() == NULL) {
 702     mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
 703   }
 704   const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
 705   Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
 706   if (mopc != Op_VectorMaskToLong) {
 707     maskoper = ConvI2L(maskoper);
 708   }
 709   set_result(maskoper);
 710 
 711   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 712   return true;
 713 }
 714 
 715 // public static
 716 // <V,
 717 //  Sh extends VectorShuffle<E>,
 718 //  E>
 719 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
 720 //                   Class<? extends Sh> shuffleClass, Sh s, int length,
 721 //                   ShuffleToVectorOperation<V, Sh, E> defaultImpl)
 722 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
 723   const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
 724   const TypeInstPtr* elem_klass    = gvn().type(argument(1))->isa_instptr();
 725   const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
 726   Node*              shuffle       = argument(3);
 727   const TypeInt*     vlen          = gvn().type(argument(4))->isa_int();
 728 
 729   if (vector_klass == NULL || elem_klass == NULL || shuffle_klass == NULL || shuffle->is_top() || vlen == NULL) {
 730     return false; // dead code
 731   }
 732   if (!vlen->is_con() || vector_klass->const_oop() == NULL || shuffle_klass->const_oop() == NULL) {
 733     return false; // not enough info for intrinsification
 734   }
 735   if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
 736     if (C->print_intrinsics()) {
 737       tty->print_cr("  ** klass argument not initialized");
 738     }
 739     return false;
 740   }
 741 

 760   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 761   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 762 
 763   // Unbox shuffle with true flag to indicate its load shuffle to vector
 764   // shuffle is a byte array
 765   Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, T_BYTE, num_elem, true);
 766 
 767   // cast byte to target element type
 768   shuffle_vec = gvn().transform(VectorCastNode::make(cast_vopc, shuffle_vec, elem_bt, num_elem));
 769 
 770   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 771   const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 772 
 773   // Box vector
 774   Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem);
 775   set_result(res);
 776   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 777   return true;
 778 }
 779 
 780 // public static
 781 // <M,
 782 //  S extends VectorSpecies<E>,
 783 //  E>
 784 // M broadcastCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
 785 //                    long bits, S s,
 786 //                    BroadcastOperation<M, E, S> defaultImpl)
 787 bool LibraryCallKit::inline_vector_broadcast_coerced() {
 788   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 789   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 790   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 791 
 792   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
 793       vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 794     if (C->print_intrinsics()) {
 795       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
 796                     NodeClassNames[argument(0)->Opcode()],
 797                     NodeClassNames[argument(1)->Opcode()],
 798                     NodeClassNames[argument(2)->Opcode()]);
 799     }
 800     return false; // not enough info for intrinsification
 801   }
 802 
 803   if (!is_klass_initialized(vector_klass)) {
 804     if (C->print_intrinsics()) {
 805       tty->print_cr("  ** klass argument not initialized");
 806     }

 813     }
 814     return false; // should be primitive type
 815   }
 816   BasicType elem_bt = elem_type->basic_type();
 817   int num_elem = vlen->get_con();
 818   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 819   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 820 
 821   // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
 822   if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt,
 823                             (is_vector_mask(vbox_klass) ? VecMaskUseStore : VecMaskNotUsed), true /*has_scalar_args*/)) {
 824     if (C->print_intrinsics()) {
 825       tty->print_cr("  ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d",
 826                     num_elem, type2name(elem_bt),
 827                     is_vector_mask(vbox_klass) ? 1 : 0);
 828     }
 829     return false; // not supported
 830   }
 831 
 832   Node* bits = argument(3); // long

 833   Node* elem = NULL;
 834   switch (elem_bt) {
 835     case T_BOOLEAN: // fall-through
 836     case T_BYTE:    // fall-through
 837     case T_SHORT:   // fall-through
 838     case T_CHAR:    // fall-through
 839     case T_INT: {
 840       elem = gvn().transform(new ConvL2INode(bits));
 841       break;
 842     }
 843     case T_DOUBLE: {
 844       elem = gvn().transform(new MoveL2DNode(bits));
 845       break;
 846     }
 847     case T_FLOAT: {
 848       bits = gvn().transform(new ConvL2INode(bits));
 849       elem = gvn().transform(new MoveI2FNode(bits));
 850       break;
 851     }
 852     case T_LONG: {
 853       elem = bits; // no conversion needed
 854       break;
 855     }
 856     default: fatal("%s", type2name(elem_bt));
 857   }
 858 
 859   Node* broadcast = VectorNode::scalar2vector(elem, num_elem, Type::get_const_basic_type(elem_bt), is_vector_mask(vbox_klass));
 860   broadcast = gvn().transform(broadcast);
 861 
 862   Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
 863   set_result(box);
 864   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 865   return true;
 866 }
 867 
 868 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type) {
 869   assert(arr_type != NULL, "unexpected");
 870   BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
 871   if (elem_bt == arr_elem_bt) {
 872     return true;
 873   } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
 874     // Load/store of short vector from/to char[] is supported
 875     return true;
 876   } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
 877     // Load/store of byte vector from/to boolean[] is supported
 878     return true;
 879   } else {
 880     return false;
 881   }
 882 }
 883 
 884 // public static
 885 // <C,
 886 //  VM,
 887 //  E,
 888 //  S extends VectorSpecies<E>>
 889 // VM load(Class<? extends VM> vmClass, Class<E> elementType, int length,
 890 //         Object base, long offset,    // Unsafe addressing
 891 //         C container, int index, S s,     // Arguments for default implementation
 892 //         LoadOperation<C, VM, E, S> defaultImpl)
 893 //
 894 // public static
 895 // <C,
 896 //  V extends Vector<?>>
 897 // void store(Class<?> vectorClass, Class<?> elementType, int length,
 898 //            Object base, long offset,    // Unsafe addressing
 899 //            V v,
 900 //            C container, int index,      // Arguments for default implementation
 901 //            StoreVectorOperation<C, V> defaultImpl)
 902 
 903 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
 904   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 905   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 906   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 907 
 908   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
 909       vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 910     if (C->print_intrinsics()) {
 911       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
 912                     NodeClassNames[argument(0)->Opcode()],
 913                     NodeClassNames[argument(1)->Opcode()],
 914                     NodeClassNames[argument(2)->Opcode()]);
 915     }
 916     return false; // not enough info for intrinsification
 917   }
 918   if (!is_klass_initialized(vector_klass)) {
 919     if (C->print_intrinsics()) {
 920       tty->print_cr("  ** klass argument not initialized");
 921     }

 936   if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
 937     if (C->print_intrinsics()) {
 938       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
 939                     is_store, is_store ? "store" : "load",
 940                     num_elem, type2name(elem_bt));
 941     }
 942     return false; // not supported
 943   }
 944 
 945   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 946   bool is_mask = is_vector_mask(vbox_klass);
 947 
 948   Node* base = argument(3);
 949   Node* offset = ConvL2X(argument(4));
 950 
 951   // Save state and restore on bailout
 952   uint old_sp = sp();
 953   SafePointNode* old_map = clone_map();
 954 
 955   Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
 956 
 957   // The memory barrier checks are based on ones for unsafe access.
 958   // This is not 1-1 implementation.
 959   const Type *const base_type = gvn().type(base);
 960 
 961   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
 962   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
 963 
 964   const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
 965   const bool in_heap   = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
 966 
 967   const bool is_mixed_access = !in_heap && !in_native;
 968 
 969   const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == NULL);
 970 
 971   const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
 972 
 973   // Now handle special case where load/store happens from/to byte array but element type is not byte.
 974   bool using_byte_array = arr_type != NULL && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE;
 975   // Handle loading masks.
 976   // If there is no consistency between array and vector element types, it must be special byte array case or loading masks
 977   if (arr_type != NULL && !using_byte_array && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type)) {
 978     if (C->print_intrinsics()) {
 979       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
 980                     is_store, is_store ? "store" : "load",
 981                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
 982     }
 983     set_map(old_map);
 984     set_sp(old_sp);
 985     return false;
 986   }
 987   // Since we are using byte array, we need to double check that the byte operations are supported by backend.
 988   if (using_byte_array) {
 989     int byte_num_elem = num_elem * type2aelembytes(elem_bt);
 990     if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, byte_num_elem, T_BYTE, VecMaskNotUsed)
 991         || !arch_supports_vector(Op_VectorReinterpret, byte_num_elem, T_BYTE, VecMaskNotUsed)) {
 992       if (C->print_intrinsics()) {

1010       set_sp(old_sp);
1011       return false; // not supported
1012     }
1013     if (!is_store) {
1014       if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1015         set_map(old_map);
1016         set_sp(old_sp);
1017         return false; // not supported
1018       }
1019     } else {
1020       if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
1021         set_map(old_map);
1022         set_sp(old_sp);
1023         return false; // not supported
1024       }
1025     }
1026   }
1027 
1028   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1029 
1030   if (needs_cpu_membar) {
1031     insert_mem_bar(Op_MemBarCPUOrder);
1032   }
1033 
1034   if (is_store) {
1035     Node* val = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1036     if (val == NULL) {
1037       set_map(old_map);
1038       set_sp(old_sp);
1039       return false; // operand unboxing failed
1040     }
1041     set_all_memory(reset_memory());
1042 
1043     // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
1044     int store_num_elem = num_elem;
1045     if (using_byte_array) {
1046       store_num_elem = num_elem * type2aelembytes(elem_bt);
1047       const TypeVect* to_vect_type = TypeVect::make(T_BYTE, store_num_elem);
1048       val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1049     }
1050 
1051     Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
1052     set_memory(vstore, addr_type);
1053   } else {
1054     // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
1055     Node* vload = NULL;
1056     if (using_byte_array) {
1057       int load_num_elem = num_elem * type2aelembytes(elem_bt);
1058       vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, load_num_elem, T_BYTE));
1059       const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1060       vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1061     } else {
1062       // Special handle for masks
1063       if (is_mask) {
1064         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
1065         vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));

1066       } else {
1067         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
1068       }
1069     }
1070     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1071     set_result(box);
1072   }
1073 
1074   old_map->destruct(&_gvn);
1075 
1076   if (needs_cpu_membar) {
1077     insert_mem_bar(Op_MemBarCPUOrder);
1078   }
1079 
1080   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1081   return true;
1082 }
1083 
1084 // public static
1085 // <C,
1086 //  V extends Vector<?>,
1087 //  E,
1088 //  S extends VectorSpecies<E>,
1089 //  M extends VectorMask<E>>
1090 // V loadMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1091 //              int length, Object base, long offset, M m,
1092 //              C container, int index, S s,  // Arguments for default implementation
1093 //              LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1094 //
1095 // public static
1096 // <C,
1097 //  V extends Vector<E>,
1098 //  M extends VectorMask<E>,
1099 //  E>
1100 // void storeMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1101 //                  int length, Object base, long offset,
1102 //                  V v, M m,
1103 //                  C container, int index,  // Arguments for default implementation
1104 //                  StoreVectorMaskedOperation<C, V, M, E> defaultImpl) {
1105 //
1106 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1107   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1108   const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1109   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1110   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1111 
1112   if (vector_klass == NULL || mask_klass == NULL || elem_klass == NULL || vlen == NULL ||
1113       vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
1114       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1115     if (C->print_intrinsics()) {
1116       tty->print_cr("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1117                     NodeClassNames[argument(0)->Opcode()],
1118                     NodeClassNames[argument(1)->Opcode()],
1119                     NodeClassNames[argument(2)->Opcode()],
1120                     NodeClassNames[argument(3)->Opcode()]);
1121     }
1122     return false; // not enough info for intrinsification
1123   }
1124   if (!is_klass_initialized(vector_klass)) {
1125     if (C->print_intrinsics()) {
1126       tty->print_cr("  ** klass argument not initialized");
1127     }
1128     return false;
1129   }
1130 
1131   if (!is_klass_initialized(mask_klass)) {
1132     if (C->print_intrinsics()) {
1133       tty->print_cr("  ** mask klass argument not initialized");
1134     }
1135     return false;
1136   }
1137 
1138   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1139   if (!elem_type->is_primitive_type()) {
1140     if (C->print_intrinsics()) {
1141       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1142     }
1143     return false; // should be primitive type
1144   }
1145 
1146   BasicType elem_bt = elem_type->basic_type();
1147   int num_elem = vlen->get_con();
1148 
1149   Node* base = argument(4);
1150   Node* offset = ConvL2X(argument(5));
1151 
1152   // Save state and restore on bailout
1153   uint old_sp = sp();
1154   SafePointNode* old_map = clone_map();
1155 
1156   Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1157   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1158   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1159 
1160   // Now handle special case where load/store happens from/to byte array but element type is not byte.
1161   bool using_byte_array = arr_type != NULL && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE;
1162   // If there is no consistency between array and vector element types, it must be special byte array case
1163   if (arr_type != NULL && !using_byte_array && !elem_consistent_with_arr(elem_bt, arr_type)) {
1164     if (C->print_intrinsics()) {
1165       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
1166                     is_store, is_store ? "storeMasked" : "loadMasked",
1167                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1168     }
1169     set_map(old_map);
1170     set_sp(old_sp);
1171     return false;
1172   }
1173 
1174   int mem_num_elem = using_byte_array ? num_elem * type2aelembytes(elem_bt) : num_elem;
1175   BasicType mem_elem_bt = using_byte_array ? T_BYTE : elem_bt;
1176   bool use_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
1177                                             mem_num_elem, mem_elem_bt,
1178                                             (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred));
1179   // Masked vector store operation needs the architecture predicate feature. We need to check
1180   // whether the predicated vector operation is supported by backend.
1181   if (is_store && !use_predicate) {
1182     if (C->print_intrinsics()) {
1183       tty->print_cr("  ** not supported: op=storeMasked vlen=%d etype=%s using_byte_array=%d",
1184                     num_elem, type2name(elem_bt), using_byte_array ? 1 : 0);
1185     }
1186     set_map(old_map);
1187     set_sp(old_sp);
1188     return false;
1189   }
1190 
1191   // This only happens for masked vector load. If predicate is not supported, then check whether
1192   // the normal vector load and blend operations are supported by backend.
1193   if (!use_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1194       !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1195     if (C->print_intrinsics()) {
1196       tty->print_cr("  ** not supported: op=loadMasked vlen=%d etype=%s using_byte_array=%d",
1197                     num_elem, type2name(elem_bt), using_byte_array ? 1 : 0);
1198     }
1199     set_map(old_map);
1200     set_sp(old_sp);
1201     return false;
1202   }
1203 
1204   // Since we are using byte array, we need to double check that the vector reinterpret operation
1205   // with byte type is supported by backend.
1206   if (using_byte_array) {
1207     if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1208       if (C->print_intrinsics()) {
1209         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s using_byte_array=1",
1210                       is_store, is_store ? "storeMasked" : "loadMasked",
1211                       num_elem, type2name(elem_bt));
1212       }
1213       set_map(old_map);
1214       set_sp(old_sp);
1215       return false;
1216     }
1217   }
1218 
1219   // Since it needs to unbox the mask, we need to double check that the related load operations
1220   // for mask are supported by backend.
1221   if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1222     if (C->print_intrinsics()) {
1223       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s",
1224                       is_store, is_store ? "storeMasked" : "loadMasked",
1225                       num_elem, type2name(elem_bt));
1226     }
1227     set_map(old_map);
1228     set_sp(old_sp);
1229     return false;
1230   }
1231 
1232   // Can base be NULL? Otherwise, always on-heap access.
1233   bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1234   if (can_access_non_heap) {
1235     insert_mem_bar(Op_MemBarCPUOrder);
1236   }
1237 
1238   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1239   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1240   assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1241   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1242   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1243 
1244   Node* mask = unbox_vector(is_store ? argument(8) : argument(7), mbox_type, elem_bt, num_elem);
1245   if (mask == NULL) {
1246     if (C->print_intrinsics()) {
1247       tty->print_cr("  ** unbox failed mask=%s",
1248                     is_store ? NodeClassNames[argument(8)->Opcode()]
1249                              : NodeClassNames[argument(7)->Opcode()]);
1250     }
1251     set_map(old_map);
1252     set_sp(old_sp);
1253     return false;
1254   }
1255 
1256   if (is_store) {
1257     Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
1258     if (val == NULL) {
1259       if (C->print_intrinsics()) {
1260         tty->print_cr("  ** unbox failed vector=%s",
1261                       NodeClassNames[argument(7)->Opcode()]);
1262       }
1263       set_map(old_map);
1264       set_sp(old_sp);
1265       return false; // operand unboxing failed
1266     }
1267     set_all_memory(reset_memory());
1268 
1269     if (using_byte_array) {
1270       // Reinterpret the incoming vector to byte vector.
1271       const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1272       val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1273       // Reinterpret the vector mask to byte type.
1274       const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1275       const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1276       mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1277     }
1278     Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1279     set_memory(vstore, addr_type);
1280   } else {
1281     Node* vload = NULL;
1282 
1283     if (using_byte_array) {
1284       // Reinterpret the vector mask to byte type.
1285       const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1286       const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1287       mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1288     }
1289 
1290     if (use_predicate) {
1291       // Generate masked load vector node if predicate feature is supported.
1292       const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1293       vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1294     } else {
1295       // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1296       Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1297       zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, Type::get_const_basic_type(mem_elem_bt)));
1298       vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1299       vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1300     }
1301 
1302     if (using_byte_array) {
1303       const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1304       vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1305     }
1306 
1307     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1308     set_result(box);
1309   }
1310 
1311   old_map->destruct(&_gvn);
1312 
1313   if (can_access_non_heap) {
1314     insert_mem_bar(Op_MemBarCPUOrder);
1315   }
1316 
1317   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1318   return true;
1319 }
1320 
1321 // <C,
1322 //  V extends Vector<?>,
1323 //  W extends Vector<Integer>,
1324 //  S extends VectorSpecies<E>,
1325 //  M extends VectorMask<E>,
1326 //  E>
1327 // V loadWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int length,
1328 //               Class<? extends Vector<Integer>> vectorIndexClass,
1329 //               Object base, long offset, // Unsafe addressing
1330 //               W index_vector, M m,
1331 //               C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
1332 //               LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1333 //
1334 //  <C,
1335 //   V extends Vector<E>,
1336 //   W extends Vector<Integer>,
1337 //   M extends VectorMask<E>,
1338 //   E>
1339 //  void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1340 //                    int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset,    // Unsafe addressing
1341 //                    W index_vector, V v, M m,
1342 //                    C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1343 //                    StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1344 //
1345 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1346   const TypeInstPtr* vector_klass     = gvn().type(argument(0))->isa_instptr();
1347   const TypeInstPtr* mask_klass       = gvn().type(argument(1))->isa_instptr();
1348   const TypeInstPtr* elem_klass       = gvn().type(argument(2))->isa_instptr();
1349   const TypeInt*     vlen             = gvn().type(argument(3))->isa_int();
1350   const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1351 
1352   if (vector_klass == NULL || elem_klass == NULL || vector_idx_klass == NULL || vlen == NULL ||
1353       vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || vector_idx_klass->const_oop() == NULL || !vlen->is_con()) {
1354     if (C->print_intrinsics()) {
1355       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1356                     NodeClassNames[argument(0)->Opcode()],

1357                     NodeClassNames[argument(2)->Opcode()],
1358                     NodeClassNames[argument(3)->Opcode()],
1359                     NodeClassNames[argument(4)->Opcode()]);
1360     }
1361     return false; // not enough info for intrinsification
1362   }
1363 
1364   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1365     if (C->print_intrinsics()) {
1366       tty->print_cr("  ** klass argument not initialized");
1367     }
1368     return false;
1369   }
1370 
1371   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1372   if (!elem_type->is_primitive_type()) {
1373     if (C->print_intrinsics()) {
1374       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1375     }
1376     return false; // should be primitive type
1377   }
1378 
1379   BasicType elem_bt = elem_type->basic_type();
1380   int num_elem = vlen->get_con();
1381 
1382   const Type* vmask_type = gvn().type(is_scatter ? argument(10) : argument(9));
1383   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1384   if (is_masked_op) {
1385     if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
1386       if (C->print_intrinsics()) {
1387         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1388       }
1389       return false; // not enough info for intrinsification
1390     }
1391 
1392     if (!is_klass_initialized(mask_klass)) {
1393       if (C->print_intrinsics()) {
1394         tty->print_cr("  ** mask klass argument not initialized");
1395       }
1396       return false;
1397     }
1398 
1399     if (vmask_type->maybe_null()) {
1400       if (C->print_intrinsics()) {
1401         tty->print_cr("  ** null mask values are not allowed for masked op");
1402       }
1403       return false;
1404     }
1405 
1406     // Check whether the predicated gather/scatter node is supported by architecture.
1407     if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt,
1408                               (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred))) {
1409       if (C->print_intrinsics()) {
1410         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1411                       is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1412                       num_elem, type2name(elem_bt));
1413       }
1414       return false; // not supported
1415     }
1416   } else {
1417     // Check whether the normal gather/scatter node is supported for non-masked operation.
1418     if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1419       if (C->print_intrinsics()) {
1420         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1421                       is_scatter, is_scatter ? "scatter" : "gather",
1422                       num_elem, type2name(elem_bt));
1423       }
1424       return false; // not supported
1425     }

1426   }
1427 
1428   // Check that the vector holding indices is supported by architecture
1429   if (!arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
1430       if (C->print_intrinsics()) {
1431         tty->print_cr("  ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1432                       is_scatter, is_scatter ? "scatter" : "gather",
1433                       num_elem, is_masked_op ? 1 : 0);
1434       }
1435       return false; // not supported
1436   }
1437 
1438   Node* base = argument(5);
1439   Node* offset = ConvL2X(argument(6));
1440 
1441   // Save state and restore on bailout
1442   uint old_sp = sp();
1443   SafePointNode* old_map = clone_map();
1444 
1445   Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1446 
1447   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1448   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1449 
1450   // The array must be consistent with vector type
1451   if (arr_type == NULL || (arr_type != NULL && !elem_consistent_with_arr(elem_bt, arr_type))) {
1452     if (C->print_intrinsics()) {
1453       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1454                     is_scatter, is_scatter ? "scatter" : "gather",
1455                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1456     }
1457     set_map(old_map);
1458     set_sp(old_sp);
1459     return false;
1460   }
1461 
1462   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1463   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);

1464   ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();

1465   if (vbox_idx_klass == NULL) {
1466     set_map(old_map);
1467     set_sp(old_sp);
1468     return false;
1469   }
1470 
1471   const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1472   Node* index_vect = unbox_vector(argument(8), vbox_idx_type, T_INT, num_elem);

1473   if (index_vect == NULL) {
1474     set_map(old_map);
1475     set_sp(old_sp);
1476     return false;
1477   }
1478 
1479   Node* mask = NULL;
1480   if (is_masked_op) {
1481     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1482     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1483     mask = unbox_vector(is_scatter ? argument(10) : argument(9), mbox_type, elem_bt, num_elem);
1484     if (mask == NULL) {
1485       if (C->print_intrinsics()) {
1486         tty->print_cr("  ** unbox failed mask=%s",
1487                     is_scatter ? NodeClassNames[argument(10)->Opcode()]
1488                                : NodeClassNames[argument(9)->Opcode()]);
1489       }
1490       set_map(old_map);
1491       set_sp(old_sp);
1492       return false;
1493     }
1494   }
1495 
1496   const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1497   if (is_scatter) {
1498     Node* val = unbox_vector(argument(9), vbox_type, elem_bt, num_elem);
1499     if (val == NULL) {
1500       set_map(old_map);
1501       set_sp(old_sp);
1502       return false; // operand unboxing failed
1503     }
1504     set_all_memory(reset_memory());
1505 
1506     Node* vstore = NULL;
1507     if (mask != NULL) {
1508       vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, index_vect, mask));
1509     } else {
1510       vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1511     }
1512     set_memory(vstore, addr_type);
1513   } else {
1514     Node* vload = NULL;
1515     if (mask != NULL) {
1516       vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_vect, mask));
1517     } else {
1518       vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1519     }
1520     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1521     set_result(box);
1522   }
1523 
1524   old_map->destruct(&_gvn);
1525 
1526   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1527   return true;
1528 }
1529 
1530 // public static
1531 // <V extends Vector<E>,
1532 //  M extends VectorMask<E>,
1533 //  E>
1534 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1535 //                       Class<E> elementType, int length, V v, M m,
1536 //                       ReductionOperation<V, M> defaultImpl)
1537 bool LibraryCallKit::inline_vector_reduction() {
1538   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
1539   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1540   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
1541   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
1542   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
1543 
1544   if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
1545       !opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1546     if (C->print_intrinsics()) {
1547       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1548                     NodeClassNames[argument(0)->Opcode()],
1549                     NodeClassNames[argument(1)->Opcode()],
1550                     NodeClassNames[argument(3)->Opcode()],
1551                     NodeClassNames[argument(4)->Opcode()]);
1552     }
1553     return false; // not enough info for intrinsification
1554   }
1555   if (!is_klass_initialized(vector_klass)) {
1556     if (C->print_intrinsics()) {
1557       tty->print_cr("  ** klass argument not initialized");
1558     }
1559     return false;
1560   }
1561   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1562   if (!elem_type->is_primitive_type()) {
1563     if (C->print_intrinsics()) {
1564       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1565     }
1566     return false; // should be primitive type
1567   }
1568 
1569   const Type* vmask_type = gvn().type(argument(6));
1570   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1571   if (is_masked_op) {
1572     if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
1573       if (C->print_intrinsics()) {
1574         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1575       }
1576       return false; // not enough info for intrinsification
1577     }
1578 
1579     if (!is_klass_initialized(mask_klass)) {
1580       if (C->print_intrinsics()) {
1581         tty->print_cr("  ** mask klass argument not initialized");
1582       }
1583       return false;
1584     }
1585 
1586     if (vmask_type->maybe_null()) {
1587       if (C->print_intrinsics()) {
1588         tty->print_cr("  ** null mask values are not allowed for masked op");
1589       }
1590       return false;
1591     }
1592   }
1593 
1594   BasicType elem_bt = elem_type->basic_type();
1595   int num_elem = vlen->get_con();

1596   int opc  = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1597   int sopc = ReductionNode::opcode(opc, elem_bt);
1598 
1599   // When using mask, mask use type needs to be VecMaskUseLoad.
1600   if (!arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1601     if (C->print_intrinsics()) {
1602       tty->print_cr("  ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1603                     sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1604     }
1605     return false;
1606   }
1607 
1608   // Return true if current platform has implemented the masked operation with predicate feature.
1609   bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1610   if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1611     if (C->print_intrinsics()) {
1612       tty->print_cr("  ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1613                     sopc, num_elem, type2name(elem_bt));
1614     }
1615     return false;
1616   }
1617 
1618   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1619   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1620 
1621   Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1622   if (opd == NULL) {
1623     return false; // operand unboxing failed
1624   }
1625 
1626   Node* mask = NULL;
1627   if (is_masked_op) {
1628     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1629     assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1630     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1631     mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1632     if (mask == NULL) {
1633       if (C->print_intrinsics()) {
1634         tty->print_cr("  ** unbox failed mask=%s",
1635                       NodeClassNames[argument(6)->Opcode()]);
1636       }
1637       return false;
1638     }
1639   }
1640 
1641   Node* init = ReductionNode::make_reduction_input(gvn(), opc, elem_bt);
1642   Node* value = NULL;
1643   if (mask == NULL) {
1644     assert(!is_masked_op, "Masked op needs the mask value never null");
1645     value = ReductionNode::make(opc, NULL, init, opd, elem_bt);
1646   } else {
1647     if (use_predicate) {
1648       value = ReductionNode::make(opc, NULL, init, opd, elem_bt);
1649       value->add_req(mask);
1650       value->add_flag(Node::Flag_is_predicated_vector);
1651     } else {
1652       Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, Type::get_const_basic_type(elem_bt)));
1653       value = gvn().transform(new VectorBlendNode(reduce_identity, opd, mask));
1654       value = ReductionNode::make(opc, NULL, init, value, elem_bt);
1655     }
1656   }
1657   value = gvn().transform(value);
1658 
1659   Node* bits = NULL;
1660   switch (elem_bt) {
1661     case T_BYTE:
1662     case T_SHORT:
1663     case T_INT: {
1664       bits = gvn().transform(new ConvI2LNode(value));
1665       break;
1666     }
1667     case T_FLOAT: {
1668       value = gvn().transform(new MoveF2INode(value));
1669       bits  = gvn().transform(new ConvI2LNode(value));
1670       break;
1671     }
1672     case T_DOUBLE: {
1673       bits = gvn().transform(new MoveD2LNode(value));
1674       break;
1675     }
1676     case T_LONG: {
1677       bits = value; // no conversion needed
1678       break;
1679     }
1680     default: fatal("%s", type2name(elem_bt));
1681   }
1682   set_result(bits);
1683   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1684   return true;
1685 }
1686 
1687 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1688 //                                V v1, V v2,
1689 //                                BiFunction<V, V, Boolean> defaultImpl)
1690 //
1691 bool LibraryCallKit::inline_vector_test() {
1692   const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1693   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1694   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1695   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1696 
1697   if (cond == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
1698       !cond->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1699     if (C->print_intrinsics()) {
1700       tty->print_cr("  ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1701                     NodeClassNames[argument(0)->Opcode()],
1702                     NodeClassNames[argument(1)->Opcode()],
1703                     NodeClassNames[argument(2)->Opcode()],
1704                     NodeClassNames[argument(3)->Opcode()]);
1705     }
1706     return false; // not enough info for intrinsification
1707   }
1708   if (!is_klass_initialized(vector_klass)) {
1709     if (C->print_intrinsics()) {

1730                     cond->get_con(), num_elem, type2name(elem_bt),
1731                     is_vector_mask(vbox_klass));
1732     }
1733     return false;
1734   }
1735 
1736   Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1737   Node* opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1738   if (opd1 == NULL || opd2 == NULL) {
1739     return false; // operand unboxing failed
1740   }
1741   Node* test = new VectorTestNode(opd1, opd2, booltest);
1742   test = gvn().transform(test);
1743 
1744   set_result(test);
1745   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1746   return true;
1747 }
1748 
1749 // public static
1750 // <V extends Vector<E>,
1751 //  M extends VectorMask<E>,
1752 //  E>
1753 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1754 //         V v1, V v2, M m,
1755 //         VectorBlendOp<V, M, E> defaultImpl)

1756 bool LibraryCallKit::inline_vector_blend() {
1757   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1758   const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1759   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1760   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1761 
1762   if (mask_klass == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL) {
1763     return false; // dead code
1764   }
1765   if (mask_klass->const_oop() == NULL || vector_klass->const_oop() == NULL ||
1766       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1767     if (C->print_intrinsics()) {
1768       tty->print_cr("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1769                     NodeClassNames[argument(0)->Opcode()],
1770                     NodeClassNames[argument(1)->Opcode()],
1771                     NodeClassNames[argument(2)->Opcode()],
1772                     NodeClassNames[argument(3)->Opcode()]);
1773     }
1774     return false; // not enough info for intrinsification
1775   }

1802 
1803   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1804   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1805 
1806   Node* v1   = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1807   Node* v2   = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1808   Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1809 
1810   if (v1 == NULL || v2 == NULL || mask == NULL) {
1811     return false; // operand unboxing failed
1812   }
1813 
1814   Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1815 
1816   Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1817   set_result(box);
1818   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1819   return true;
1820 }
1821 
1822 //  public static
1823 //  <V extends Vector<E>,
1824 //   M extends VectorMask<E>,
1825 //   E>
1826 //  M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1827 //            V v1, V v2, M m,
1828 //            VectorCompareOp<V,M> defaultImpl)
1829 bool LibraryCallKit::inline_vector_compare() {
1830   const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1831   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1832   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
1833   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
1834   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
1835 
1836   if (cond == NULL || vector_klass == NULL || mask_klass == NULL || elem_klass == NULL || vlen == NULL) {
1837     return false; // dead code
1838   }
1839   if (!cond->is_con() || vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
1840       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1841     if (C->print_intrinsics()) {
1842       tty->print_cr("  ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1843                     NodeClassNames[argument(0)->Opcode()],
1844                     NodeClassNames[argument(1)->Opcode()],
1845                     NodeClassNames[argument(2)->Opcode()],
1846                     NodeClassNames[argument(3)->Opcode()],
1847                     NodeClassNames[argument(4)->Opcode()]);
1848     }

1876     }
1877   }
1878 
1879   if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1880     if (C->print_intrinsics()) {
1881       tty->print_cr("  ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1882                     cond->get_con(), num_elem, type2name(elem_bt));
1883     }
1884     return false;
1885   }
1886 
1887   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1888   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1889 
1890   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1891   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1892 
1893   Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1894   Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1895 
1896   bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1897   Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : NULL;
1898   if (is_masked_op && mask == NULL) {
1899     if (C->print_intrinsics()) {
1900       tty->print_cr("  ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1901                     cond->get_con(), num_elem, type2name(elem_bt));
1902     }
1903     return false;
1904   }
1905 
1906   bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1907   if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1908     if (C->print_intrinsics()) {
1909       tty->print_cr("  ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1910                     cond->get_con(), num_elem, type2name(elem_bt));
1911     }
1912     return false;
1913   }
1914 
1915   if (v1 == NULL || v2 == NULL) {
1916     return false; // operand unboxing failed
1917   }
1918   BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1919   ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1920 
1921   const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1922   Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1923 
1924   if (is_masked_op) {
1925     if (use_predicate) {
1926       operation->add_req(mask);
1927       operation->add_flag(Node::Flag_is_predicated_vector);
1928     } else {
1929       operation = gvn().transform(operation);
1930       operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1931     }
1932   }
1933 
1934   operation = gvn().transform(operation);
1935 
1936   Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1937   set_result(box);
1938   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1939   return true;
1940 }
1941 
1942 // public static
1943 // <V extends Vector<E>,
1944 //  Sh extends VectorShuffle<E>,
1945 //  M extends VectorMask<E>,
1946 //  E>
1947 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1948 //               V v1, Sh sh, M m,
1949 //               VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1950 bool LibraryCallKit::inline_vector_rearrange() {
1951   const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
1952   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1953   const TypeInstPtr* mask_klass    = gvn().type(argument(2))->isa_instptr();
1954   const TypeInstPtr* elem_klass    = gvn().type(argument(3))->isa_instptr();
1955   const TypeInt*     vlen          = gvn().type(argument(4))->isa_int();
1956 
1957   if (vector_klass == NULL  || shuffle_klass == NULL ||  elem_klass == NULL || vlen == NULL) {
1958     return false; // dead code
1959   }
1960   if (shuffle_klass->const_oop() == NULL ||
1961       vector_klass->const_oop()  == NULL ||
1962       elem_klass->const_oop()    == NULL ||
1963       !vlen->is_con()) {
1964     if (C->print_intrinsics()) {
1965       tty->print_cr("  ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1966                     NodeClassNames[argument(0)->Opcode()],
1967                     NodeClassNames[argument(1)->Opcode()],
1968                     NodeClassNames[argument(3)->Opcode()],
1969                     NodeClassNames[argument(4)->Opcode()]);
1970     }
1971     return false; // not enough info for intrinsification
1972   }
1973   if (!is_klass_initialized(vector_klass)  ||
1974       !is_klass_initialized(shuffle_klass)) {
1975     if (C->print_intrinsics()) {
1976       tty->print_cr("  ** klass argument not initialized");
1977     }
1978     return false;
1979   }
1980   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1981   if (!elem_type->is_primitive_type()) {
1982     if (C->print_intrinsics()) {
1983       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1984     }
1985     return false; // should be primitive type
1986   }
1987   BasicType elem_bt = elem_type->basic_type();
1988   BasicType shuffle_bt = elem_bt;
1989   int num_elem = vlen->get_con();
1990 
1991   if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
1992     if (C->print_intrinsics()) {
1993       tty->print_cr("  ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1994                     num_elem, type2name(elem_bt));
1995     }
1996     return false; // not supported
1997   }
1998 
1999   bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
2000   bool use_predicate = is_masked_op;
2001   if (is_masked_op &&
2002       (mask_klass == NULL ||
2003        mask_klass->const_oop() == NULL ||
2004        !is_klass_initialized(mask_klass))) {
2005     if (C->print_intrinsics()) {
2006       tty->print_cr("  ** mask_klass argument not initialized");
2007     }
2008   }
2009   VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2010   if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
2011     use_predicate = false;
2012     if(!is_masked_op ||
2013        (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2014         !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)     ||
2015         !arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed))) {
2016       if (C->print_intrinsics()) {
2017         tty->print_cr("  ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
2018                       num_elem, type2name(elem_bt));
2019       }
2020       return false; // not supported
2021     }

2022   }
2023   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2024   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2025 
2026   ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
2027   const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
2028 
2029   Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2030   Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
2031 
2032   if (v1 == NULL || shuffle == NULL) {
2033     return false; // operand unboxing failed
2034   }
2035 
2036   Node* mask = NULL;
2037   if (is_masked_op) {
2038     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2039     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2040     mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2041     if (mask == NULL) {
2042       if (C->print_intrinsics()) {
2043         tty->print_cr("  ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
2044                       num_elem, type2name(elem_bt));
2045       }
2046       return false;
2047     }
2048   }
2049 
2050   Node* rearrange = new VectorRearrangeNode(v1, shuffle);
2051   if (is_masked_op) {
2052     if (use_predicate) {
2053       rearrange->add_req(mask);
2054       rearrange->add_flag(Node::Flag_is_predicated_vector);
2055     } else {
2056       const TypeVect* vt = v1->bottom_type()->is_vect();
2057       rearrange = gvn().transform(rearrange);
2058       Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2059       Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, Type::get_const_basic_type(elem_bt)));
2060       rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2061     }
2062   }
2063   rearrange = gvn().transform(rearrange);
2064 
2065   Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2066   set_result(box);
2067   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2068   return true;
2069 }
2070 
2071 static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
2072   address addr = NULL;
2073   assert(UseVectorStubs, "sanity");
2074   assert(name_ptr != NULL, "unexpected");
2075   assert((vop >= VectorSupport::VECTOR_OP_SVML_START) && (vop <= VectorSupport::VECTOR_OP_SVML_END), "unexpected");
2076   int op = vop - VectorSupport::VECTOR_OP_SVML_START;
2077 
2078   switch(bits) {
2079     case 64:  //fallthough
2080     case 128: //fallthough
2081     case 256: //fallthough
2082     case 512:
2083       if (bt == T_FLOAT) {

2109 
2110   // Get address for svml method.
2111   address addr = get_svml_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
2112 
2113   if (addr == NULL) {
2114     return NULL;
2115   }
2116 
2117   assert(name != NULL, "name must not be null");
2118   Node* operation = make_runtime_call(RC_VECTOR,
2119                                       call_type,
2120                                       addr,
2121                                       name,
2122                                       TypePtr::BOTTOM,
2123                                       opd1,
2124                                       opd2);
2125   return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
2126 }
2127 
2128 //  public static
2129 //  <V extends Vector<E>,
2130 //   M extends VectorMask<E>,
2131 //   E>
2132 //  V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2133 //                 Class<E> elementType, int length,
2134 //                 V v, int n, M m,
2135 //                 VectorBroadcastIntOp<V, M> defaultImpl)
2136 bool LibraryCallKit::inline_vector_broadcast_int() {
2137   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
2138   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2139   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
2140   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
2141   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
2142 
2143   if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL) {
2144     return false; // dead code
2145   }
2146   if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
2147     if (C->print_intrinsics()) {
2148       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2149                     NodeClassNames[argument(0)->Opcode()],
2150                     NodeClassNames[argument(1)->Opcode()],
2151                     NodeClassNames[argument(3)->Opcode()],
2152                     NodeClassNames[argument(4)->Opcode()]);
2153     }
2154     return false; // not enough info for intrinsification
2155   }
2156   if (!is_klass_initialized(vector_klass)) {
2157     if (C->print_intrinsics()) {
2158       tty->print_cr("  ** klass argument not initialized");
2159     }
2160     return false;
2161   }
2162 
2163   const Type* vmask_type = gvn().type(argument(7));
2164   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2165   if (is_masked_op) {
2166     if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
2167       if (C->print_intrinsics()) {
2168         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2169       }
2170       return false; // not enough info for intrinsification
2171     }
2172 
2173     if (!is_klass_initialized(mask_klass)) {
2174       if (C->print_intrinsics()) {
2175         tty->print_cr("  ** mask klass argument not initialized");
2176       }
2177       return false;
2178     }
2179 
2180     if (vmask_type->maybe_null()) {
2181       if (C->print_intrinsics()) {
2182         tty->print_cr("  ** null mask values are not allowed for masked op");
2183       }
2184       return false;
2185     }
2186   }
2187 
2188   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2189   if (!elem_type->is_primitive_type()) {
2190     if (C->print_intrinsics()) {
2191       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2192     }
2193     return false; // should be primitive type
2194   }
2195 
2196   int num_elem = vlen->get_con();
2197   BasicType elem_bt = elem_type->basic_type();
2198   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2199 
2200   bool is_shift  = VectorNode::is_shift_opcode(opc);
2201   bool is_rotate = VectorNode::is_rotate_opcode(opc);
2202 
2203   if (opc == 0 || (!is_shift && !is_rotate)) {
2204     if (C->print_intrinsics()) {
2205       tty->print_cr("  ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2206     }
2207     return false; // operation not supported
2208   }
2209 
2210   int sopc = VectorNode::opcode(opc, elem_bt);
2211   if (sopc == 0) {
2212     if (C->print_intrinsics()) {
2213       tty->print_cr("  ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2214     }
2215     return false; // operation not supported
2216   }
2217 
2218   Node* cnt  = argument(6);
2219   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2220   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2221   const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2222 
2223   // If CPU supports vector constant rotate instructions pass it directly
2224   bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2225                          Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2226   bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2227 
2228   VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2229   bool use_predicate = is_masked_op;
2230 
2231   if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2232     use_predicate = false;
2233     if (!is_masked_op ||
2234         (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2235          !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2236 
2237       if (C->print_intrinsics()) {
2238         tty->print_cr("  ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2239                       sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2240       }
2241       return false; // not supported
2242     }

2243   }
2244 
2245   Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2246   Node* opd2 = NULL;
2247   if (is_shift) {
2248     opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2249   } else {
2250     assert(is_rotate, "unexpected operation");
2251     if (!is_const_rotate) {
2252       const Type * type_bt = Type::get_const_basic_type(elem_bt);
2253       cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2254       opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, type_bt));
2255     } else {
2256       // Constant shift value.
2257       opd2 = cnt;
2258     }
2259   }
2260 
2261   if (opd1 == NULL || opd2 == NULL) {
2262     return false;
2263   }

2264 
2265   Node* mask = NULL;
2266   if (is_masked_op) {
2267     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2268     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2269     mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2270     if (mask == NULL) {
2271       if (C->print_intrinsics()) {
2272         tty->print_cr("  ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2273       }
2274       return false;
2275     }
2276   }
2277 
2278   Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2279   if (is_masked_op && mask != NULL) {
2280     if (use_predicate) {
2281       operation->add_req(mask);
2282       operation->add_flag(Node::Flag_is_predicated_vector);
2283     } else {
2284       operation = gvn().transform(operation);
2285       operation = new VectorBlendNode(opd1, operation, mask);
2286     }
2287   }
2288   operation = gvn().transform(operation);
2289   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2290   set_result(vbox);
2291   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2292   return true;
2293 }
2294 
2295 // public static <VOUT extends VectorPayload,
2296 //                 VIN extends VectorPayload,
2297 //                   S extends VectorSpecies>
2298 // VOUT convert(int oprId,
2299 //           Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2300 //           Class<?>   toVectorClass, Class<?>   toElementType, int   toVLen,
2301 //           VIN v, S s,
2302 //           VectorConvertOp<VOUT, VIN, S> defaultImpl)
2303 //
2304 bool LibraryCallKit::inline_vector_convert() {
2305   const TypeInt*     opr               = gvn().type(argument(0))->isa_int();
2306 
2307   const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2308   const TypeInstPtr* elem_klass_from   = gvn().type(argument(2))->isa_instptr();
2309   const TypeInt*     vlen_from         = gvn().type(argument(3))->isa_int();
2310 
2311   const TypeInstPtr* vector_klass_to   = gvn().type(argument(4))->isa_instptr();
2312   const TypeInstPtr* elem_klass_to     = gvn().type(argument(5))->isa_instptr();
2313   const TypeInt*     vlen_to           = gvn().type(argument(6))->isa_int();
2314 
2315   if (opr == NULL ||
2316       vector_klass_from == NULL || elem_klass_from == NULL || vlen_from == NULL ||
2317       vector_klass_to   == NULL || elem_klass_to   == NULL || vlen_to   == NULL) {
2318     return false; // dead code
2319   }
2320   if (!opr->is_con() ||
2321       vector_klass_from->const_oop() == NULL || elem_klass_from->const_oop() == NULL || !vlen_from->is_con() ||
2322       vector_klass_to->const_oop() == NULL || elem_klass_to->const_oop() == NULL || !vlen_to->is_con()) {

2343          opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2344   bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST);
2345 
2346   ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2347   ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2348   if (is_vector_shuffle(vbox_klass_from)) {
2349     return false; // vector shuffles aren't supported
2350   }
2351   bool is_mask = is_vector_mask(vbox_klass_from);
2352 
2353   ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2354   if (!elem_type_from->is_primitive_type()) {
2355     return false; // should be primitive type
2356   }
2357   BasicType elem_bt_from = elem_type_from->basic_type();
2358   ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2359   if (!elem_type_to->is_primitive_type()) {
2360     return false; // should be primitive type
2361   }
2362   BasicType elem_bt_to = elem_type_to->basic_type();



2363 
2364   int num_elem_from = vlen_from->get_con();
2365   int num_elem_to = vlen_to->get_con();
2366 
2367   // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2368   // since we may need to change size.
2369   if (!arch_supports_vector(Op_VectorReinterpret,
2370                             num_elem_from,
2371                             elem_bt_from,
2372                             is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2373     if (C->print_intrinsics()) {
2374       tty->print_cr("  ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2375                     is_cast ? "cast" : "reinterpret",
2376                     num_elem_from, type2name(elem_bt_from), is_mask);
2377     }
2378     return false;
2379   }
2380 
2381   // Check whether we can support resizing/reinterpreting to the new size.
2382   if (!arch_supports_vector(Op_VectorReinterpret,

2388                     is_cast ? "cast" : "reinterpret",
2389                     num_elem_to, type2name(elem_bt_to), is_mask);
2390     }
2391     return false;
2392   }
2393 
2394   // At this point, we know that both input and output vector registers are supported
2395   // by the architecture. Next check if the casted type is simply to same type - which means
2396   // that it is actually a resize and not a cast.
2397   if (is_cast && elem_bt_from == elem_bt_to) {
2398     is_cast = false;
2399   }
2400 
2401   const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2402 
2403   Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2404   if (opd1 == NULL) {
2405     return false;
2406   }
2407 
2408   const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2409   const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2410 
2411   // Safety check to prevent casting if source mask is of type vector
2412   // and destination mask of type predicate vector and vice-versa.
2413   // From X86 standpoint, this case will only arise over KNL target,
2414   // where certain masks (depending on the species) are either propagated
2415   // through a vector or predicate register.
2416   if (is_mask &&
2417       ((src_type->isa_vectmask() == NULL && dst_type->isa_vectmask()) ||
2418        (dst_type->isa_vectmask() == NULL && src_type->isa_vectmask()))) {
2419     return false;
2420   }
2421 
2422   Node* op = opd1;
2423   if (is_cast) {
2424     BasicType new_elem_bt_to = elem_bt_to;
2425     BasicType new_elem_bt_from = elem_bt_from;
2426     if (is_mask && is_floating_point_type(elem_bt_from)) {
2427       new_elem_bt_from = elem_bt_from == T_FLOAT ? T_INT : T_LONG;
2428     }
2429     int cast_vopc = VectorCastNode::opcode(new_elem_bt_from);
2430     // Make sure that cast is implemented to particular type/size combination.
2431     if (!arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2432       if (C->print_intrinsics()) {
2433         tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2434                       cast_vopc,
2435                       num_elem_to, type2name(elem_bt_to), is_mask);
2436       }
2437       return false;
2438     }
2439 
2440     if (num_elem_from < num_elem_to) {
2441       // Since input and output number of elements are not consistent, we need to make sure we
2442       // properly size. Thus, first make a cast that retains the number of elements from source.
2443       int num_elem_for_cast = num_elem_from;
2444 
2445       // It is possible that arch does not support this intermediate vector size
2446       // TODO More complex logic required here to handle this corner case for the sizes.
2447       if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2448         if (C->print_intrinsics()) {
2449           tty->print_cr("  ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",

2463 
2464       // It is possible that arch does not support this intermediate vector size
2465       // TODO More complex logic required here to handle this corner case for the sizes.
2466       if (!arch_supports_vector(Op_VectorReinterpret,
2467                                 num_elem_for_resize,
2468                                 elem_bt_from,
2469                                 VecMaskNotUsed)) {
2470         if (C->print_intrinsics()) {
2471           tty->print_cr("  ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2472                         num_elem_for_resize, type2name(elem_bt_from), is_mask);
2473         }
2474         return false;
2475       }
2476 
2477       op = gvn().transform(new VectorReinterpretNode(op,
2478                                                      src_type,
2479                                                      TypeVect::make(elem_bt_from,
2480                                                                     num_elem_for_resize)));
2481       op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2482     } else {
2483       if (is_mask) {
2484         if ((dst_type->isa_vectmask() && src_type->isa_vectmask()) ||
2485             (type2aelembytes(elem_bt_from) == type2aelembytes(elem_bt_to))) {
2486           op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2487         } else {
2488           // Special handling for casting operation involving floating point types.
2489           // Case A) F -> X :=  F -> VectorMaskCast (F->I/L [NOP]) -> VectorCast[I/L]2X
2490           // Case B) X -> F :=  X -> VectorCastX2[I/L] -> VectorMaskCast ([I/L]->F [NOP])
2491           // Case C) F -> F :=  VectorMaskCast (F->I/L [NOP]) -> VectorCast[I/L]2[L/I] -> VectotMaskCast (L/I->F [NOP])
2492           if (is_floating_point_type(elem_bt_from)) {
2493             const TypeVect* new_src_type = TypeVect::make(new_elem_bt_from, num_elem_to, is_mask);
2494             op = gvn().transform(new VectorMaskCastNode(op, new_src_type));
2495           }
2496           if (is_floating_point_type(elem_bt_to)) {
2497             new_elem_bt_to = elem_bt_to == T_FLOAT ? T_INT : T_LONG;
2498           }
2499           op = gvn().transform(VectorCastNode::make(cast_vopc, op, new_elem_bt_to, num_elem_to));
2500           if (new_elem_bt_to != elem_bt_to) {
2501             op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2502           }
2503         }
2504       } else {
2505         // Since input and output number of elements match, and since we know this vector size is
2506         // supported, simply do a cast with no resize needed.
2507         op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2508       }
2509     }
2510   } else if (Type::cmp(src_type, dst_type) != 0) {
2511     assert(!is_cast, "must be reinterpret");
2512     op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2513   }
2514 
2515   const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2516   Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2517   set_result(vbox);
2518   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2519   return true;
2520 }
2521 
2522 //  public static
2523 //  <V extends Vector<E>,
2524 //   E>
2525 //  V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2526 //           V vec, int ix, long val,
2527 //           VecInsertOp<V> defaultImpl)

2528 bool LibraryCallKit::inline_vector_insert() {
2529   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2530   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2531   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2532   const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
2533 
2534   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || idx == NULL) {
2535     return false; // dead code
2536   }
2537   if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
2538     if (C->print_intrinsics()) {
2539       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2540                     NodeClassNames[argument(0)->Opcode()],
2541                     NodeClassNames[argument(1)->Opcode()],
2542                     NodeClassNames[argument(2)->Opcode()],
2543                     NodeClassNames[argument(4)->Opcode()]);
2544     }
2545     return false; // not enough info for intrinsification
2546   }
2547   if (!is_klass_initialized(vector_klass)) {

2596       insert_val = gvn().transform(new MoveI2FNode(insert_val));
2597       break;
2598     case T_DOUBLE:
2599       insert_val = gvn().transform(new MoveL2DNode(insert_val));
2600       break;
2601     case T_LONG:
2602       // no conversion needed
2603       break;
2604     default: fatal("%s", type2name(elem_bt)); break;
2605   }
2606 
2607   Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con()));
2608 
2609   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2610   set_result(vbox);
2611   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2612   return true;
2613 }
2614 
2615 //  public static
2616 //  <V extends Vector<E>,
2617 //   E>
2618 //  long extract(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2619 //               V vec, int ix,
2620 //               VecExtractOp<V> defaultImpl)

2621 bool LibraryCallKit::inline_vector_extract() {
2622   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2623   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2624   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2625   const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
2626 
2627   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || idx == NULL) {
2628     return false; // dead code
2629   }
2630   if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
2631     if (C->print_intrinsics()) {
2632       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2633                     NodeClassNames[argument(0)->Opcode()],
2634                     NodeClassNames[argument(1)->Opcode()],
2635                     NodeClassNames[argument(2)->Opcode()],
2636                     NodeClassNames[argument(4)->Opcode()]);
2637     }
2638     return false; // not enough info for intrinsification
2639   }
2640   if (!is_klass_initialized(vector_klass)) {
< prev index next >