1 /*
   2  * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "ci/ciSymbols.hpp"
  27 #include "classfile/vmSymbols.hpp"
  28 #include "opto/library_call.hpp"
  29 #include "opto/runtime.hpp"
  30 #include "opto/vectornode.hpp"
  31 #include "prims/vectorSupport.hpp"
  32 #include "runtime/stubRoutines.hpp"
  33 
  34 #ifdef ASSERT
  35 static bool is_vector(ciKlass* klass) {
  36   return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
  37 }
  38 
  39 static bool check_vbox(const TypeInstPtr* vbox_type) {
  40   assert(vbox_type->klass_is_exact(), "");
  41 
  42   ciInstanceKlass* ik = vbox_type->klass()->as_instance_klass();
  43   assert(is_vector(ik), "not a vector");
  44 
  45   ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
  46   assert(fd1 != NULL, "element type info is missing");
  47 
  48   ciConstant val1 = fd1->constant_value();
  49   BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
  50   assert(is_java_primitive(elem_bt), "element type info is missing");
  51 
  52   ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
  53   assert(fd2 != NULL, "vector length info is missing");
  54 
  55   ciConstant val2 = fd2->constant_value();
  56   assert(val2.as_int() > 0, "vector length info is missing");
  57 
  58   return true;
  59 }
  60 #endif
  61 
  62 static bool is_vector_mask(ciKlass* klass) {
  63   return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
  64 }
  65 
  66 static bool is_vector_shuffle(ciKlass* klass) {
  67   return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
  68 }
  69 
  70 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
  71                                                  VectorMaskUseType mask_use_type, bool has_scalar_args) {
  72   bool is_supported = true;
  73 
  74   // has_scalar_args flag is true only for non-constant scalar shift count,
  75   // since in this case shift needs to be broadcasted.
  76   if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
  77        (has_scalar_args &&
  78          !arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed))) {
  79     is_supported = false;
  80   }
  81 
  82   if (is_supported) {
  83     // Check whether mask unboxing is supported.
  84     if ((mask_use_type & VecMaskUseLoad) != 0) {
  85       if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt)) {

  86       #ifndef PRODUCT
  87         if (C->print_intrinsics()) {
  88           tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
  89                         NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
  90         }
  91       #endif
  92         return false;
  93       }
  94     }
  95 
  96     if ((mask_use_type & VecMaskUsePred) != 0) {
  97       if (!Matcher::has_predicated_vectors() ||
  98           !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
  99       #ifndef PRODUCT
 100         if (C->print_intrinsics()) {
 101           tty->print_cr("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
 102                         NodeClassNames[opc], type2name(elem_bt), num_elem);
 103         }
 104       #endif
 105         return false;
 106       }
 107     }
 108   }
 109 
 110   int lshiftopc, rshiftopc;
 111   switch(elem_bt) {
 112     case T_BYTE:
 113       lshiftopc = Op_LShiftI;
 114       rshiftopc = Op_URShiftB;
 115       break;
 116     case T_SHORT:
 117       lshiftopc = Op_LShiftI;
 118       rshiftopc = Op_URShiftS;
 119       break;
 120     case T_INT:
 121       lshiftopc = Op_LShiftI;
 122       rshiftopc = Op_URShiftI;
 123       break;
 124     case T_LONG:
 125       lshiftopc = Op_LShiftL;
 126       rshiftopc = Op_URShiftL;
 127       break;
 128     default:
 129       assert(false, "Unexpected type");
 130   }
 131   int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
 132   int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
 133   if (!is_supported &&
 134       arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
 135       arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
 136       arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
 137     is_supported = true;
 138   }
 139   return is_supported;
 140 }
 141 
 142 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
 143   assert(EnableVectorSupport, "");
 144 
 145   PreserveReexecuteState preexecs(this);
 146   jvms()->set_should_reexecute(true);
 147 
 148   VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
 149   set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
 150   make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
 151   set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
 152   set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
 153   Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
 154 
 155   assert(check_vbox(vbox_type), "");
 156   const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->klass()));
 157   VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
 158   return gvn().transform(vbox);
 159 }
 160 
 161 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) {
 162   assert(EnableVectorSupport, "");
 163   const TypeInstPtr* vbox_type_v = gvn().type(v)->is_instptr();
 164   if (vbox_type->klass() != vbox_type_v->klass()) {
 165     return NULL; // arguments don't agree on vector shapes
 166   }
 167   if (vbox_type_v->maybe_null()) {
 168     return NULL; // no nulls are allowed
 169   }
 170   assert(check_vbox(vbox_type), "");
 171   const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->klass()));
 172   Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector));
 173   return unbox;
 174 }
 175 
 176 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
 177   assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
 178   juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
 179   Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
 180   Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
 181   return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
 182 }
 183 
 184 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
 185   // Check that the operation is valid.
 186   if (sopc <= 0) {
 187 #ifndef PRODUCT
 188     if (C->print_intrinsics()) {
 189       tty->print_cr("  ** Rejected intrinsification because no valid vector op could be extracted");
 190     }
 191 #endif
 192     return false;
 193   }
 194 
 195   if (VectorNode::is_vector_rotate(sopc)) {
 196     if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
 197 #ifndef PRODUCT
 198       if (C->print_intrinsics()) {
 199         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
 200                       NodeClassNames[sopc], type2name(type), num_elem);
 201       }
 202 #endif
 203       return false;
 204     }
 205   } else if (VectorNode::is_vector_integral_negate(sopc)) {
 206     if (!VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, false)) {
 207 #ifndef PRODUCT
 208       if (C->print_intrinsics()) {
 209         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support integral vector negate",
 210                       NodeClassNames[sopc], type2name(type), num_elem);
 211       }
 212 #endif
 213       return false;
 214     }
 215   } else {
 216     // Check that architecture supports this op-size-type combination.
 217     if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
 218 #ifndef PRODUCT
 219       if (C->print_intrinsics()) {
 220         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support it",
 221                       NodeClassNames[sopc], type2name(type), num_elem);
 222       }
 223 #endif
 224       return false;
 225     } else {
 226       assert(Matcher::match_rule_supported(sopc), "must be supported");
 227     }
 228   }
 229 
 230   if (num_elem == 1) {
 231     if (mask_use_type != VecMaskNotUsed) {
 232 #ifndef PRODUCT
 233       if (C->print_intrinsics()) {
 234         tty->print_cr("  ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
 235                       NodeClassNames[sopc], type2name(type), num_elem);
 236       }
 237 #endif
 238       return false;
 239     }
 240 
 241     if (sopc != 0) {
 242       if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
 243 #ifndef PRODUCT
 244         if (C->print_intrinsics()) {
 245           tty->print_cr("  ** Not a svml call or load/store vector op (%s,%s,%d)",
 246                         NodeClassNames[sopc], type2name(type), num_elem);
 247         }
 248 #endif
 249         return false;
 250       }
 251     }
 252   }
 253 
 254   if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
 255       Matcher::supports_vector_variable_shifts() == false) {
 256     if (C->print_intrinsics()) {
 257       tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
 258                     NodeClassNames[sopc], type2name(type), num_elem);
 259     }
 260     return false;
 261   }
 262 
 263   // Check whether mask unboxing is supported.
 264   if ((mask_use_type & VecMaskUseLoad) != 0) {
 265     if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type)) {

 266     #ifndef PRODUCT
 267       if (C->print_intrinsics()) {
 268         tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
 269                       NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
 270       }
 271     #endif
 272       return false;
 273     }
 274   }
 275 
 276   // Check whether mask boxing is supported.
 277   if ((mask_use_type & VecMaskUseStore) != 0) {
 278     if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type)) {

 279     #ifndef PRODUCT
 280       if (C->print_intrinsics()) {
 281         tty->print_cr("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
 282                       NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
 283       }
 284     #endif
 285       return false;
 286     }
 287   }
 288 
 289   if ((mask_use_type & VecMaskUsePred) != 0) {
 290     bool is_supported = false;
 291     if (Matcher::has_predicated_vectors()) {
 292       if (VectorNode::is_vector_integral_negate(sopc)) {
 293         is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
 294       } else {
 295         is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
 296       }
 297     }
 298 
 299     if (!is_supported) {
 300     #ifndef PRODUCT
 301       if (C->print_intrinsics()) {
 302         tty->print_cr("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
 303                       NodeClassNames[sopc], type2name(type), num_elem);
 304       }
 305     #endif
 306       return false;
 307     }
 308   }
 309 
 310   return true;
 311 }
 312 
 313 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
 314   if (vec_klass->const_oop() == NULL) {
 315     return false; // uninitialized or some kind of unsafe access
 316   }
 317   assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != NULL, "klass instance expected");
 318   ciInstanceKlass* klass =  vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
 319   return klass->is_initialized();
 320 }
 321 
 322 // public static
 323 // <V extends Vector<E>,
 324 //  M extends VectorMask<E>,
 325 //  E>
 326 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 327 //           int length, V v, M m,
 328 //           UnaryOperation<V, M> defaultImpl)
 329 //
 330 // public static
 331 // <V,
 332 //  M extends VectorMask<E>,
 333 //  E>
 334 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 335 //            int length, V v1, V v2, M m,
 336 //            BinaryOperation<V, M> defaultImpl)
 337 //
 338 // public static
 339 // <V extends Vector<E>,
 340 //  M extends VectorMask<E>,
 341 //  E>
 342 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 343 //             int length, V v1, V v2, V v3, M m,
 344 //             TernaryOperation<V, M> defaultImpl)
 345 //
 346 bool LibraryCallKit::inline_vector_nary_operation(int n) {
 347   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
 348   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
 349   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
 350   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
 351   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
 352 
 353   if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
 354       !opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 355     if (C->print_intrinsics()) {
 356       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
 357                     NodeClassNames[argument(0)->Opcode()],
 358                     NodeClassNames[argument(1)->Opcode()],
 359                     NodeClassNames[argument(3)->Opcode()],
 360                     NodeClassNames[argument(4)->Opcode()]);
 361     }
 362     return false; // not enough info for intrinsification
 363   }
 364 
 365   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 366   if (!elem_type->is_primitive_type()) {
 367     if (C->print_intrinsics()) {
 368       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 369     }
 370     return false; // should be primitive type
 371   }
 372   if (!is_klass_initialized(vector_klass)) {
 373     if (C->print_intrinsics()) {
 374       tty->print_cr("  ** klass argument not initialized");
 375     }
 376     return false;
 377   }
 378 
 379   // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
 380   // is used to control this operation.
 381   const Type* vmask_type = gvn().type(argument(n + 5));
 382   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
 383   if (is_masked_op) {
 384     if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
 385       if (C->print_intrinsics()) {
 386         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
 387       }
 388       return false; // not enough info for intrinsification
 389     }
 390 
 391     if (!is_klass_initialized(mask_klass)) {
 392       if (C->print_intrinsics()) {
 393         tty->print_cr("  ** mask klass argument not initialized");
 394       }
 395       return false;
 396     }
 397 
 398     if (vmask_type->maybe_null()) {
 399       if (C->print_intrinsics()) {
 400         tty->print_cr("  ** null mask values are not allowed for masked op");
 401       }
 402       return false;
 403     }
 404   }
 405 
 406   BasicType elem_bt = elem_type->basic_type();
 407   int num_elem = vlen->get_con();
 408   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
 409   int sopc = VectorNode::opcode(opc, elem_bt);
 410   if ((opc != Op_CallLeafVector) && (sopc == 0)) {
 411     if (C->print_intrinsics()) {
 412       tty->print_cr("  ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
 413     }
 414     return false; // operation not supported
 415   }
 416   if (num_elem == 1) {
 417     if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
 418       if (C->print_intrinsics()) {
 419         tty->print_cr("  ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
 420                       n, opc, num_elem, type2name(elem_bt));
 421       }
 422       return false;
 423     }
 424   }
 425   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 426   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 427 
 428   if (is_vector_mask(vbox_klass)) {
 429     assert(!is_masked_op, "mask operations do not need mask to control");
 430   }
 431 
 432   if (opc == Op_CallLeafVector) {
 433     if (!UseVectorStubs) {
 434       if (C->print_intrinsics()) {
 435         tty->print_cr("  ** vector stubs support is disabled");
 436       }
 437       return false;
 438     }
 439     if (!Matcher::supports_vector_calling_convention()) {
 440       if (C->print_intrinsics()) {
 441         tty->print_cr("  ** no vector calling conventions supported");
 442       }
 443       return false;
 444     }
 445     if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
 446       if (C->print_intrinsics()) {
 447         tty->print_cr("  ** vector size (vlen=%d, etype=%s) is not supported",
 448                       num_elem, type2name(elem_bt));
 449       }
 450       return false;
 451     }
 452   }
 453 
 454   // When using mask, mask use type needs to be VecMaskUseLoad.
 455   VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
 456                                       : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
 457   if ((sopc != 0) && !arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
 458     if (C->print_intrinsics()) {
 459       tty->print_cr("  ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
 460                     n, sopc, num_elem, type2name(elem_bt),
 461                     is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
 462     }
 463     return false; // not supported
 464   }
 465 
 466   // Return true if current platform has implemented the masked operation with predicate feature.
 467   bool use_predicate = is_masked_op && sopc != 0 && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
 468   if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
 469     if (C->print_intrinsics()) {
 470       tty->print_cr("  ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
 471                     n, sopc, num_elem, type2name(elem_bt));
 472     }
 473     return false;
 474   }
 475 
 476   Node* opd1 = NULL; Node* opd2 = NULL; Node* opd3 = NULL;
 477   switch (n) {
 478     case 3: {
 479       opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
 480       if (opd3 == NULL) {
 481         if (C->print_intrinsics()) {
 482           tty->print_cr("  ** unbox failed v3=%s",
 483                         NodeClassNames[argument(7)->Opcode()]);
 484         }
 485         return false;
 486       }
 487       // fall-through
 488     }
 489     case 2: {
 490       opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
 491       if (opd2 == NULL) {
 492         if (C->print_intrinsics()) {
 493           tty->print_cr("  ** unbox failed v2=%s",
 494                         NodeClassNames[argument(6)->Opcode()]);
 495         }
 496         return false;
 497       }
 498       // fall-through
 499     }
 500     case 1: {
 501       opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
 502       if (opd1 == NULL) {
 503         if (C->print_intrinsics()) {
 504           tty->print_cr("  ** unbox failed v1=%s",
 505                         NodeClassNames[argument(5)->Opcode()]);
 506         }
 507         return false;
 508       }
 509       break;
 510     }
 511     default: fatal("unsupported arity: %d", n);
 512   }
 513 
 514   Node* mask = NULL;
 515   if (is_masked_op) {
 516     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
 517     assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
 518     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
 519     mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
 520     if (mask == NULL) {
 521       if (C->print_intrinsics()) {
 522         tty->print_cr("  ** unbox failed mask=%s",
 523                       NodeClassNames[argument(n + 5)->Opcode()]);
 524       }
 525       return false;
 526     }
 527   }
 528 
 529   Node* operation = NULL;
 530   if (opc == Op_CallLeafVector) {
 531     assert(UseVectorStubs, "sanity");
 532     operation = gen_call_to_svml(opr->get_con(), elem_bt, num_elem, opd1, opd2);
 533     if (operation == NULL) {
 534       if (C->print_intrinsics()) {
 535         tty->print_cr("  ** svml call failed for %s_%s_%d",
 536                          (elem_bt == T_FLOAT)?"float":"double",
 537                          VectorSupport::svmlname[opr->get_con() - VectorSupport::VECTOR_OP_SVML_START],
 538                          num_elem * type2aelembytes(elem_bt));
 539       }
 540       return false;
 541      }
 542   } else {
 543     const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
 544     switch (n) {
 545       case 1:
 546       case 2: {
 547         operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc));
 548         break;
 549       }
 550       case 3: {
 551         operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
 552         break;
 553       }
 554       default: fatal("unsupported arity: %d", n);
 555     }
 556   }
 557 
 558   if (is_masked_op && mask != NULL) {
 559     if (use_predicate) {
 560       operation->add_req(mask);
 561       operation->add_flag(Node::Flag_is_predicated_vector);
 562     } else {
 563       operation = gvn().transform(operation);

 564       operation = new VectorBlendNode(opd1, operation, mask);
 565     }
 566   }
 567   operation = gvn().transform(operation);
 568 
 569   // Wrap it up in VectorBox to keep object type information.
 570   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
 571   set_result(vbox);
 572   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 573   return true;
 574 }
 575 
 576 // <Sh extends VectorShuffle<E>,  E>
 577 //  Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
 578 //                  int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
 579 bool LibraryCallKit::inline_vector_shuffle_iota() {
 580   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
 581   const TypeInt*     vlen          = gvn().type(argument(3))->isa_int();
 582   const TypeInt*     start_val     = gvn().type(argument(4))->isa_int();
 583   const TypeInt*     step_val      = gvn().type(argument(5))->isa_int();
 584   const TypeInt*     wrap          = gvn().type(argument(6))->isa_int();
 585 
 586   Node* start = argument(4);
 587   Node* step  = argument(5);
 588 
 589   if (shuffle_klass == NULL || vlen == NULL || start_val == NULL || step_val == NULL || wrap == NULL) {
 590     return false; // dead code
 591   }
 592   if (!vlen->is_con() || !is_power_of_2(vlen->get_con()) ||
 593       shuffle_klass->const_oop() == NULL || !wrap->is_con()) {
 594     return false; // not enough info for intrinsification
 595   }
 596   if (!is_klass_initialized(shuffle_klass)) {
 597     if (C->print_intrinsics()) {
 598       tty->print_cr("  ** klass argument not initialized");
 599     }
 600     return false;
 601   }
 602 
 603   int do_wrap = wrap->get_con();
 604   int num_elem = vlen->get_con();
 605   BasicType elem_bt = T_BYTE;
 606 
 607   if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed)) {
 608     return false;
 609   }
 610   if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed)) {
 611     return false;
 612   }
 613   if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) {
 614     return false;
 615   }
 616   if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
 617     return false;
 618   }
 619   if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
 620     return false;
 621   }
 622   if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
 623     return false;
 624   }
 625 
 626   const Type * type_bt = Type::get_const_basic_type(elem_bt);
 627   const TypeVect * vt  = TypeVect::make(type_bt, num_elem);
 628 
 629   Node* res =  gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
 630 
 631   if(!step_val->is_con() || !is_power_of_2(step_val->get_con())) {
 632     Node* bcast_step     = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt));
 633     res = gvn().transform(VectorNode::make(Op_MulI, res, bcast_step, num_elem, elem_bt));
 634   } else if (step_val->get_con() > 1) {
 635     Node* cnt = gvn().makecon(TypeInt::make(log2i_exact(step_val->get_con())));
 636     Node* shift_cnt = vector_shift_count(cnt, Op_LShiftI, elem_bt, num_elem);
 637     res = gvn().transform(VectorNode::make(Op_LShiftVB, res, shift_cnt, vt));
 638   }
 639 
 640   if (!start_val->is_con() || start_val->get_con() != 0) {
 641     Node* bcast_start    = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt));
 642     res = gvn().transform(VectorNode::make(Op_AddI, res, bcast_start, num_elem, elem_bt));
 643   }
 644 
 645   Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1));
 646   Node * bcast_mod  = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt));
 647   if(do_wrap)  {
 648     // Wrap the indices greater than lane count.
 649     res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
 650   } else {
 651     ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::ge));
 652     Node * lane_cnt  = gvn().makecon(TypeInt::make(num_elem));
 653     Node * bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt));
 654     const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
 655     Node* mask = gvn().transform(new VectorMaskCmpNode(BoolTest::ge, bcast_lane_cnt, res, pred_node, vmask_type));
 656 
 657     // Make the indices greater than lane count as -ve values. This matches the java side implementation.
 658     res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
 659     Node * biased_val = gvn().transform(VectorNode::make(Op_SubI, res, bcast_lane_cnt, num_elem, elem_bt));
 660     res = gvn().transform(new VectorBlendNode(biased_val, res, mask));
 661   }
 662 
 663   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 664   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 665 
 666   // Wrap it up in VectorBox to keep object type information.
 667   res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
 668   set_result(res);
 669   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 670   return true;
 671 }
 672 
 673 // <E, M>
 674 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
 675 //                          int length, M m, VectorMaskOp<M> defaultImpl)
 676 bool LibraryCallKit::inline_vector_mask_operation() {
 677   const TypeInt*     oper       = gvn().type(argument(0))->isa_int();
 678   const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
 679   const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
 680   const TypeInt*     vlen       = gvn().type(argument(3))->isa_int();
 681   Node*              mask       = argument(4);
 682 
 683   if (mask_klass == NULL || elem_klass == NULL || mask->is_top() || vlen == NULL) {
 684     return false; // dead code
 685   }
 686 
 687   if (!is_klass_initialized(mask_klass)) {
 688     if (C->print_intrinsics()) {
 689       tty->print_cr("  ** klass argument not initialized");
 690     }
 691     return false;
 692   }
 693 
 694   int num_elem = vlen->get_con();
 695   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 696   BasicType elem_bt = elem_type->basic_type();
 697 
 698   if (!arch_supports_vector(Op_LoadVector, num_elem, T_BOOLEAN, VecMaskNotUsed)) {
 699     if (C->print_intrinsics()) {
 700       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 701                     Op_LoadVector, num_elem, type2name(T_BOOLEAN));
 702     }
 703     return false; // not supported
 704   }
 705 
 706   int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
 707   if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskNotUsed)) {
 708     if (C->print_intrinsics()) {
 709       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 710                     mopc, num_elem, type2name(elem_bt));
 711     }
 712     return false; // not supported
 713   }
 714 
 715   const Type* elem_ty = Type::get_const_basic_type(elem_bt);
 716   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
 717   const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
 718   Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
 719   if (mask_vec == NULL) {
 720     if (C->print_intrinsics()) {
 721         tty->print_cr("  ** unbox failed mask=%s",
 722                       NodeClassNames[argument(4)->Opcode()]);
 723     }
 724     return false;
 725   }
 726 
 727   if (mask_vec->bottom_type()->isa_vectmask() == NULL) {
 728     mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
 729   }
 730   const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
 731   Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
 732   if (mopc != Op_VectorMaskToLong) {
 733     maskoper = ConvI2L(maskoper);
 734   }
 735   set_result(maskoper);
 736 
 737   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 738   return true;
 739 }
 740 
 741 // public static
 742 // <V,
 743 //  Sh extends VectorShuffle<E>,
 744 //  E>
 745 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
 746 //                   Class<? extends Sh> shuffleClass, Sh s, int length,
 747 //                   ShuffleToVectorOperation<V, Sh, E> defaultImpl)
 748 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
 749   const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
 750   const TypeInstPtr* elem_klass    = gvn().type(argument(1))->isa_instptr();
 751   const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
 752   Node*              shuffle       = argument(3);
 753   const TypeInt*     vlen          = gvn().type(argument(4))->isa_int();
 754 
 755   if (vector_klass == NULL || elem_klass == NULL || shuffle_klass == NULL || shuffle->is_top() || vlen == NULL) {
 756     return false; // dead code
 757   }
 758   if (!vlen->is_con() || vector_klass->const_oop() == NULL || shuffle_klass->const_oop() == NULL) {
 759     return false; // not enough info for intrinsification
 760   }
 761   if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
 762     if (C->print_intrinsics()) {
 763       tty->print_cr("  ** klass argument not initialized");
 764     }
 765     return false;
 766   }
 767 
 768   int num_elem = vlen->get_con();
 769   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 770   BasicType elem_bt = elem_type->basic_type();
 771 
 772   if (num_elem < 4) {
 773     return false;
 774   }
 775 
 776   int cast_vopc = VectorCastNode::opcode(T_BYTE); // from shuffle of type T_BYTE
 777   // Make sure that cast is implemented to particular type/size combination.
 778   if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
 779     if (C->print_intrinsics()) {
 780       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 781         cast_vopc, num_elem, type2name(elem_bt));
 782     }
 783     return false;
 784   }
 785 
 786   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 787   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 788 
 789   // Unbox shuffle with true flag to indicate its load shuffle to vector
 790   // shuffle is a byte array
 791   Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, T_BYTE, num_elem, true);
 792 
 793   // cast byte to target element type
 794   shuffle_vec = gvn().transform(VectorCastNode::make(cast_vopc, shuffle_vec, elem_bt, num_elem));
 795 
 796   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 797   const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 798 
 799   // Box vector
 800   Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem);
 801   set_result(res);
 802   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 803   return true;
 804 }
 805 
 806 // public static
 807 // <M,
 808 //  S extends VectorSpecies<E>,
 809 //  E>
 810 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
 811 //                    long bits, int mode, S s,
 812 //                    BroadcastOperation<M, E, S> defaultImpl)
 813 bool LibraryCallKit::inline_vector_frombits_coerced() {
 814   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 815   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 816   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 817   const TypeLong*    bits_type    = gvn().type(argument(3))->isa_long();
 818   // Mode argument determines the mode of operation it can take following values:-
 819   // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
 820   // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
 821   const TypeInt*     mode         = gvn().type(argument(5))->isa_int();
 822 
 823   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || mode == NULL ||
 824       bits_type == NULL || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL ||
 825       !vlen->is_con() || !mode->is_con()) {
 826     if (C->print_intrinsics()) {
 827       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
 828                     NodeClassNames[argument(0)->Opcode()],
 829                     NodeClassNames[argument(1)->Opcode()],
 830                     NodeClassNames[argument(2)->Opcode()],
 831                     NodeClassNames[argument(5)->Opcode()]);
 832     }
 833     return false; // not enough info for intrinsification
 834   }
 835 
 836   if (!is_klass_initialized(vector_klass)) {
 837     if (C->print_intrinsics()) {
 838       tty->print_cr("  ** klass argument not initialized");
 839     }
 840     return false;
 841   }
 842   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 843   if (!elem_type->is_primitive_type()) {
 844     if (C->print_intrinsics()) {
 845       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 846     }
 847     return false; // should be primitive type
 848   }
 849   BasicType elem_bt = elem_type->basic_type();
 850   int num_elem = vlen->get_con();
 851   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 852   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 853 
 854   bool is_mask = is_vector_mask(vbox_klass);
 855   int  bcast_mode = mode->get_con();
 856   VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
 857   int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : VectorNode::replicate_opcode(elem_bt);
 858 
 859   if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
 860     if (C->print_intrinsics()) {
 861       tty->print_cr("  ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
 862                     num_elem, type2name(elem_bt),
 863                     is_mask ? 1 : 0,
 864                     bcast_mode);
 865     }
 866     return false; // not supported
 867   }
 868 
 869   Node* broadcast = NULL;
 870   Node* bits = argument(3);
 871   Node* elem = bits;
 872 
 873   if (opc == Op_VectorLongToMask) {
 874     const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
 875     if (vt->isa_vectmask()) {
 876       broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
 877     } else {
 878       const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
 879       broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
 880       broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
 881     }
 882   } else {
 883     switch (elem_bt) {
 884       case T_BOOLEAN: // fall-through
 885       case T_BYTE:    // fall-through
 886       case T_SHORT:   // fall-through
 887       case T_CHAR:    // fall-through
 888       case T_INT: {
 889         elem = gvn().transform(new ConvL2INode(bits));
 890         break;
 891       }
 892       case T_DOUBLE: {
 893         elem = gvn().transform(new MoveL2DNode(bits));
 894         break;
 895       }
 896       case T_FLOAT: {
 897         bits = gvn().transform(new ConvL2INode(bits));
 898         elem = gvn().transform(new MoveI2FNode(bits));
 899         break;
 900       }
 901       case T_LONG: {
 902         // no conversion needed
 903         break;
 904       }
 905       default: fatal("%s", type2name(elem_bt));
 906     }
 907     broadcast = VectorNode::scalar2vector(elem, num_elem, Type::get_const_basic_type(elem_bt), is_mask);
 908     broadcast = gvn().transform(broadcast);
 909   }
 910 
 911   Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
 912   set_result(box);
 913   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 914   return true;
 915 }
 916 
 917 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type) {
 918   assert(arr_type != NULL, "unexpected");
 919   BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
 920   if (elem_bt == arr_elem_bt) {
 921     return true;
 922   } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
 923     // Load/store of short vector from/to char[] is supported
 924     return true;
 925   } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
 926     // Load/store of byte vector from/to boolean[] is supported
 927     return true;
 928   } else {
 929     return false;
 930   }
 931 }
 932 
 933 // public static
 934 // <C,
 935 //  VM,
 936 //  E,
 937 //  S extends VectorSpecies<E>>
 938 // VM load(Class<? extends VM> vmClass, Class<E> elementType, int length,
 939 //         Object base, long offset,    // Unsafe addressing
 940 //         C container, int index, S s,     // Arguments for default implementation
 941 //         LoadOperation<C, VM, E, S> defaultImpl)
 942 //
 943 // public static
 944 // <C,
 945 //  V extends Vector<?>>
 946 // void store(Class<?> vectorClass, Class<?> elementType, int length,
 947 //            Object base, long offset,    // Unsafe addressing
 948 //            V v,
 949 //            C container, int index,      // Arguments for default implementation
 950 //            StoreVectorOperation<C, V> defaultImpl)
 951 
 952 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
 953   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 954   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 955   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 956 
 957   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
 958       vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 959     if (C->print_intrinsics()) {
 960       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
 961                     NodeClassNames[argument(0)->Opcode()],
 962                     NodeClassNames[argument(1)->Opcode()],
 963                     NodeClassNames[argument(2)->Opcode()]);
 964     }
 965     return false; // not enough info for intrinsification
 966   }
 967   if (!is_klass_initialized(vector_klass)) {
 968     if (C->print_intrinsics()) {
 969       tty->print_cr("  ** klass argument not initialized");
 970     }
 971     return false;
 972   }
 973 
 974   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 975   if (!elem_type->is_primitive_type()) {
 976     if (C->print_intrinsics()) {
 977       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 978     }
 979     return false; // should be primitive type
 980   }
 981   BasicType elem_bt = elem_type->basic_type();
 982   int num_elem = vlen->get_con();
 983 
 984   // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
 985   if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
 986     if (C->print_intrinsics()) {
 987       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
 988                     is_store, is_store ? "store" : "load",
 989                     num_elem, type2name(elem_bt));
 990     }
 991     return false; // not supported
 992   }
 993 
 994   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 995   bool is_mask = is_vector_mask(vbox_klass);
 996 
 997   Node* base = argument(3);
 998   Node* offset = ConvL2X(argument(4));
 999 
1000   // Save state and restore on bailout
1001   uint old_sp = sp();
1002   SafePointNode* old_map = clone_map();
1003 
1004   Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
1005 
1006   // The memory barrier checks are based on ones for unsafe access.
1007   // This is not 1-1 implementation.
1008   const Type *const base_type = gvn().type(base);
1009 
1010   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1011   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1012 
1013   const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
1014   const bool in_heap   = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
1015 
1016   const bool is_mixed_access = !in_heap && !in_native;
1017 
1018   const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == NULL);
1019 
1020   const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
1021 
1022   // Now handle special case where load/store happens from/to byte array but element type is not byte.
1023   bool using_byte_array = arr_type != NULL && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE;
1024   // Handle loading masks.
1025   // If there is no consistency between array and vector element types, it must be special byte array case or loading masks
1026   if (arr_type != NULL && !using_byte_array && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type)) {
1027     if (C->print_intrinsics()) {
1028       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1029                     is_store, is_store ? "store" : "load",
1030                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1031     }
1032     set_map(old_map);
1033     set_sp(old_sp);
1034     return false;
1035   }
1036   // Since we are using byte array, we need to double check that the byte operations are supported by backend.
1037   if (using_byte_array) {
1038     int byte_num_elem = num_elem * type2aelembytes(elem_bt);
1039     if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, byte_num_elem, T_BYTE, VecMaskNotUsed)
1040         || !arch_supports_vector(Op_VectorReinterpret, byte_num_elem, T_BYTE, VecMaskNotUsed)) {
1041       if (C->print_intrinsics()) {
1042         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
1043                       is_store, is_store ? "store" : "load",
1044                       byte_num_elem, type2name(elem_bt));
1045       }
1046       set_map(old_map);
1047       set_sp(old_sp);
1048       return false; // not supported
1049     }
1050   }
1051   if (is_mask) {
1052     if (!arch_supports_vector(Op_LoadVector, num_elem, T_BOOLEAN, VecMaskNotUsed)) {
1053       if (C->print_intrinsics()) {
1054         tty->print_cr("  ** not supported: arity=%d op=%s/mask vlen=%d etype=bit ismask=no",
1055                       is_store, is_store ? "store" : "load",
1056                       num_elem);
1057       }
1058       set_map(old_map);
1059       set_sp(old_sp);
1060       return false; // not supported
1061     }
1062     if (!is_store) {
1063       if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1064         set_map(old_map);
1065         set_sp(old_sp);
1066         return false; // not supported
1067       }
1068     } else {
1069       if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
1070         set_map(old_map);
1071         set_sp(old_sp);
1072         return false; // not supported
1073       }
1074     }
1075   }
1076 
1077   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1078 
1079   if (needs_cpu_membar) {
1080     insert_mem_bar(Op_MemBarCPUOrder);
1081   }
1082 
1083   if (is_store) {
1084     Node* val = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1085     if (val == NULL) {
1086       set_map(old_map);
1087       set_sp(old_sp);
1088       return false; // operand unboxing failed
1089     }
1090     set_all_memory(reset_memory());
1091 
1092     // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
1093     int store_num_elem = num_elem;
1094     if (using_byte_array) {
1095       store_num_elem = num_elem * type2aelembytes(elem_bt);
1096       const TypeVect* to_vect_type = TypeVect::make(T_BYTE, store_num_elem);
1097       val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1098     }
1099 


1100     Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
1101     set_memory(vstore, addr_type);
1102   } else {
1103     // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
1104     Node* vload = NULL;
1105     if (using_byte_array) {
1106       int load_num_elem = num_elem * type2aelembytes(elem_bt);
1107       vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, load_num_elem, T_BYTE));
1108       const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1109       vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1110     } else {
1111       // Special handle for masks
1112       if (is_mask) {
1113         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
1114         vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
1115       } else {
1116         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
1117       }
1118     }
1119     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1120     set_result(box);
1121   }
1122 
1123   old_map->destruct(&_gvn);
1124 
1125   if (needs_cpu_membar) {
1126     insert_mem_bar(Op_MemBarCPUOrder);
1127   }
1128 
1129   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1130   return true;
1131 }
1132 
1133 // public static
1134 // <C,
1135 //  V extends Vector<?>,
1136 //  E,
1137 //  S extends VectorSpecies<E>,
1138 //  M extends VectorMask<E>>
1139 // V loadMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1140 //              int length, Object base, long offset, M m,
1141 //              C container, int index, S s,  // Arguments for default implementation
1142 //              LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1143 //
1144 // public static
1145 // <C,
1146 //  V extends Vector<E>,
1147 //  M extends VectorMask<E>,
1148 //  E>
1149 // void storeMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1150 //                  int length, Object base, long offset,
1151 //                  V v, M m,
1152 //                  C container, int index,  // Arguments for default implementation
1153 //                  StoreVectorMaskedOperation<C, V, M, E> defaultImpl) {
1154 //
1155 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1156   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1157   const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1158   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1159   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1160 
1161   if (vector_klass == NULL || mask_klass == NULL || elem_klass == NULL || vlen == NULL ||
1162       vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
1163       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1164     if (C->print_intrinsics()) {
1165       tty->print_cr("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1166                     NodeClassNames[argument(0)->Opcode()],
1167                     NodeClassNames[argument(1)->Opcode()],
1168                     NodeClassNames[argument(2)->Opcode()],
1169                     NodeClassNames[argument(3)->Opcode()]);
1170     }
1171     return false; // not enough info for intrinsification
1172   }
1173   if (!is_klass_initialized(vector_klass)) {
1174     if (C->print_intrinsics()) {
1175       tty->print_cr("  ** klass argument not initialized");
1176     }
1177     return false;
1178   }
1179 
1180   if (!is_klass_initialized(mask_klass)) {
1181     if (C->print_intrinsics()) {
1182       tty->print_cr("  ** mask klass argument not initialized");
1183     }
1184     return false;
1185   }
1186 
1187   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1188   if (!elem_type->is_primitive_type()) {
1189     if (C->print_intrinsics()) {
1190       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1191     }
1192     return false; // should be primitive type
1193   }
1194 
1195   BasicType elem_bt = elem_type->basic_type();
1196   int num_elem = vlen->get_con();
1197 
1198   Node* base = argument(4);
1199   Node* offset = ConvL2X(argument(5));
1200 
1201   // Save state and restore on bailout
1202   uint old_sp = sp();
1203   SafePointNode* old_map = clone_map();
1204 
1205   Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1206   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1207   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1208 
1209   // Now handle special case where load/store happens from/to byte array but element type is not byte.
1210   bool using_byte_array = arr_type != NULL && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE;
1211   // If there is no consistency between array and vector element types, it must be special byte array case
1212   if (arr_type != NULL && !using_byte_array && !elem_consistent_with_arr(elem_bt, arr_type)) {
1213     if (C->print_intrinsics()) {
1214       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
1215                     is_store, is_store ? "storeMasked" : "loadMasked",
1216                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1217     }
1218     set_map(old_map);
1219     set_sp(old_sp);
1220     return false;
1221   }
1222 
1223   int mem_num_elem = using_byte_array ? num_elem * type2aelembytes(elem_bt) : num_elem;
1224   BasicType mem_elem_bt = using_byte_array ? T_BYTE : elem_bt;
1225   bool use_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
1226                                             mem_num_elem, mem_elem_bt,
1227                                             (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred));
1228   // Masked vector store operation needs the architecture predicate feature. We need to check
1229   // whether the predicated vector operation is supported by backend.
1230   if (is_store && !use_predicate) {
1231     if (C->print_intrinsics()) {
1232       tty->print_cr("  ** not supported: op=storeMasked vlen=%d etype=%s using_byte_array=%d",
1233                     num_elem, type2name(elem_bt), using_byte_array ? 1 : 0);
1234     }
1235     set_map(old_map);
1236     set_sp(old_sp);
1237     return false;
1238   }
1239 
1240   // This only happens for masked vector load. If predicate is not supported, then check whether
1241   // the normal vector load and blend operations are supported by backend.
1242   if (!use_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1243       !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1244     if (C->print_intrinsics()) {
1245       tty->print_cr("  ** not supported: op=loadMasked vlen=%d etype=%s using_byte_array=%d",
1246                     num_elem, type2name(elem_bt), using_byte_array ? 1 : 0);
1247     }
1248     set_map(old_map);
1249     set_sp(old_sp);
1250     return false;
1251   }
1252 
1253   // Since we are using byte array, we need to double check that the vector reinterpret operation
1254   // with byte type is supported by backend.
1255   if (using_byte_array) {
1256     if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1257       if (C->print_intrinsics()) {
1258         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s using_byte_array=1",
1259                       is_store, is_store ? "storeMasked" : "loadMasked",
1260                       num_elem, type2name(elem_bt));
1261       }
1262       set_map(old_map);
1263       set_sp(old_sp);
1264       return false;
1265     }
1266   }
1267 
1268   // Since it needs to unbox the mask, we need to double check that the related load operations
1269   // for mask are supported by backend.
1270   if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1271     if (C->print_intrinsics()) {
1272       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s",
1273                       is_store, is_store ? "storeMasked" : "loadMasked",
1274                       num_elem, type2name(elem_bt));
1275     }
1276     set_map(old_map);
1277     set_sp(old_sp);
1278     return false;
1279   }
1280 
1281   // Can base be NULL? Otherwise, always on-heap access.
1282   bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1283   if (can_access_non_heap) {
1284     insert_mem_bar(Op_MemBarCPUOrder);
1285   }
1286 
1287   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1288   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1289   assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1290   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1291   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1292 
1293   Node* mask = unbox_vector(is_store ? argument(8) : argument(7), mbox_type, elem_bt, num_elem);
1294   if (mask == NULL) {
1295     if (C->print_intrinsics()) {
1296       tty->print_cr("  ** unbox failed mask=%s",
1297                     is_store ? NodeClassNames[argument(8)->Opcode()]
1298                              : NodeClassNames[argument(7)->Opcode()]);
1299     }
1300     set_map(old_map);
1301     set_sp(old_sp);
1302     return false;
1303   }
1304 
1305   if (is_store) {
1306     Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
1307     if (val == NULL) {
1308       if (C->print_intrinsics()) {
1309         tty->print_cr("  ** unbox failed vector=%s",
1310                       NodeClassNames[argument(7)->Opcode()]);
1311       }
1312       set_map(old_map);
1313       set_sp(old_sp);
1314       return false; // operand unboxing failed
1315     }
1316     set_all_memory(reset_memory());
1317 
1318     if (using_byte_array) {
1319       // Reinterpret the incoming vector to byte vector.
1320       const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1321       val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1322       // Reinterpret the vector mask to byte type.
1323       const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1324       const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1325       mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1326     }
1327     Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1328     set_memory(vstore, addr_type);
1329   } else {
1330     Node* vload = NULL;
1331 
1332     if (using_byte_array) {
1333       // Reinterpret the vector mask to byte type.
1334       const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1335       const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1336       mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1337     }
1338 
1339     if (use_predicate) {
1340       // Generate masked load vector node if predicate feature is supported.
1341       const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1342       vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1343     } else {
1344       // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1345       Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1346       zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, Type::get_const_basic_type(mem_elem_bt)));
1347       vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1348       vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1349     }
1350 
1351     if (using_byte_array) {
1352       const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1353       vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1354     }
1355 
1356     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1357     set_result(box);
1358   }
1359 
1360   old_map->destruct(&_gvn);
1361 
1362   if (can_access_non_heap) {
1363     insert_mem_bar(Op_MemBarCPUOrder);
1364   }
1365 
1366   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1367   return true;
1368 }
1369 
1370 // <C,
1371 //  V extends Vector<?>,
1372 //  W extends Vector<Integer>,
1373 //  S extends VectorSpecies<E>,
1374 //  M extends VectorMask<E>,
1375 //  E>
1376 // V loadWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int length,
1377 //               Class<? extends Vector<Integer>> vectorIndexClass,
1378 //               Object base, long offset, // Unsafe addressing
1379 //               W index_vector, M m,
1380 //               C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
1381 //               LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1382 //
1383 //  <C,
1384 //   V extends Vector<E>,
1385 //   W extends Vector<Integer>,
1386 //   M extends VectorMask<E>,
1387 //   E>
1388 //  void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1389 //                    int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset,    // Unsafe addressing
1390 //                    W index_vector, V v, M m,
1391 //                    C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1392 //                    StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1393 //
1394 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1395   const TypeInstPtr* vector_klass     = gvn().type(argument(0))->isa_instptr();
1396   const TypeInstPtr* mask_klass       = gvn().type(argument(1))->isa_instptr();
1397   const TypeInstPtr* elem_klass       = gvn().type(argument(2))->isa_instptr();
1398   const TypeInt*     vlen             = gvn().type(argument(3))->isa_int();
1399   const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1400 
1401   if (vector_klass == NULL || elem_klass == NULL || vector_idx_klass == NULL || vlen == NULL ||
1402       vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || vector_idx_klass->const_oop() == NULL || !vlen->is_con()) {
1403     if (C->print_intrinsics()) {
1404       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1405                     NodeClassNames[argument(0)->Opcode()],
1406                     NodeClassNames[argument(2)->Opcode()],
1407                     NodeClassNames[argument(3)->Opcode()],
1408                     NodeClassNames[argument(4)->Opcode()]);
1409     }
1410     return false; // not enough info for intrinsification
1411   }
1412 
1413   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1414     if (C->print_intrinsics()) {
1415       tty->print_cr("  ** klass argument not initialized");
1416     }
1417     return false;
1418   }
1419 
1420   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1421   if (!elem_type->is_primitive_type()) {
1422     if (C->print_intrinsics()) {
1423       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1424     }
1425     return false; // should be primitive type
1426   }
1427 
1428   BasicType elem_bt = elem_type->basic_type();
1429   int num_elem = vlen->get_con();
1430 
1431   const Type* vmask_type = gvn().type(is_scatter ? argument(10) : argument(9));
1432   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1433   if (is_masked_op) {
1434     if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
1435       if (C->print_intrinsics()) {
1436         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1437       }
1438       return false; // not enough info for intrinsification
1439     }
1440 
1441     if (!is_klass_initialized(mask_klass)) {
1442       if (C->print_intrinsics()) {
1443         tty->print_cr("  ** mask klass argument not initialized");
1444       }
1445       return false;
1446     }
1447 
1448     if (vmask_type->maybe_null()) {
1449       if (C->print_intrinsics()) {
1450         tty->print_cr("  ** null mask values are not allowed for masked op");
1451       }
1452       return false;
1453     }
1454 
1455     // Check whether the predicated gather/scatter node is supported by architecture.
1456     if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt,
1457                               (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred))) {
1458       if (C->print_intrinsics()) {
1459         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1460                       is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1461                       num_elem, type2name(elem_bt));
1462       }
1463       return false; // not supported
1464     }
1465   } else {
1466     // Check whether the normal gather/scatter node is supported for non-masked operation.
1467     if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1468       if (C->print_intrinsics()) {
1469         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1470                       is_scatter, is_scatter ? "scatter" : "gather",
1471                       num_elem, type2name(elem_bt));
1472       }
1473       return false; // not supported
1474     }
1475   }
1476 
1477   // Check that the vector holding indices is supported by architecture
1478   if (!arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
1479       if (C->print_intrinsics()) {
1480         tty->print_cr("  ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1481                       is_scatter, is_scatter ? "scatter" : "gather",
1482                       num_elem, is_masked_op ? 1 : 0);
1483       }
1484       return false; // not supported
1485   }
1486 
1487   Node* base = argument(5);
1488   Node* offset = ConvL2X(argument(6));
1489 
1490   // Save state and restore on bailout
1491   uint old_sp = sp();
1492   SafePointNode* old_map = clone_map();
1493 
1494   Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1495 
1496   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1497   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1498 
1499   // The array must be consistent with vector type
1500   if (arr_type == NULL || (arr_type != NULL && !elem_consistent_with_arr(elem_bt, arr_type))) {
1501     if (C->print_intrinsics()) {
1502       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1503                     is_scatter, is_scatter ? "scatter" : "gather",
1504                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1505     }
1506     set_map(old_map);
1507     set_sp(old_sp);
1508     return false;
1509   }
1510 
1511   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1512   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1513   ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1514   if (vbox_idx_klass == NULL) {
1515     set_map(old_map);
1516     set_sp(old_sp);
1517     return false;
1518   }
1519 
1520   const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1521   Node* index_vect = unbox_vector(argument(8), vbox_idx_type, T_INT, num_elem);
1522   if (index_vect == NULL) {
1523     set_map(old_map);
1524     set_sp(old_sp);
1525     return false;
1526   }
1527 
1528   Node* mask = NULL;
1529   if (is_masked_op) {
1530     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1531     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1532     mask = unbox_vector(is_scatter ? argument(10) : argument(9), mbox_type, elem_bt, num_elem);
1533     if (mask == NULL) {
1534       if (C->print_intrinsics()) {
1535         tty->print_cr("  ** unbox failed mask=%s",
1536                     is_scatter ? NodeClassNames[argument(10)->Opcode()]
1537                                : NodeClassNames[argument(9)->Opcode()]);
1538       }
1539       set_map(old_map);
1540       set_sp(old_sp);
1541       return false;
1542     }
1543   }
1544 
1545   const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1546   if (is_scatter) {
1547     Node* val = unbox_vector(argument(9), vbox_type, elem_bt, num_elem);
1548     if (val == NULL) {
1549       set_map(old_map);
1550       set_sp(old_sp);
1551       return false; // operand unboxing failed
1552     }
1553     set_all_memory(reset_memory());
1554 
1555     Node* vstore = NULL;
1556     if (mask != NULL) {
1557       vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, index_vect, mask));
1558     } else {
1559       vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1560     }
1561     set_memory(vstore, addr_type);
1562   } else {
1563     Node* vload = NULL;
1564     if (mask != NULL) {
1565       vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_vect, mask));
1566     } else {
1567       vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1568     }
1569     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1570     set_result(box);
1571   }
1572 
1573   old_map->destruct(&_gvn);
1574 
1575   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1576   return true;
1577 }
1578 
1579 // public static
1580 // <V extends Vector<E>,
1581 //  M extends VectorMask<E>,
1582 //  E>
1583 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1584 //                       Class<E> elementType, int length, V v, M m,
1585 //                       ReductionOperation<V, M> defaultImpl)
1586 bool LibraryCallKit::inline_vector_reduction() {
1587   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
1588   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1589   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
1590   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
1591   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
1592 
1593   if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
1594       !opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1595     if (C->print_intrinsics()) {
1596       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1597                     NodeClassNames[argument(0)->Opcode()],
1598                     NodeClassNames[argument(1)->Opcode()],
1599                     NodeClassNames[argument(3)->Opcode()],
1600                     NodeClassNames[argument(4)->Opcode()]);
1601     }
1602     return false; // not enough info for intrinsification
1603   }
1604   if (!is_klass_initialized(vector_klass)) {
1605     if (C->print_intrinsics()) {
1606       tty->print_cr("  ** klass argument not initialized");
1607     }
1608     return false;
1609   }
1610   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1611   if (!elem_type->is_primitive_type()) {
1612     if (C->print_intrinsics()) {
1613       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1614     }
1615     return false; // should be primitive type
1616   }
1617 
1618   const Type* vmask_type = gvn().type(argument(6));
1619   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1620   if (is_masked_op) {
1621     if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
1622       if (C->print_intrinsics()) {
1623         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1624       }
1625       return false; // not enough info for intrinsification
1626     }
1627 
1628     if (!is_klass_initialized(mask_klass)) {
1629       if (C->print_intrinsics()) {
1630         tty->print_cr("  ** mask klass argument not initialized");
1631       }
1632       return false;
1633     }
1634 
1635     if (vmask_type->maybe_null()) {
1636       if (C->print_intrinsics()) {
1637         tty->print_cr("  ** null mask values are not allowed for masked op");
1638       }
1639       return false;
1640     }
1641   }
1642 
1643   BasicType elem_bt = elem_type->basic_type();
1644   int num_elem = vlen->get_con();
1645   int opc  = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1646   int sopc = ReductionNode::opcode(opc, elem_bt);
1647 
1648   // When using mask, mask use type needs to be VecMaskUseLoad.
1649   if (!arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1650     if (C->print_intrinsics()) {
1651       tty->print_cr("  ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1652                     sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1653     }
1654     return false;
1655   }
1656 
1657   // Return true if current platform has implemented the masked operation with predicate feature.
1658   bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1659   if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1660     if (C->print_intrinsics()) {
1661       tty->print_cr("  ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1662                     sopc, num_elem, type2name(elem_bt));
1663     }
1664     return false;
1665   }
1666 
1667   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1668   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1669 
1670   Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1671   if (opd == NULL) {
1672     return false; // operand unboxing failed
1673   }
1674 
1675   Node* mask = NULL;
1676   if (is_masked_op) {
1677     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1678     assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1679     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1680     mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1681     if (mask == NULL) {
1682       if (C->print_intrinsics()) {
1683         tty->print_cr("  ** unbox failed mask=%s",
1684                       NodeClassNames[argument(6)->Opcode()]);
1685       }
1686       return false;
1687     }
1688   }
1689 
1690   Node* init = ReductionNode::make_reduction_input(gvn(), opc, elem_bt);
1691   Node* value = NULL;
1692   if (mask == NULL) {
1693     assert(!is_masked_op, "Masked op needs the mask value never null");
1694     value = ReductionNode::make(opc, NULL, init, opd, elem_bt);
1695   } else {
1696     if (use_predicate) {
1697       value = ReductionNode::make(opc, NULL, init, opd, elem_bt);
1698       value->add_req(mask);
1699       value->add_flag(Node::Flag_is_predicated_vector);
1700     } else {
1701       Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, Type::get_const_basic_type(elem_bt)));
1702       value = gvn().transform(new VectorBlendNode(reduce_identity, opd, mask));
1703       value = ReductionNode::make(opc, NULL, init, value, elem_bt);
1704     }
1705   }
1706   value = gvn().transform(value);
1707 
1708   Node* bits = NULL;
1709   switch (elem_bt) {
1710     case T_BYTE:
1711     case T_SHORT:
1712     case T_INT: {
1713       bits = gvn().transform(new ConvI2LNode(value));
1714       break;
1715     }
1716     case T_FLOAT: {
1717       value = gvn().transform(new MoveF2INode(value));
1718       bits  = gvn().transform(new ConvI2LNode(value));
1719       break;
1720     }
1721     case T_DOUBLE: {
1722       bits = gvn().transform(new MoveD2LNode(value));
1723       break;
1724     }
1725     case T_LONG: {
1726       bits = value; // no conversion needed
1727       break;
1728     }
1729     default: fatal("%s", type2name(elem_bt));
1730   }
1731   set_result(bits);
1732   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1733   return true;
1734 }
1735 
1736 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1737 //                                V v1, V v2,
1738 //                                BiFunction<V, V, Boolean> defaultImpl)
1739 //
1740 bool LibraryCallKit::inline_vector_test() {
1741   const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1742   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1743   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1744   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1745 
1746   if (cond == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
1747       !cond->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1748     if (C->print_intrinsics()) {
1749       tty->print_cr("  ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1750                     NodeClassNames[argument(0)->Opcode()],
1751                     NodeClassNames[argument(1)->Opcode()],
1752                     NodeClassNames[argument(2)->Opcode()],
1753                     NodeClassNames[argument(3)->Opcode()]);
1754     }
1755     return false; // not enough info for intrinsification
1756   }
1757   if (!is_klass_initialized(vector_klass)) {
1758     if (C->print_intrinsics()) {
1759       tty->print_cr("  ** klass argument not initialized");
1760     }
1761     return false;
1762   }
1763   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1764   if (!elem_type->is_primitive_type()) {
1765     if (C->print_intrinsics()) {
1766       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1767     }
1768     return false; // should be primitive type
1769   }
1770   BasicType elem_bt = elem_type->basic_type();
1771   int num_elem = vlen->get_con();
1772   BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1773   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1774   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1775 
1776   if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1777     if (C->print_intrinsics()) {
1778       tty->print_cr("  ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1779                     cond->get_con(), num_elem, type2name(elem_bt),
1780                     is_vector_mask(vbox_klass));
1781     }
1782     return false;
1783   }
1784 
1785   Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1786   Node* opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1787   if (opd1 == NULL || opd2 == NULL) {
1788     return false; // operand unboxing failed
1789   }
1790   Node* test = new VectorTestNode(opd1, opd2, booltest);
1791   test = gvn().transform(test);
1792 
1793   set_result(test);
1794   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1795   return true;
1796 }
1797 
1798 // public static
1799 // <V extends Vector<E>,
1800 //  M extends VectorMask<E>,
1801 //  E>
1802 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1803 //         V v1, V v2, M m,
1804 //         VectorBlendOp<V, M, E> defaultImpl)
1805 bool LibraryCallKit::inline_vector_blend() {
1806   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1807   const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1808   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1809   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1810 
1811   if (mask_klass == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL) {
1812     return false; // dead code
1813   }
1814   if (mask_klass->const_oop() == NULL || vector_klass->const_oop() == NULL ||
1815       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1816     if (C->print_intrinsics()) {
1817       tty->print_cr("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1818                     NodeClassNames[argument(0)->Opcode()],
1819                     NodeClassNames[argument(1)->Opcode()],
1820                     NodeClassNames[argument(2)->Opcode()],
1821                     NodeClassNames[argument(3)->Opcode()]);
1822     }
1823     return false; // not enough info for intrinsification
1824   }
1825   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1826     if (C->print_intrinsics()) {
1827       tty->print_cr("  ** klass argument not initialized");
1828     }
1829     return false;
1830   }
1831   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1832   if (!elem_type->is_primitive_type()) {
1833     if (C->print_intrinsics()) {
1834       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1835     }
1836     return false; // should be primitive type
1837   }
1838   BasicType elem_bt = elem_type->basic_type();
1839   BasicType mask_bt = elem_bt;
1840   int num_elem = vlen->get_con();
1841 
1842   if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1843     if (C->print_intrinsics()) {
1844       tty->print_cr("  ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1845                     num_elem, type2name(elem_bt));
1846     }
1847     return false; // not supported
1848   }
1849   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1850   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1851 
1852   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1853   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1854 
1855   Node* v1   = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1856   Node* v2   = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1857   Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1858 
1859   if (v1 == NULL || v2 == NULL || mask == NULL) {
1860     return false; // operand unboxing failed
1861   }
1862 
1863   Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1864 
1865   Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1866   set_result(box);
1867   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1868   return true;
1869 }
1870 
1871 //  public static
1872 //  <V extends Vector<E>,
1873 //   M extends VectorMask<E>,
1874 //   E>
1875 //  M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1876 //            V v1, V v2, M m,
1877 //            VectorCompareOp<V,M> defaultImpl)
1878 bool LibraryCallKit::inline_vector_compare() {
1879   const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1880   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1881   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
1882   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
1883   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
1884 
1885   if (cond == NULL || vector_klass == NULL || mask_klass == NULL || elem_klass == NULL || vlen == NULL) {
1886     return false; // dead code
1887   }
1888   if (!cond->is_con() || vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
1889       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1890     if (C->print_intrinsics()) {
1891       tty->print_cr("  ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1892                     NodeClassNames[argument(0)->Opcode()],
1893                     NodeClassNames[argument(1)->Opcode()],
1894                     NodeClassNames[argument(2)->Opcode()],
1895                     NodeClassNames[argument(3)->Opcode()],
1896                     NodeClassNames[argument(4)->Opcode()]);
1897     }
1898     return false; // not enough info for intrinsification
1899   }
1900   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1901     if (C->print_intrinsics()) {
1902       tty->print_cr("  ** klass argument not initialized");
1903     }
1904     return false;
1905   }
1906   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1907   if (!elem_type->is_primitive_type()) {
1908     if (C->print_intrinsics()) {
1909       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1910     }
1911     return false; // should be primitive type
1912   }
1913 
1914   int num_elem = vlen->get_con();
1915   BasicType elem_bt = elem_type->basic_type();
1916   BasicType mask_bt = elem_bt;
1917 
1918   if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1919     if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1920       if (C->print_intrinsics()) {
1921         tty->print_cr("  ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1922                       cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1923       }
1924       return false;
1925     }
1926   }
1927 
1928   if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1929     if (C->print_intrinsics()) {
1930       tty->print_cr("  ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1931                     cond->get_con(), num_elem, type2name(elem_bt));
1932     }
1933     return false;
1934   }
1935 
1936   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1937   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1938 
1939   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1940   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1941 
1942   Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1943   Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1944 
1945   bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1946   Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : NULL;
1947   if (is_masked_op && mask == NULL) {
1948     if (C->print_intrinsics()) {
1949       tty->print_cr("  ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1950                     cond->get_con(), num_elem, type2name(elem_bt));
1951     }
1952     return false;
1953   }
1954 
1955   bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1956   if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1957     if (C->print_intrinsics()) {
1958       tty->print_cr("  ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1959                     cond->get_con(), num_elem, type2name(elem_bt));
1960     }
1961     return false;
1962   }
1963 
1964   if (v1 == NULL || v2 == NULL) {
1965     return false; // operand unboxing failed
1966   }
1967   BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1968   ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1969 
1970   const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1971   Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1972 
1973   if (is_masked_op) {
1974     if (use_predicate) {
1975       operation->add_req(mask);
1976       operation->add_flag(Node::Flag_is_predicated_vector);
1977     } else {
1978       operation = gvn().transform(operation);
1979       operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1980     }
1981   }
1982 
1983   operation = gvn().transform(operation);
1984 
1985   Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1986   set_result(box);
1987   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1988   return true;
1989 }
1990 
1991 // public static
1992 // <V extends Vector<E>,
1993 //  Sh extends VectorShuffle<E>,
1994 //  M extends VectorMask<E>,
1995 //  E>
1996 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1997 //               V v1, Sh sh, M m,
1998 //               VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1999 bool LibraryCallKit::inline_vector_rearrange() {
2000   const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
2001   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
2002   const TypeInstPtr* mask_klass    = gvn().type(argument(2))->isa_instptr();
2003   const TypeInstPtr* elem_klass    = gvn().type(argument(3))->isa_instptr();
2004   const TypeInt*     vlen          = gvn().type(argument(4))->isa_int();
2005 
2006   if (vector_klass == NULL  || shuffle_klass == NULL ||  elem_klass == NULL || vlen == NULL) {
2007     return false; // dead code
2008   }
2009   if (shuffle_klass->const_oop() == NULL ||
2010       vector_klass->const_oop()  == NULL ||
2011       elem_klass->const_oop()    == NULL ||
2012       !vlen->is_con()) {
2013     if (C->print_intrinsics()) {
2014       tty->print_cr("  ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
2015                     NodeClassNames[argument(0)->Opcode()],
2016                     NodeClassNames[argument(1)->Opcode()],
2017                     NodeClassNames[argument(3)->Opcode()],
2018                     NodeClassNames[argument(4)->Opcode()]);
2019     }
2020     return false; // not enough info for intrinsification
2021   }
2022   if (!is_klass_initialized(vector_klass)  ||
2023       !is_klass_initialized(shuffle_klass)) {
2024     if (C->print_intrinsics()) {
2025       tty->print_cr("  ** klass argument not initialized");
2026     }
2027     return false;
2028   }
2029   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2030   if (!elem_type->is_primitive_type()) {
2031     if (C->print_intrinsics()) {
2032       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2033     }
2034     return false; // should be primitive type
2035   }
2036   BasicType elem_bt = elem_type->basic_type();
2037   BasicType shuffle_bt = elem_bt;
2038   int num_elem = vlen->get_con();
2039 
2040   if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
2041     if (C->print_intrinsics()) {
2042       tty->print_cr("  ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
2043                     num_elem, type2name(elem_bt));
2044     }
2045     return false; // not supported
2046   }
2047 
2048   bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
2049   bool use_predicate = is_masked_op;
2050   if (is_masked_op &&
2051       (mask_klass == NULL ||
2052        mask_klass->const_oop() == NULL ||
2053        !is_klass_initialized(mask_klass))) {
2054     if (C->print_intrinsics()) {
2055       tty->print_cr("  ** mask_klass argument not initialized");
2056     }
2057   }
2058   VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2059   if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
2060     use_predicate = false;
2061     if(!is_masked_op ||
2062        (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2063         !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)     ||
2064         !arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed))) {
2065       if (C->print_intrinsics()) {
2066         tty->print_cr("  ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
2067                       num_elem, type2name(elem_bt));
2068       }
2069       return false; // not supported
2070     }
2071   }
2072   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2073   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2074 
2075   ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
2076   const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
2077 
2078   Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2079   Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
2080 
2081   if (v1 == NULL || shuffle == NULL) {
2082     return false; // operand unboxing failed
2083   }
2084 
2085   Node* mask = NULL;
2086   if (is_masked_op) {
2087     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2088     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2089     mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2090     if (mask == NULL) {
2091       if (C->print_intrinsics()) {
2092         tty->print_cr("  ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
2093                       num_elem, type2name(elem_bt));
2094       }
2095       return false;
2096     }
2097   }
2098 
2099   Node* rearrange = new VectorRearrangeNode(v1, shuffle);
2100   if (is_masked_op) {
2101     if (use_predicate) {
2102       rearrange->add_req(mask);
2103       rearrange->add_flag(Node::Flag_is_predicated_vector);
2104     } else {
2105       const TypeVect* vt = v1->bottom_type()->is_vect();
2106       rearrange = gvn().transform(rearrange);
2107       Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2108       Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, Type::get_const_basic_type(elem_bt)));
2109       rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2110     }
2111   }
2112   rearrange = gvn().transform(rearrange);
2113 
2114   Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2115   set_result(box);
2116   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2117   return true;
2118 }
2119 
2120 static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
2121   address addr = NULL;
2122   assert(UseVectorStubs, "sanity");
2123   assert(name_ptr != NULL, "unexpected");
2124   assert((vop >= VectorSupport::VECTOR_OP_SVML_START) && (vop <= VectorSupport::VECTOR_OP_SVML_END), "unexpected");
2125   int op = vop - VectorSupport::VECTOR_OP_SVML_START;
2126 
2127   switch(bits) {
2128     case 64:  //fallthough
2129     case 128: //fallthough
2130     case 256: //fallthough
2131     case 512:
2132       if (bt == T_FLOAT) {
2133         snprintf(name_ptr, name_len, "vector_%s_float%d", VectorSupport::svmlname[op], bits);
2134         addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op];
2135       } else {
2136         assert(bt == T_DOUBLE, "must be FP type only");
2137         snprintf(name_ptr, name_len, "vector_%s_double%d", VectorSupport::svmlname[op], bits);
2138         addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op];
2139       }
2140       break;
2141     default:
2142       snprintf(name_ptr, name_len, "invalid");
2143       addr = NULL;
2144       Unimplemented();
2145       break;
2146   }
2147 
2148   return addr;
2149 }
2150 
2151 Node* LibraryCallKit::gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
2152   assert(UseVectorStubs, "sanity");
2153   assert(vector_api_op_id >= VectorSupport::VECTOR_OP_SVML_START && vector_api_op_id <= VectorSupport::VECTOR_OP_SVML_END, "need valid op id");
2154   assert(opd1 != NULL, "must not be null");
2155   const TypeVect* vt = TypeVect::make(bt, num_elem);
2156   const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != NULL ? 2 : 1, vt, vt);
2157   char name[100] = "";
2158 
2159   // Get address for svml method.
2160   address addr = get_svml_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
2161 
2162   if (addr == NULL) {
2163     return NULL;
2164   }
2165 
2166   assert(name != NULL, "name must not be null");
2167   Node* operation = make_runtime_call(RC_VECTOR,
2168                                       call_type,
2169                                       addr,
2170                                       name,
2171                                       TypePtr::BOTTOM,
2172                                       opd1,
2173                                       opd2);
2174   return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
2175 }
2176 
2177 //  public static
2178 //  <V extends Vector<E>,
2179 //   M extends VectorMask<E>,
2180 //   E>
2181 //  V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2182 //                 Class<E> elementType, int length,
2183 //                 V v, int n, M m,
2184 //                 VectorBroadcastIntOp<V, M> defaultImpl)
2185 bool LibraryCallKit::inline_vector_broadcast_int() {
2186   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
2187   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2188   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
2189   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
2190   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
2191 
2192   if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL) {
2193     return false; // dead code
2194   }
2195   if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
2196     if (C->print_intrinsics()) {
2197       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2198                     NodeClassNames[argument(0)->Opcode()],
2199                     NodeClassNames[argument(1)->Opcode()],
2200                     NodeClassNames[argument(3)->Opcode()],
2201                     NodeClassNames[argument(4)->Opcode()]);
2202     }
2203     return false; // not enough info for intrinsification
2204   }
2205   if (!is_klass_initialized(vector_klass)) {
2206     if (C->print_intrinsics()) {
2207       tty->print_cr("  ** klass argument not initialized");
2208     }
2209     return false;
2210   }
2211 
2212   const Type* vmask_type = gvn().type(argument(7));
2213   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2214   if (is_masked_op) {
2215     if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
2216       if (C->print_intrinsics()) {
2217         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2218       }
2219       return false; // not enough info for intrinsification
2220     }
2221 
2222     if (!is_klass_initialized(mask_klass)) {
2223       if (C->print_intrinsics()) {
2224         tty->print_cr("  ** mask klass argument not initialized");
2225       }
2226       return false;
2227     }
2228 
2229     if (vmask_type->maybe_null()) {
2230       if (C->print_intrinsics()) {
2231         tty->print_cr("  ** null mask values are not allowed for masked op");
2232       }
2233       return false;
2234     }
2235   }
2236 
2237   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2238   if (!elem_type->is_primitive_type()) {
2239     if (C->print_intrinsics()) {
2240       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2241     }
2242     return false; // should be primitive type
2243   }
2244 
2245   int num_elem = vlen->get_con();
2246   BasicType elem_bt = elem_type->basic_type();
2247   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2248 
2249   bool is_shift  = VectorNode::is_shift_opcode(opc);
2250   bool is_rotate = VectorNode::is_rotate_opcode(opc);
2251 
2252   if (opc == 0 || (!is_shift && !is_rotate)) {
2253     if (C->print_intrinsics()) {
2254       tty->print_cr("  ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2255     }
2256     return false; // operation not supported
2257   }
2258 
2259   int sopc = VectorNode::opcode(opc, elem_bt);
2260   if (sopc == 0) {
2261     if (C->print_intrinsics()) {
2262       tty->print_cr("  ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2263     }
2264     return false; // operation not supported
2265   }
2266 
2267   Node* cnt  = argument(6);
2268   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2269   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2270   const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2271 
2272   // If CPU supports vector constant rotate instructions pass it directly
2273   bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2274                          Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2275   bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2276 
2277   VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2278   bool use_predicate = is_masked_op;
2279 
2280   if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2281     use_predicate = false;
2282     if (!is_masked_op ||
2283         (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2284          !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2285 
2286       if (C->print_intrinsics()) {
2287         tty->print_cr("  ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2288                       sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2289       }
2290       return false; // not supported
2291     }
2292   }
2293 
2294   Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2295   Node* opd2 = NULL;
2296   if (is_shift) {
2297     opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2298   } else {
2299     assert(is_rotate, "unexpected operation");
2300     if (!is_const_rotate) {
2301       const Type * type_bt = Type::get_const_basic_type(elem_bt);
2302       cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2303       opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, type_bt));
2304     } else {
2305       // Constant shift value.
2306       opd2 = cnt;
2307     }
2308   }
2309 
2310   if (opd1 == NULL || opd2 == NULL) {
2311     return false;
2312   }
2313 
2314   Node* mask = NULL;
2315   if (is_masked_op) {
2316     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2317     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2318     mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2319     if (mask == NULL) {
2320       if (C->print_intrinsics()) {
2321         tty->print_cr("  ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2322       }
2323       return false;
2324     }
2325   }
2326 
2327   Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2328   if (is_masked_op && mask != NULL) {
2329     if (use_predicate) {
2330       operation->add_req(mask);
2331       operation->add_flag(Node::Flag_is_predicated_vector);
2332     } else {
2333       operation = gvn().transform(operation);
2334       operation = new VectorBlendNode(opd1, operation, mask);
2335     }
2336   }
2337   operation = gvn().transform(operation);
2338   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2339   set_result(vbox);
2340   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2341   return true;
2342 }
2343 
2344 // public static <VOUT extends VectorPayload,
2345 //                 VIN extends VectorPayload,
2346 //                   S extends VectorSpecies>
2347 // VOUT convert(int oprId,
2348 //           Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2349 //           Class<?>   toVectorClass, Class<?>   toElementType, int   toVLen,
2350 //           VIN v, S s,
2351 //           VectorConvertOp<VOUT, VIN, S> defaultImpl)
2352 //
2353 bool LibraryCallKit::inline_vector_convert() {
2354   const TypeInt*     opr               = gvn().type(argument(0))->isa_int();
2355 
2356   const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2357   const TypeInstPtr* elem_klass_from   = gvn().type(argument(2))->isa_instptr();
2358   const TypeInt*     vlen_from         = gvn().type(argument(3))->isa_int();
2359 
2360   const TypeInstPtr* vector_klass_to   = gvn().type(argument(4))->isa_instptr();
2361   const TypeInstPtr* elem_klass_to     = gvn().type(argument(5))->isa_instptr();
2362   const TypeInt*     vlen_to           = gvn().type(argument(6))->isa_int();
2363 
2364   if (opr == NULL ||
2365       vector_klass_from == NULL || elem_klass_from == NULL || vlen_from == NULL ||
2366       vector_klass_to   == NULL || elem_klass_to   == NULL || vlen_to   == NULL) {
2367     return false; // dead code
2368   }
2369   if (!opr->is_con() ||
2370       vector_klass_from->const_oop() == NULL || elem_klass_from->const_oop() == NULL || !vlen_from->is_con() ||
2371       vector_klass_to->const_oop() == NULL || elem_klass_to->const_oop() == NULL || !vlen_to->is_con()) {
2372     if (C->print_intrinsics()) {
2373       tty->print_cr("  ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2374                     NodeClassNames[argument(0)->Opcode()],
2375                     NodeClassNames[argument(1)->Opcode()],
2376                     NodeClassNames[argument(2)->Opcode()],
2377                     NodeClassNames[argument(3)->Opcode()],
2378                     NodeClassNames[argument(4)->Opcode()],
2379                     NodeClassNames[argument(5)->Opcode()],
2380                     NodeClassNames[argument(6)->Opcode()]);
2381     }
2382     return false; // not enough info for intrinsification
2383   }
2384   if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2385     if (C->print_intrinsics()) {
2386       tty->print_cr("  ** klass argument not initialized");
2387     }
2388     return false;
2389   }
2390 
2391   assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST  ||
2392          opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
2393          opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2394   bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2395   bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2396 
2397   ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2398   ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2399   if (is_vector_shuffle(vbox_klass_from)) {
2400     return false; // vector shuffles aren't supported
2401   }
2402   bool is_mask = is_vector_mask(vbox_klass_from);
2403 
2404   ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2405   if (!elem_type_from->is_primitive_type()) {
2406     return false; // should be primitive type
2407   }
2408   BasicType elem_bt_from = elem_type_from->basic_type();
2409   ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2410   if (!elem_type_to->is_primitive_type()) {
2411     return false; // should be primitive type
2412   }
2413   BasicType elem_bt_to = elem_type_to->basic_type();
2414 
2415   int num_elem_from = vlen_from->get_con();
2416   int num_elem_to = vlen_to->get_con();
2417 
2418   // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2419   // since we may need to change size.
2420   if (!arch_supports_vector(Op_VectorReinterpret,
2421                             num_elem_from,
2422                             elem_bt_from,
2423                             is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2424     if (C->print_intrinsics()) {
2425       tty->print_cr("  ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2426                     is_cast ? "cast" : "reinterpret",
2427                     num_elem_from, type2name(elem_bt_from), is_mask);
2428     }
2429     return false;
2430   }
2431 
2432   // Check whether we can support resizing/reinterpreting to the new size.
2433   if (!arch_supports_vector(Op_VectorReinterpret,
2434                             num_elem_to,
2435                             elem_bt_to,
2436                             is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2437     if (C->print_intrinsics()) {
2438       tty->print_cr("  ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2439                     is_cast ? "cast" : "reinterpret",
2440                     num_elem_to, type2name(elem_bt_to), is_mask);
2441     }
2442     return false;
2443   }
2444 
2445   // At this point, we know that both input and output vector registers are supported
2446   // by the architecture. Next check if the casted type is simply to same type - which means
2447   // that it is actually a resize and not a cast.
2448   if (is_cast && elem_bt_from == elem_bt_to) {
2449     is_cast = false;
2450   }
2451 
2452   const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2453 
2454   Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2455   if (opd1 == NULL) {
2456     return false;
2457   }
2458 
2459   const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2460   const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2461 
2462   // Safety check to prevent casting if source mask is of type vector
2463   // and destination mask of type predicate vector and vice-versa.
2464   // From X86 standpoint, this case will only arise over KNL target,
2465   // where certain masks (depending on the species) are either propagated
2466   // through a vector or predicate register.
2467   if (is_mask &&
2468       ((src_type->isa_vectmask() == NULL && dst_type->isa_vectmask()) ||
2469        (dst_type->isa_vectmask() == NULL && src_type->isa_vectmask()))) {
2470     return false;
2471   }
2472 
2473   Node* op = opd1;
2474   if (is_cast) {
2475     BasicType new_elem_bt_to = elem_bt_to;
2476     BasicType new_elem_bt_from = elem_bt_from;
2477     if (is_mask && is_floating_point_type(elem_bt_from)) {
2478       new_elem_bt_from = elem_bt_from == T_FLOAT ? T_INT : T_LONG;
2479     }
2480     int cast_vopc = VectorCastNode::opcode(new_elem_bt_from, !is_ucast);
2481     // Make sure that cast is implemented to particular type/size combination.
2482     if (!arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2483       if (C->print_intrinsics()) {
2484         tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2485                       cast_vopc,
2486                       num_elem_to, type2name(elem_bt_to), is_mask);
2487       }
2488       return false;
2489     }
2490 
2491     if (num_elem_from < num_elem_to) {
2492       // Since input and output number of elements are not consistent, we need to make sure we
2493       // properly size. Thus, first make a cast that retains the number of elements from source.
2494       int num_elem_for_cast = num_elem_from;
2495 
2496       // It is possible that arch does not support this intermediate vector size
2497       // TODO More complex logic required here to handle this corner case for the sizes.
2498       if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2499         if (C->print_intrinsics()) {
2500           tty->print_cr("  ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2501                         cast_vopc,
2502                         num_elem_for_cast, type2name(elem_bt_to), is_mask);
2503         }
2504         return false;
2505       }
2506 
2507       op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2508       // Now ensure that the destination gets properly resized to needed size.
2509       op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2510     } else if (num_elem_from > num_elem_to) {
2511       // Since number elements from input is larger than output, simply reduce size of input (we are supposed to
2512       // drop top elements anyway).
2513       int num_elem_for_resize = num_elem_to;
2514 
2515       // It is possible that arch does not support this intermediate vector size
2516       // TODO More complex logic required here to handle this corner case for the sizes.
2517       if (!arch_supports_vector(Op_VectorReinterpret,
2518                                 num_elem_for_resize,
2519                                 elem_bt_from,
2520                                 VecMaskNotUsed)) {
2521         if (C->print_intrinsics()) {
2522           tty->print_cr("  ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2523                         num_elem_for_resize, type2name(elem_bt_from), is_mask);
2524         }
2525         return false;
2526       }
2527 
2528       op = gvn().transform(new VectorReinterpretNode(op,
2529                                                      src_type,
2530                                                      TypeVect::make(elem_bt_from,
2531                                                                     num_elem_for_resize)));
2532       op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2533     } else {
2534       if (is_mask) {
2535         if ((dst_type->isa_vectmask() && src_type->isa_vectmask()) ||
2536             (type2aelembytes(elem_bt_from) == type2aelembytes(elem_bt_to))) {
2537           op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2538         } else {
2539           op = VectorMaskCastNode::makeCastNode(&gvn(), op, dst_type);
2540         }
2541       } else {
2542         // Since input and output number of elements match, and since we know this vector size is
2543         // supported, simply do a cast with no resize needed.
2544         op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2545       }
2546     }
2547   } else if (Type::cmp(src_type, dst_type) != 0) {
2548     assert(!is_cast, "must be reinterpret");
2549     op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2550   }
2551 
2552   const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2553   Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2554   set_result(vbox);
2555   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2556   return true;
2557 }
2558 
2559 //  public static
2560 //  <V extends Vector<E>,
2561 //   E>
2562 //  V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2563 //           V vec, int ix, long val,
2564 //           VecInsertOp<V> defaultImpl)
2565 bool LibraryCallKit::inline_vector_insert() {
2566   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2567   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2568   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2569   const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
2570 
2571   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || idx == NULL) {
2572     return false; // dead code
2573   }
2574   if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
2575     if (C->print_intrinsics()) {
2576       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2577                     NodeClassNames[argument(0)->Opcode()],
2578                     NodeClassNames[argument(1)->Opcode()],
2579                     NodeClassNames[argument(2)->Opcode()],
2580                     NodeClassNames[argument(4)->Opcode()]);
2581     }
2582     return false; // not enough info for intrinsification
2583   }
2584   if (!is_klass_initialized(vector_klass)) {
2585     if (C->print_intrinsics()) {
2586       tty->print_cr("  ** klass argument not initialized");
2587     }
2588     return false;
2589   }
2590   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2591   if (!elem_type->is_primitive_type()) {
2592     if (C->print_intrinsics()) {
2593       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2594     }
2595     return false; // should be primitive type
2596   }
2597   BasicType elem_bt = elem_type->basic_type();
2598   int num_elem = vlen->get_con();
2599   if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2600     if (C->print_intrinsics()) {
2601       tty->print_cr("  ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2602                     num_elem, type2name(elem_bt));
2603     }
2604     return false; // not supported
2605   }
2606 
2607   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2608   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2609 
2610   Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2611   if (opd == NULL) {
2612     return false;
2613   }
2614 
2615   Node* insert_val = argument(5);
2616   assert(gvn().type(insert_val)->isa_long() != NULL, "expected to be long");
2617 
2618   // Convert insert value back to its appropriate type.
2619   switch (elem_bt) {
2620     case T_BYTE:
2621       insert_val = gvn().transform(new ConvL2INode(insert_val));
2622       insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::BYTE));
2623       break;
2624     case T_SHORT:
2625       insert_val = gvn().transform(new ConvL2INode(insert_val));
2626       insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::SHORT));
2627       break;
2628     case T_INT:
2629       insert_val = gvn().transform(new ConvL2INode(insert_val));
2630       break;
2631     case T_FLOAT:
2632       insert_val = gvn().transform(new ConvL2INode(insert_val));
2633       insert_val = gvn().transform(new MoveI2FNode(insert_val));
2634       break;
2635     case T_DOUBLE:
2636       insert_val = gvn().transform(new MoveL2DNode(insert_val));
2637       break;
2638     case T_LONG:
2639       // no conversion needed
2640       break;
2641     default: fatal("%s", type2name(elem_bt)); break;
2642   }
2643 
2644   Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con()));
2645 
2646   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2647   set_result(vbox);
2648   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2649   return true;
2650 }
2651 
2652 //  public static
2653 //  <V extends Vector<E>,
2654 //   E>
2655 //  long extract(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2656 //               V vec, int ix,
2657 //               VecExtractOp<V> defaultImpl)
2658 bool LibraryCallKit::inline_vector_extract() {
2659   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2660   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2661   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2662   const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
2663 
2664   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || idx == NULL) {
2665     return false; // dead code
2666   }
2667   if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
2668     if (C->print_intrinsics()) {
2669       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2670                     NodeClassNames[argument(0)->Opcode()],
2671                     NodeClassNames[argument(1)->Opcode()],
2672                     NodeClassNames[argument(2)->Opcode()],
2673                     NodeClassNames[argument(4)->Opcode()]);
2674     }
2675     return false; // not enough info for intrinsification
2676   }
2677   if (!is_klass_initialized(vector_klass)) {
2678     if (C->print_intrinsics()) {
2679       tty->print_cr("  ** klass argument not initialized");
2680     }
2681     return false;
2682   }
2683   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2684   if (!elem_type->is_primitive_type()) {
2685     if (C->print_intrinsics()) {
2686       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2687     }
2688     return false; // should be primitive type
2689   }
2690   BasicType elem_bt = elem_type->basic_type();
2691   int num_elem = vlen->get_con();
2692   int vopc = ExtractNode::opcode(elem_bt);
2693   if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2694     if (C->print_intrinsics()) {
2695       tty->print_cr("  ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2696                     num_elem, type2name(elem_bt));
2697     }
2698     return false; // not supported
2699   }
2700 
2701   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2702   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2703 
2704   Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2705   if (opd == NULL) {
2706     return false;
2707   }
2708 
2709   Node* operation = gvn().transform(ExtractNode::make(opd, idx->get_con(), elem_bt));
2710 
2711   Node* bits = NULL;
2712   switch (elem_bt) {
2713     case T_BYTE:
2714     case T_SHORT:
2715     case T_INT: {
2716       bits = gvn().transform(new ConvI2LNode(operation));
2717       break;
2718     }
2719     case T_FLOAT: {
2720       bits = gvn().transform(new MoveF2INode(operation));
2721       bits = gvn().transform(new ConvI2LNode(bits));
2722       break;
2723     }
2724     case T_DOUBLE: {
2725       bits = gvn().transform(new MoveD2LNode(operation));
2726       break;
2727     }
2728     case T_LONG: {
2729       bits = operation; // no conversion needed
2730       break;
2731     }
2732     default: fatal("%s", type2name(elem_bt));
2733   }
2734 
2735   set_result(bits);
2736   return true;
2737 }
2738 






























































































--- EOF ---