1 /*
   2  * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "ci/ciSymbols.hpp"
  27 #include "classfile/vmSymbols.hpp"
  28 #include "opto/library_call.hpp"
  29 #include "opto/runtime.hpp"
  30 #include "opto/vectornode.hpp"
  31 #include "prims/vectorSupport.hpp"
  32 #include "runtime/stubRoutines.hpp"
  33 
  34 #ifdef ASSERT
  35 static bool is_vector(ciKlass* klass) {
  36   return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
  37 }
  38 
  39 static bool check_vbox(const TypeInstPtr* vbox_type) {
  40   assert(vbox_type->klass_is_exact(), "");
  41 
  42   ciInstanceKlass* ik = vbox_type->klass()->as_instance_klass();
  43   assert(is_vector(ik), "not a vector");
  44 
  45   ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
  46   assert(fd1 != NULL, "element type info is missing");
  47 
  48   ciConstant val1 = fd1->constant_value();
  49   BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
  50   assert(is_java_primitive(elem_bt), "element type info is missing");
  51 
  52   ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
  53   assert(fd2 != NULL, "vector length info is missing");
  54 
  55   ciConstant val2 = fd2->constant_value();
  56   assert(val2.as_int() > 0, "vector length info is missing");
  57 
  58   return true;
  59 }
  60 #endif
  61 
  62 static bool is_vector_mask(ciKlass* klass) {
  63   return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
  64 }
  65 
  66 static bool is_vector_shuffle(ciKlass* klass) {
  67   return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
  68 }
  69 
  70 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
  71                                                  VectorMaskUseType mask_use_type, bool has_scalar_args) {
  72   bool is_supported = true;
  73 
  74   // has_scalar_args flag is true only for non-constant scalar shift count,
  75   // since in this case shift needs to be broadcasted.
  76   if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
  77        (has_scalar_args &&
  78          !arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed))) {
  79     is_supported = false;
  80   }
  81 
  82   if (is_supported) {
  83     // Check whether mask unboxing is supported.
  84     if ((mask_use_type & VecMaskUseLoad) != 0) {
  85       if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt) ||
  86           !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
  87       #ifndef PRODUCT
  88         if (C->print_intrinsics()) {
  89           tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
  90                         NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
  91         }
  92       #endif
  93         return false;
  94       }
  95     }
  96 
  97     if ((mask_use_type & VecMaskUsePred) != 0) {
  98       if (!Matcher::has_predicated_vectors() ||
  99           !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
 100       #ifndef PRODUCT
 101         if (C->print_intrinsics()) {
 102           tty->print_cr("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
 103                         NodeClassNames[opc], type2name(elem_bt), num_elem);
 104         }
 105       #endif
 106         return false;
 107       }
 108     }
 109   }
 110 
 111   int lshiftopc, rshiftopc;
 112   switch(elem_bt) {
 113     case T_BYTE:
 114       lshiftopc = Op_LShiftI;
 115       rshiftopc = Op_URShiftB;
 116       break;
 117     case T_SHORT:
 118       lshiftopc = Op_LShiftI;
 119       rshiftopc = Op_URShiftS;
 120       break;
 121     case T_INT:
 122       lshiftopc = Op_LShiftI;
 123       rshiftopc = Op_URShiftI;
 124       break;
 125     case T_LONG:
 126       lshiftopc = Op_LShiftL;
 127       rshiftopc = Op_URShiftL;
 128       break;
 129     default:
 130       assert(false, "Unexpected type");
 131   }
 132   int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
 133   int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
 134   if (!is_supported &&
 135       arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
 136       arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
 137       arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
 138     is_supported = true;
 139   }
 140   return is_supported;
 141 }
 142 
 143 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
 144   assert(EnableVectorSupport, "");
 145 
 146   PreserveReexecuteState preexecs(this);
 147   jvms()->set_should_reexecute(true);
 148 
 149   VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
 150   set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
 151   make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
 152   set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
 153   set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
 154   Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
 155 
 156   assert(check_vbox(vbox_type), "");
 157   const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->klass()));
 158   VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
 159   return gvn().transform(vbox);
 160 }
 161 
 162 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) {
 163   assert(EnableVectorSupport, "");
 164   const TypeInstPtr* vbox_type_v = gvn().type(v)->is_instptr();
 165   if (vbox_type->klass() != vbox_type_v->klass()) {
 166     return NULL; // arguments don't agree on vector shapes
 167   }
 168   if (vbox_type_v->maybe_null()) {
 169     return NULL; // no nulls are allowed
 170   }
 171   assert(check_vbox(vbox_type), "");
 172   const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->klass()));
 173   Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector));
 174   return unbox;
 175 }
 176 
 177 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
 178   assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
 179   juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
 180   Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
 181   Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
 182   return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
 183 }
 184 
 185 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
 186   // Check that the operation is valid.
 187   if (sopc <= 0) {
 188 #ifndef PRODUCT
 189     if (C->print_intrinsics()) {
 190       tty->print_cr("  ** Rejected intrinsification because no valid vector op could be extracted");
 191     }
 192 #endif
 193     return false;
 194   }
 195 
 196   if (VectorNode::is_vector_rotate(sopc)) {
 197     if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
 198 #ifndef PRODUCT
 199       if (C->print_intrinsics()) {
 200         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
 201                       NodeClassNames[sopc], type2name(type), num_elem);
 202       }
 203 #endif
 204       return false;
 205     }
 206   } else if (VectorNode::is_vector_integral_negate(sopc)) {
 207     if (!VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, false)) {
 208 #ifndef PRODUCT
 209       if (C->print_intrinsics()) {
 210         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support integral vector negate",
 211                       NodeClassNames[sopc], type2name(type), num_elem);
 212       }
 213 #endif
 214       return false;
 215     }
 216   } else {
 217     // Check that architecture supports this op-size-type combination.
 218     if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
 219 #ifndef PRODUCT
 220       if (C->print_intrinsics()) {
 221         tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support it",
 222                       NodeClassNames[sopc], type2name(type), num_elem);
 223       }
 224 #endif
 225       return false;
 226     } else {
 227       assert(Matcher::match_rule_supported(sopc), "must be supported");
 228     }
 229   }
 230 
 231   if (num_elem == 1) {
 232     if (mask_use_type != VecMaskNotUsed) {
 233 #ifndef PRODUCT
 234       if (C->print_intrinsics()) {
 235         tty->print_cr("  ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
 236                       NodeClassNames[sopc], type2name(type), num_elem);
 237       }
 238 #endif
 239       return false;
 240     }
 241 
 242     if (sopc != 0) {
 243       if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
 244 #ifndef PRODUCT
 245         if (C->print_intrinsics()) {
 246           tty->print_cr("  ** Not a svml call or load/store vector op (%s,%s,%d)",
 247                         NodeClassNames[sopc], type2name(type), num_elem);
 248         }
 249 #endif
 250         return false;
 251       }
 252     }
 253   }
 254 
 255   if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
 256       Matcher::supports_vector_variable_shifts() == false) {
 257     if (C->print_intrinsics()) {
 258       tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
 259                     NodeClassNames[sopc], type2name(type), num_elem);
 260     }
 261     return false;
 262   }
 263 
 264   // Check whether mask unboxing is supported.
 265   if ((mask_use_type & VecMaskUseLoad) != 0) {
 266     if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type) ||
 267         !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
 268     #ifndef PRODUCT
 269       if (C->print_intrinsics()) {
 270         tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
 271                       NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
 272       }
 273     #endif
 274       return false;
 275     }
 276   }
 277 
 278   // Check whether mask boxing is supported.
 279   if ((mask_use_type & VecMaskUseStore) != 0) {
 280     if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type) ||
 281         !Matcher::match_rule_supported_vector(Op_StoreVector, num_elem, T_BOOLEAN)) {
 282     #ifndef PRODUCT
 283       if (C->print_intrinsics()) {
 284         tty->print_cr("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
 285                       NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
 286       }
 287     #endif
 288       return false;
 289     }
 290   }
 291 
 292   if ((mask_use_type & VecMaskUsePred) != 0) {
 293     bool is_supported = false;
 294     if (Matcher::has_predicated_vectors()) {
 295       if (VectorNode::is_vector_integral_negate(sopc)) {
 296         is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
 297       } else {
 298         is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
 299       }
 300     }
 301 
 302     if (!is_supported) {
 303     #ifndef PRODUCT
 304       if (C->print_intrinsics()) {
 305         tty->print_cr("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
 306                       NodeClassNames[sopc], type2name(type), num_elem);
 307       }
 308     #endif
 309       return false;
 310     }
 311   }
 312 
 313   return true;
 314 }
 315 
 316 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
 317   if (vec_klass->const_oop() == NULL) {
 318     return false; // uninitialized or some kind of unsafe access
 319   }
 320   assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != NULL, "klass instance expected");
 321   ciInstanceKlass* klass =  vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
 322   return klass->is_initialized();
 323 }
 324 
 325 // public static
 326 // <V extends Vector<E>,
 327 //  M extends VectorMask<E>,
 328 //  E>
 329 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 330 //           int length, V v, M m,
 331 //           UnaryOperation<V, M> defaultImpl)
 332 //
 333 // public static
 334 // <V,
 335 //  M extends VectorMask<E>,
 336 //  E>
 337 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 338 //            int length, V v1, V v2, M m,
 339 //            BinaryOperation<V, M> defaultImpl)
 340 //
 341 // public static
 342 // <V extends Vector<E>,
 343 //  M extends VectorMask<E>,
 344 //  E>
 345 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 346 //             int length, V v1, V v2, V v3, M m,
 347 //             TernaryOperation<V, M> defaultImpl)
 348 //
 349 bool LibraryCallKit::inline_vector_nary_operation(int n) {
 350   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
 351   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
 352   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
 353   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
 354   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
 355 
 356   if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
 357       !opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 358     if (C->print_intrinsics()) {
 359       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
 360                     NodeClassNames[argument(0)->Opcode()],
 361                     NodeClassNames[argument(1)->Opcode()],
 362                     NodeClassNames[argument(3)->Opcode()],
 363                     NodeClassNames[argument(4)->Opcode()]);
 364     }
 365     return false; // not enough info for intrinsification
 366   }
 367 
 368   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 369   if (!elem_type->is_primitive_type()) {
 370     if (C->print_intrinsics()) {
 371       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 372     }
 373     return false; // should be primitive type
 374   }
 375   if (!is_klass_initialized(vector_klass)) {
 376     if (C->print_intrinsics()) {
 377       tty->print_cr("  ** klass argument not initialized");
 378     }
 379     return false;
 380   }
 381 
 382   // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
 383   // is used to control this operation.
 384   const Type* vmask_type = gvn().type(argument(n + 5));
 385   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
 386   if (is_masked_op) {
 387     if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
 388       if (C->print_intrinsics()) {
 389         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
 390       }
 391       return false; // not enough info for intrinsification
 392     }
 393 
 394     if (!is_klass_initialized(mask_klass)) {
 395       if (C->print_intrinsics()) {
 396         tty->print_cr("  ** mask klass argument not initialized");
 397       }
 398       return false;
 399     }
 400 
 401     if (vmask_type->maybe_null()) {
 402       if (C->print_intrinsics()) {
 403         tty->print_cr("  ** null mask values are not allowed for masked op");
 404       }
 405       return false;
 406     }
 407   }
 408 
 409   BasicType elem_bt = elem_type->basic_type();
 410   int num_elem = vlen->get_con();
 411   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
 412   int sopc = VectorNode::opcode(opc, elem_bt);
 413   if ((opc != Op_CallLeafVector) && (sopc == 0)) {
 414     if (C->print_intrinsics()) {
 415       tty->print_cr("  ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
 416     }
 417     return false; // operation not supported
 418   }
 419   if (num_elem == 1) {
 420     if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
 421       if (C->print_intrinsics()) {
 422         tty->print_cr("  ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
 423                       n, opc, num_elem, type2name(elem_bt));
 424       }
 425       return false;
 426     }
 427   }
 428   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 429   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 430 
 431   if (is_vector_mask(vbox_klass)) {
 432     assert(!is_masked_op, "mask operations do not need mask to control");
 433   }
 434 
 435   if (opc == Op_CallLeafVector) {
 436     if (!UseVectorStubs) {
 437       if (C->print_intrinsics()) {
 438         tty->print_cr("  ** vector stubs support is disabled");
 439       }
 440       return false;
 441     }
 442     if (!Matcher::supports_vector_calling_convention()) {
 443       if (C->print_intrinsics()) {
 444         tty->print_cr("  ** no vector calling conventions supported");
 445       }
 446       return false;
 447     }
 448     if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
 449       if (C->print_intrinsics()) {
 450         tty->print_cr("  ** vector size (vlen=%d, etype=%s) is not supported",
 451                       num_elem, type2name(elem_bt));
 452       }
 453       return false;
 454     }
 455   }
 456 
 457   // When using mask, mask use type needs to be VecMaskUseLoad.
 458   VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
 459                                       : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
 460   if ((sopc != 0) && !arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
 461     if (C->print_intrinsics()) {
 462       tty->print_cr("  ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
 463                     n, sopc, num_elem, type2name(elem_bt),
 464                     is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
 465     }
 466     return false; // not supported
 467   }
 468 
 469   // Return true if current platform has implemented the masked operation with predicate feature.
 470   bool use_predicate = is_masked_op && sopc != 0 && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
 471   if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
 472     if (C->print_intrinsics()) {
 473       tty->print_cr("  ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
 474                     n, sopc, num_elem, type2name(elem_bt));
 475     }
 476     return false;
 477   }
 478 
 479   Node* opd1 = NULL; Node* opd2 = NULL; Node* opd3 = NULL;
 480   switch (n) {
 481     case 3: {
 482       opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
 483       if (opd3 == NULL) {
 484         if (C->print_intrinsics()) {
 485           tty->print_cr("  ** unbox failed v3=%s",
 486                         NodeClassNames[argument(7)->Opcode()]);
 487         }
 488         return false;
 489       }
 490       // fall-through
 491     }
 492     case 2: {
 493       opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
 494       if (opd2 == NULL) {
 495         if (C->print_intrinsics()) {
 496           tty->print_cr("  ** unbox failed v2=%s",
 497                         NodeClassNames[argument(6)->Opcode()]);
 498         }
 499         return false;
 500       }
 501       // fall-through
 502     }
 503     case 1: {
 504       opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
 505       if (opd1 == NULL) {
 506         if (C->print_intrinsics()) {
 507           tty->print_cr("  ** unbox failed v1=%s",
 508                         NodeClassNames[argument(5)->Opcode()]);
 509         }
 510         return false;
 511       }
 512       break;
 513     }
 514     default: fatal("unsupported arity: %d", n);
 515   }
 516 
 517   Node* mask = NULL;
 518   if (is_masked_op) {
 519     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
 520     assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
 521     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
 522     mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
 523     if (mask == NULL) {
 524       if (C->print_intrinsics()) {
 525         tty->print_cr("  ** unbox failed mask=%s",
 526                       NodeClassNames[argument(n + 5)->Opcode()]);
 527       }
 528       return false;
 529     }
 530   }
 531 
 532   Node* operation = NULL;
 533   if (opc == Op_CallLeafVector) {
 534     assert(UseVectorStubs, "sanity");
 535     operation = gen_call_to_svml(opr->get_con(), elem_bt, num_elem, opd1, opd2);
 536     if (operation == NULL) {
 537       if (C->print_intrinsics()) {
 538         tty->print_cr("  ** svml call failed for %s_%s_%d",
 539                          (elem_bt == T_FLOAT)?"float":"double",
 540                          VectorSupport::svmlname[opr->get_con() - VectorSupport::VECTOR_OP_SVML_START],
 541                          num_elem * type2aelembytes(elem_bt));
 542       }
 543       return false;
 544      }
 545   } else {
 546     const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
 547     switch (n) {
 548       case 1:
 549       case 2: {
 550         operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc));
 551         break;
 552       }
 553       case 3: {
 554         operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
 555         break;
 556       }
 557       default: fatal("unsupported arity: %d", n);
 558     }
 559   }
 560 
 561   if (is_masked_op && mask != NULL) {
 562     if (use_predicate) {
 563       operation->add_req(mask);
 564       operation->add_flag(Node::Flag_is_predicated_vector);
 565     } else {
 566       operation = gvn().transform(operation);
 567       operation->add_flag(Node::Flag_is_predicated_using_blend);
 568       operation = new VectorBlendNode(opd1, operation, mask);
 569     }
 570   }
 571   operation = gvn().transform(operation);
 572 
 573   // Wrap it up in VectorBox to keep object type information.
 574   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
 575   set_result(vbox);
 576   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 577   return true;
 578 }
 579 
 580 // <Sh extends VectorShuffle<E>,  E>
 581 //  Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
 582 //                  int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
 583 bool LibraryCallKit::inline_vector_shuffle_iota() {
 584   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
 585   const TypeInt*     vlen          = gvn().type(argument(3))->isa_int();
 586   const TypeInt*     start_val     = gvn().type(argument(4))->isa_int();
 587   const TypeInt*     step_val      = gvn().type(argument(5))->isa_int();
 588   const TypeInt*     wrap          = gvn().type(argument(6))->isa_int();
 589 
 590   Node* start = argument(4);
 591   Node* step  = argument(5);
 592 
 593   if (shuffle_klass == NULL || vlen == NULL || start_val == NULL || step_val == NULL || wrap == NULL) {
 594     return false; // dead code
 595   }
 596   if (!vlen->is_con() || !is_power_of_2(vlen->get_con()) ||
 597       shuffle_klass->const_oop() == NULL || !wrap->is_con()) {
 598     return false; // not enough info for intrinsification
 599   }
 600   if (!is_klass_initialized(shuffle_klass)) {
 601     if (C->print_intrinsics()) {
 602       tty->print_cr("  ** klass argument not initialized");
 603     }
 604     return false;
 605   }
 606 
 607   int do_wrap = wrap->get_con();
 608   int num_elem = vlen->get_con();
 609   BasicType elem_bt = T_BYTE;
 610 
 611   if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed)) {
 612     return false;
 613   }
 614   if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed)) {
 615     return false;
 616   }
 617   if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) {
 618     return false;
 619   }
 620   if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
 621     return false;
 622   }
 623   if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
 624     return false;
 625   }
 626   if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
 627     return false;
 628   }
 629 
 630   const Type * type_bt = Type::get_const_basic_type(elem_bt);
 631   const TypeVect * vt  = TypeVect::make(type_bt, num_elem);
 632 
 633   Node* res =  gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
 634 
 635   if(!step_val->is_con() || !is_power_of_2(step_val->get_con())) {
 636     Node* bcast_step     = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt));
 637     res = gvn().transform(VectorNode::make(Op_MulI, res, bcast_step, num_elem, elem_bt));
 638   } else if (step_val->get_con() > 1) {
 639     Node* cnt = gvn().makecon(TypeInt::make(log2i_exact(step_val->get_con())));
 640     Node* shift_cnt = vector_shift_count(cnt, Op_LShiftI, elem_bt, num_elem);
 641     res = gvn().transform(VectorNode::make(Op_LShiftVB, res, shift_cnt, vt));
 642   }
 643 
 644   if (!start_val->is_con() || start_val->get_con() != 0) {
 645     Node* bcast_start    = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt));
 646     res = gvn().transform(VectorNode::make(Op_AddI, res, bcast_start, num_elem, elem_bt));
 647   }
 648 
 649   Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1));
 650   Node * bcast_mod  = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt));
 651   if(do_wrap)  {
 652     // Wrap the indices greater than lane count.
 653     res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
 654   } else {
 655     ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::ge));
 656     Node * lane_cnt  = gvn().makecon(TypeInt::make(num_elem));
 657     Node * bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt));
 658     const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
 659     Node* mask = gvn().transform(new VectorMaskCmpNode(BoolTest::ge, bcast_lane_cnt, res, pred_node, vmask_type));
 660 
 661     // Make the indices greater than lane count as -ve values. This matches the java side implementation.
 662     res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
 663     Node * biased_val = gvn().transform(VectorNode::make(Op_SubI, res, bcast_lane_cnt, num_elem, elem_bt));
 664     res = gvn().transform(new VectorBlendNode(biased_val, res, mask));
 665   }
 666 
 667   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 668   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 669 
 670   // Wrap it up in VectorBox to keep object type information.
 671   res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
 672   set_result(res);
 673   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 674   return true;
 675 }
 676 
 677 // <E, M>
 678 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
 679 //                          int length, M m, VectorMaskOp<M> defaultImpl)
 680 bool LibraryCallKit::inline_vector_mask_operation() {
 681   const TypeInt*     oper       = gvn().type(argument(0))->isa_int();
 682   const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
 683   const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
 684   const TypeInt*     vlen       = gvn().type(argument(3))->isa_int();
 685   Node*              mask       = argument(4);
 686 
 687   if (mask_klass == NULL || elem_klass == NULL || mask->is_top() || vlen == NULL) {
 688     return false; // dead code
 689   }
 690 
 691   if (!is_klass_initialized(mask_klass)) {
 692     if (C->print_intrinsics()) {
 693       tty->print_cr("  ** klass argument not initialized");
 694     }
 695     return false;
 696   }
 697 
 698   int num_elem = vlen->get_con();
 699   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 700   BasicType elem_bt = elem_type->basic_type();
 701 








 702   int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
 703   if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
 704     if (C->print_intrinsics()) {
 705       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 706                     mopc, num_elem, type2name(elem_bt));
 707     }
 708     return false; // not supported
 709   }
 710 
 711   const Type* elem_ty = Type::get_const_basic_type(elem_bt);
 712   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
 713   const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
 714   Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
 715   if (mask_vec == NULL) {
 716     if (C->print_intrinsics()) {
 717         tty->print_cr("  ** unbox failed mask=%s",
 718                       NodeClassNames[argument(4)->Opcode()]);
 719     }
 720     return false;
 721   }
 722 
 723   if (mask_vec->bottom_type()->isa_vectmask() == NULL) {
 724     mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
 725   }
 726   const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
 727   Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
 728   if (mopc != Op_VectorMaskToLong) {
 729     maskoper = ConvI2L(maskoper);
 730   }
 731   set_result(maskoper);
 732 
 733   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 734   return true;
 735 }
 736 
 737 // public static
 738 // <V,
 739 //  Sh extends VectorShuffle<E>,
 740 //  E>
 741 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
 742 //                   Class<? extends Sh> shuffleClass, Sh s, int length,
 743 //                   ShuffleToVectorOperation<V, Sh, E> defaultImpl)
 744 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
 745   const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
 746   const TypeInstPtr* elem_klass    = gvn().type(argument(1))->isa_instptr();
 747   const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
 748   Node*              shuffle       = argument(3);
 749   const TypeInt*     vlen          = gvn().type(argument(4))->isa_int();
 750 
 751   if (vector_klass == NULL || elem_klass == NULL || shuffle_klass == NULL || shuffle->is_top() || vlen == NULL) {
 752     return false; // dead code
 753   }
 754   if (!vlen->is_con() || vector_klass->const_oop() == NULL || shuffle_klass->const_oop() == NULL) {
 755     return false; // not enough info for intrinsification
 756   }
 757   if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
 758     if (C->print_intrinsics()) {
 759       tty->print_cr("  ** klass argument not initialized");
 760     }
 761     return false;
 762   }
 763 
 764   int num_elem = vlen->get_con();
 765   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 766   BasicType elem_bt = elem_type->basic_type();
 767 
 768   if (num_elem < 4) {
 769     return false;
 770   }
 771 
 772   int cast_vopc = VectorCastNode::opcode(T_BYTE); // from shuffle of type T_BYTE
 773   // Make sure that cast is implemented to particular type/size combination.
 774   if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
 775     if (C->print_intrinsics()) {
 776       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 777         cast_vopc, num_elem, type2name(elem_bt));
 778     }
 779     return false;
 780   }
 781 
 782   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 783   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 784 
 785   // Unbox shuffle with true flag to indicate its load shuffle to vector
 786   // shuffle is a byte array
 787   Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, T_BYTE, num_elem, true);
 788 
 789   // cast byte to target element type
 790   shuffle_vec = gvn().transform(VectorCastNode::make(cast_vopc, shuffle_vec, elem_bt, num_elem));
 791 
 792   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 793   const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 794 
 795   // Box vector
 796   Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem);
 797   set_result(res);
 798   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 799   return true;
 800 }
 801 
 802 // public static
 803 // <M,
 804 //  S extends VectorSpecies<E>,
 805 //  E>
 806 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
 807 //                    long bits, int mode, S s,
 808 //                    BroadcastOperation<M, E, S> defaultImpl)
 809 bool LibraryCallKit::inline_vector_frombits_coerced() {
 810   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 811   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 812   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 813   const TypeLong*    bits_type    = gvn().type(argument(3))->isa_long();
 814   // Mode argument determines the mode of operation it can take following values:-
 815   // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
 816   // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
 817   const TypeInt*     mode         = gvn().type(argument(5))->isa_int();
 818 
 819   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || mode == NULL ||
 820       bits_type == NULL || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL ||
 821       !vlen->is_con() || !mode->is_con()) {
 822     if (C->print_intrinsics()) {
 823       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
 824                     NodeClassNames[argument(0)->Opcode()],
 825                     NodeClassNames[argument(1)->Opcode()],
 826                     NodeClassNames[argument(2)->Opcode()],
 827                     NodeClassNames[argument(5)->Opcode()]);
 828     }
 829     return false; // not enough info for intrinsification
 830   }
 831 
 832   if (!is_klass_initialized(vector_klass)) {
 833     if (C->print_intrinsics()) {
 834       tty->print_cr("  ** klass argument not initialized");
 835     }
 836     return false;
 837   }
 838   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 839   if (!elem_type->is_primitive_type()) {
 840     if (C->print_intrinsics()) {
 841       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 842     }
 843     return false; // should be primitive type
 844   }
 845   BasicType elem_bt = elem_type->basic_type();
 846   int num_elem = vlen->get_con();
 847   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 848   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 849 
 850   bool is_mask = is_vector_mask(vbox_klass);
 851   int  bcast_mode = mode->get_con();
 852   VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
 853   int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : VectorNode::replicate_opcode(elem_bt);
 854 
 855   if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
 856     if (C->print_intrinsics()) {
 857       tty->print_cr("  ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
 858                     num_elem, type2name(elem_bt),
 859                     is_mask ? 1 : 0,
 860                     bcast_mode);
 861     }
 862     return false; // not supported
 863   }
 864 
 865   Node* broadcast = NULL;
 866   Node* bits = argument(3);
 867   Node* elem = bits;
 868 
 869   if (opc == Op_VectorLongToMask) {
 870     const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
 871     if (vt->isa_vectmask()) {
 872       broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
 873     } else {
 874       const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
 875       broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
 876       broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
 877     }
 878   } else {
 879     switch (elem_bt) {
 880       case T_BOOLEAN: // fall-through
 881       case T_BYTE:    // fall-through
 882       case T_SHORT:   // fall-through
 883       case T_CHAR:    // fall-through
 884       case T_INT: {
 885         elem = gvn().transform(new ConvL2INode(bits));
 886         break;
 887       }
 888       case T_DOUBLE: {
 889         elem = gvn().transform(new MoveL2DNode(bits));
 890         break;
 891       }
 892       case T_FLOAT: {
 893         bits = gvn().transform(new ConvL2INode(bits));
 894         elem = gvn().transform(new MoveI2FNode(bits));
 895         break;
 896       }
 897       case T_LONG: {
 898         // no conversion needed
 899         break;
 900       }
 901       default: fatal("%s", type2name(elem_bt));
 902     }
 903     broadcast = VectorNode::scalar2vector(elem, num_elem, Type::get_const_basic_type(elem_bt), is_mask);
 904     broadcast = gvn().transform(broadcast);
 905   }
 906 
 907   Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
 908   set_result(box);
 909   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 910   return true;
 911 }
 912 
 913 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type) {
 914   assert(arr_type != NULL, "unexpected");
 915   BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
 916   if (elem_bt == arr_elem_bt) {
 917     return true;
 918   } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
 919     // Load/store of short vector from/to char[] is supported
 920     return true;
 921   } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
 922     // Load/store of byte vector from/to boolean[] is supported
 923     return true;
 924   } else {
 925     return false;
 926   }
 927 }
 928 
 929 // public static
 930 // <C,
 931 //  VM,
 932 //  E,
 933 //  S extends VectorSpecies<E>>
 934 // VM load(Class<? extends VM> vmClass, Class<E> elementType, int length,
 935 //         Object base, long offset,    // Unsafe addressing
 936 //         C container, long index, S s,     // Arguments for default implementation
 937 //         LoadOperation<C, VM, E, S> defaultImpl)
 938 //
 939 // public static
 940 // <C,
 941 //  V extends Vector<?>>
 942 // void store(Class<?> vectorClass, Class<?> elementType, int length,
 943 //            Object base, long offset,    // Unsafe addressing
 944 //            V v,
 945 //            C container, long index,      // Arguments for default implementation
 946 //            StoreVectorOperation<C, V> defaultImpl)
 947 
 948 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
 949   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 950   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 951   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 952 
 953   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
 954       vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 955     if (C->print_intrinsics()) {
 956       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
 957                     NodeClassNames[argument(0)->Opcode()],
 958                     NodeClassNames[argument(1)->Opcode()],
 959                     NodeClassNames[argument(2)->Opcode()]);
 960     }
 961     return false; // not enough info for intrinsification
 962   }
 963   if (!is_klass_initialized(vector_klass)) {
 964     if (C->print_intrinsics()) {
 965       tty->print_cr("  ** klass argument not initialized");
 966     }
 967     return false;
 968   }
 969 
 970   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 971   if (!elem_type->is_primitive_type()) {
 972     if (C->print_intrinsics()) {
 973       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 974     }
 975     return false; // should be primitive type
 976   }
 977   BasicType elem_bt = elem_type->basic_type();
 978   int num_elem = vlen->get_con();
 979 
 980   // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
 981   if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
 982     if (C->print_intrinsics()) {
 983       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
 984                     is_store, is_store ? "store" : "load",
 985                     num_elem, type2name(elem_bt));
 986     }
 987     return false; // not supported
 988   }
 989 
 990   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
 991   bool is_mask = is_vector_mask(vbox_klass);
 992 
 993   Node* base = argument(3);
 994   Node* offset = ConvL2X(argument(4));
 995 
 996   // Save state and restore on bailout
 997   uint old_sp = sp();
 998   SafePointNode* old_map = clone_map();
 999 
1000   Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
1001 
1002   // The memory barrier checks are based on ones for unsafe access.
1003   // This is not 1-1 implementation.
1004   const Type *const base_type = gvn().type(base);
1005 
1006   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1007   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1008 
1009   const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
1010   const bool in_heap   = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
1011 
1012   const bool is_mixed_access = !in_heap && !in_native;
1013 
1014   const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == NULL);
1015 
1016   const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
1017 
1018   // Now handle special case where load/store happens from/to byte array but element type is not byte.
1019   bool using_byte_array = arr_type != NULL && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE;
1020   // Handle loading masks.
1021   // If there is no consistency between array and vector element types, it must be special byte array case or loading masks
1022   if (arr_type != NULL && !using_byte_array && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type)) {
1023     if (C->print_intrinsics()) {
1024       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1025                     is_store, is_store ? "store" : "load",
1026                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1027     }
1028     set_map(old_map);
1029     set_sp(old_sp);
1030     return false;
1031   }
1032   // Since we are using byte array, we need to double check that the byte operations are supported by backend.
1033   if (using_byte_array) {
1034     int byte_num_elem = num_elem * type2aelembytes(elem_bt);
1035     if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, byte_num_elem, T_BYTE, VecMaskNotUsed)
1036         || !arch_supports_vector(Op_VectorReinterpret, byte_num_elem, T_BYTE, VecMaskNotUsed)) {
1037       if (C->print_intrinsics()) {
1038         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
1039                       is_store, is_store ? "store" : "load",
1040                       byte_num_elem, type2name(elem_bt));
1041       }
1042       set_map(old_map);
1043       set_sp(old_sp);
1044       return false; // not supported
1045     }
1046   }
1047   if (is_mask) {










1048     if (!is_store) {
1049       if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1050         set_map(old_map);
1051         set_sp(old_sp);
1052         return false; // not supported
1053       }
1054     } else {
1055       if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
1056         set_map(old_map);
1057         set_sp(old_sp);
1058         return false; // not supported
1059       }
1060     }
1061   }
1062 
1063   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1064 
1065   if (needs_cpu_membar) {
1066     insert_mem_bar(Op_MemBarCPUOrder);
1067   }
1068 
1069   if (is_store) {
1070     Node* val = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1071     if (val == NULL) {
1072       set_map(old_map);
1073       set_sp(old_sp);
1074       return false; // operand unboxing failed
1075     }
1076     set_all_memory(reset_memory());
1077 
1078     // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
1079     int store_num_elem = num_elem;
1080     if (using_byte_array) {
1081       store_num_elem = num_elem * type2aelembytes(elem_bt);
1082       const TypeVect* to_vect_type = TypeVect::make(T_BYTE, store_num_elem);
1083       val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1084     }
1085     if (is_mask) {
1086       val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
1087     }
1088     Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
1089     set_memory(vstore, addr_type);
1090   } else {
1091     // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
1092     Node* vload = NULL;
1093     if (using_byte_array) {
1094       int load_num_elem = num_elem * type2aelembytes(elem_bt);
1095       vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, load_num_elem, T_BYTE));
1096       const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1097       vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1098     } else {
1099       // Special handle for masks
1100       if (is_mask) {
1101         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
1102         vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
1103       } else {
1104         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
1105       }
1106     }
1107     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1108     set_result(box);
1109   }
1110 
1111   old_map->destruct(&_gvn);
1112 
1113   if (needs_cpu_membar) {
1114     insert_mem_bar(Op_MemBarCPUOrder);
1115   }
1116 
1117   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1118   return true;
1119 }
1120 
1121 // public static
1122 // <C,
1123 //  V extends Vector<?>,
1124 //  E,
1125 //  S extends VectorSpecies<E>,
1126 //  M extends VectorMask<E>>
1127 // V loadMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1128 //              int length, Object base, long offset, M m,
1129 //              C container, long index, S s,  // Arguments for default implementation
1130 //              LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1131 //
1132 // public static
1133 // <C,
1134 //  V extends Vector<E>,
1135 //  M extends VectorMask<E>,
1136 //  E>
1137 // void storeMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1138 //                  int length, Object base, long offset,
1139 //                  V v, M m,
1140 //                  C container, long index,  // Arguments for default implementation
1141 //                  StoreVectorMaskedOperation<C, V, M, E> defaultImpl) {
1142 //
1143 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1144   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1145   const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1146   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1147   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1148 
1149   if (vector_klass == NULL || mask_klass == NULL || elem_klass == NULL || vlen == NULL ||
1150       vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
1151       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1152     if (C->print_intrinsics()) {
1153       tty->print_cr("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1154                     NodeClassNames[argument(0)->Opcode()],
1155                     NodeClassNames[argument(1)->Opcode()],
1156                     NodeClassNames[argument(2)->Opcode()],
1157                     NodeClassNames[argument(3)->Opcode()]);
1158     }
1159     return false; // not enough info for intrinsification
1160   }
1161   if (!is_klass_initialized(vector_klass)) {
1162     if (C->print_intrinsics()) {
1163       tty->print_cr("  ** klass argument not initialized");
1164     }
1165     return false;
1166   }
1167 
1168   if (!is_klass_initialized(mask_klass)) {
1169     if (C->print_intrinsics()) {
1170       tty->print_cr("  ** mask klass argument not initialized");
1171     }
1172     return false;
1173   }
1174 
1175   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1176   if (!elem_type->is_primitive_type()) {
1177     if (C->print_intrinsics()) {
1178       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1179     }
1180     return false; // should be primitive type
1181   }
1182 
1183   BasicType elem_bt = elem_type->basic_type();
1184   int num_elem = vlen->get_con();
1185 
1186   Node* base = argument(4);
1187   Node* offset = ConvL2X(argument(5));
1188 
1189   // Save state and restore on bailout
1190   uint old_sp = sp();
1191   SafePointNode* old_map = clone_map();
1192 
1193   Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1194   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1195   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1196 
1197   // Now handle special case where load/store happens from/to byte array but element type is not byte.
1198   bool using_byte_array = arr_type != NULL && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE;
1199   // If there is no consistency between array and vector element types, it must be special byte array case
1200   if (arr_type != NULL && !using_byte_array && !elem_consistent_with_arr(elem_bt, arr_type)) {
1201     if (C->print_intrinsics()) {
1202       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
1203                     is_store, is_store ? "storeMasked" : "loadMasked",
1204                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1205     }
1206     set_map(old_map);
1207     set_sp(old_sp);
1208     return false;
1209   }
1210 
1211   int mem_num_elem = using_byte_array ? num_elem * type2aelembytes(elem_bt) : num_elem;
1212   BasicType mem_elem_bt = using_byte_array ? T_BYTE : elem_bt;
1213   bool use_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
1214                                             mem_num_elem, mem_elem_bt,
1215                                             (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred));
1216   // Masked vector store operation needs the architecture predicate feature. We need to check
1217   // whether the predicated vector operation is supported by backend.
1218   if (is_store && !use_predicate) {
1219     if (C->print_intrinsics()) {
1220       tty->print_cr("  ** not supported: op=storeMasked vlen=%d etype=%s using_byte_array=%d",
1221                     num_elem, type2name(elem_bt), using_byte_array ? 1 : 0);
1222     }
1223     set_map(old_map);
1224     set_sp(old_sp);
1225     return false;
1226   }
1227 
1228   // This only happens for masked vector load. If predicate is not supported, then check whether
1229   // the normal vector load and blend operations are supported by backend.
1230   if (!use_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1231       !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1232     if (C->print_intrinsics()) {
1233       tty->print_cr("  ** not supported: op=loadMasked vlen=%d etype=%s using_byte_array=%d",
1234                     num_elem, type2name(elem_bt), using_byte_array ? 1 : 0);
1235     }
1236     set_map(old_map);
1237     set_sp(old_sp);
1238     return false;
1239   }
1240 
1241   // Since we are using byte array, we need to double check that the vector reinterpret operation
1242   // with byte type is supported by backend.
1243   if (using_byte_array) {
1244     if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1245       if (C->print_intrinsics()) {
1246         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s using_byte_array=1",
1247                       is_store, is_store ? "storeMasked" : "loadMasked",
1248                       num_elem, type2name(elem_bt));
1249       }
1250       set_map(old_map);
1251       set_sp(old_sp);
1252       return false;
1253     }
1254   }
1255 
1256   // Since it needs to unbox the mask, we need to double check that the related load operations
1257   // for mask are supported by backend.
1258   if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1259     if (C->print_intrinsics()) {
1260       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s",
1261                       is_store, is_store ? "storeMasked" : "loadMasked",
1262                       num_elem, type2name(elem_bt));
1263     }
1264     set_map(old_map);
1265     set_sp(old_sp);
1266     return false;
1267   }
1268 
1269   // Can base be NULL? Otherwise, always on-heap access.
1270   bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1271   if (can_access_non_heap) {
1272     insert_mem_bar(Op_MemBarCPUOrder);
1273   }
1274 
1275   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1276   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1277   assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1278   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1279   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1280 
1281   Node* mask = unbox_vector(is_store ? argument(8) : argument(7), mbox_type, elem_bt, num_elem);
1282   if (mask == NULL) {
1283     if (C->print_intrinsics()) {
1284       tty->print_cr("  ** unbox failed mask=%s",
1285                     is_store ? NodeClassNames[argument(8)->Opcode()]
1286                              : NodeClassNames[argument(7)->Opcode()]);
1287     }
1288     set_map(old_map);
1289     set_sp(old_sp);
1290     return false;
1291   }
1292 
1293   if (is_store) {
1294     Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
1295     if (val == NULL) {
1296       if (C->print_intrinsics()) {
1297         tty->print_cr("  ** unbox failed vector=%s",
1298                       NodeClassNames[argument(7)->Opcode()]);
1299       }
1300       set_map(old_map);
1301       set_sp(old_sp);
1302       return false; // operand unboxing failed
1303     }
1304     set_all_memory(reset_memory());
1305 
1306     if (using_byte_array) {
1307       // Reinterpret the incoming vector to byte vector.
1308       const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1309       val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1310       // Reinterpret the vector mask to byte type.
1311       const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1312       const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1313       mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1314     }
1315     Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1316     set_memory(vstore, addr_type);
1317   } else {
1318     Node* vload = NULL;
1319 
1320     if (using_byte_array) {
1321       // Reinterpret the vector mask to byte type.
1322       const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1323       const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1324       mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1325     }
1326 
1327     if (use_predicate) {
1328       // Generate masked load vector node if predicate feature is supported.
1329       const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1330       vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1331     } else {
1332       // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1333       Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1334       zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, Type::get_const_basic_type(mem_elem_bt)));
1335       vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1336       vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1337     }
1338 
1339     if (using_byte_array) {
1340       const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1341       vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1342     }
1343 
1344     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1345     set_result(box);
1346   }
1347 
1348   old_map->destruct(&_gvn);
1349 
1350   if (can_access_non_heap) {
1351     insert_mem_bar(Op_MemBarCPUOrder);
1352   }
1353 
1354   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1355   return true;
1356 }
1357 
1358 // <C,
1359 //  V extends Vector<?>,
1360 //  W extends Vector<Integer>,
1361 //  S extends VectorSpecies<E>,
1362 //  M extends VectorMask<E>,
1363 //  E>
1364 // V loadWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int length,
1365 //               Class<? extends Vector<Integer>> vectorIndexClass,
1366 //               Object base, long offset, // Unsafe addressing
1367 //               W index_vector, M m,
1368 //               C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
1369 //               LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1370 //
1371 //  <C,
1372 //   V extends Vector<E>,
1373 //   W extends Vector<Integer>,
1374 //   M extends VectorMask<E>,
1375 //   E>
1376 //  void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1377 //                    int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset,    // Unsafe addressing
1378 //                    W index_vector, V v, M m,
1379 //                    C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1380 //                    StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1381 //
1382 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1383   const TypeInstPtr* vector_klass     = gvn().type(argument(0))->isa_instptr();
1384   const TypeInstPtr* mask_klass       = gvn().type(argument(1))->isa_instptr();
1385   const TypeInstPtr* elem_klass       = gvn().type(argument(2))->isa_instptr();
1386   const TypeInt*     vlen             = gvn().type(argument(3))->isa_int();
1387   const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1388 
1389   if (vector_klass == NULL || elem_klass == NULL || vector_idx_klass == NULL || vlen == NULL ||
1390       vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || vector_idx_klass->const_oop() == NULL || !vlen->is_con()) {
1391     if (C->print_intrinsics()) {
1392       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1393                     NodeClassNames[argument(0)->Opcode()],
1394                     NodeClassNames[argument(2)->Opcode()],
1395                     NodeClassNames[argument(3)->Opcode()],
1396                     NodeClassNames[argument(4)->Opcode()]);
1397     }
1398     return false; // not enough info for intrinsification
1399   }
1400 
1401   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1402     if (C->print_intrinsics()) {
1403       tty->print_cr("  ** klass argument not initialized");
1404     }
1405     return false;
1406   }
1407 
1408   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1409   if (!elem_type->is_primitive_type()) {
1410     if (C->print_intrinsics()) {
1411       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1412     }
1413     return false; // should be primitive type
1414   }
1415 
1416   BasicType elem_bt = elem_type->basic_type();
1417   int num_elem = vlen->get_con();
1418 
1419   const Type* vmask_type = gvn().type(is_scatter ? argument(10) : argument(9));
1420   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1421   if (is_masked_op) {
1422     if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
1423       if (C->print_intrinsics()) {
1424         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1425       }
1426       return false; // not enough info for intrinsification
1427     }
1428 
1429     if (!is_klass_initialized(mask_klass)) {
1430       if (C->print_intrinsics()) {
1431         tty->print_cr("  ** mask klass argument not initialized");
1432       }
1433       return false;
1434     }
1435 
1436     if (vmask_type->maybe_null()) {
1437       if (C->print_intrinsics()) {
1438         tty->print_cr("  ** null mask values are not allowed for masked op");
1439       }
1440       return false;
1441     }
1442 
1443     // Check whether the predicated gather/scatter node is supported by architecture.
1444     if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt,
1445                               (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred))) {
1446       if (C->print_intrinsics()) {
1447         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1448                       is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1449                       num_elem, type2name(elem_bt));
1450       }
1451       return false; // not supported
1452     }
1453   } else {
1454     // Check whether the normal gather/scatter node is supported for non-masked operation.
1455     if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1456       if (C->print_intrinsics()) {
1457         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1458                       is_scatter, is_scatter ? "scatter" : "gather",
1459                       num_elem, type2name(elem_bt));
1460       }
1461       return false; // not supported
1462     }
1463   }
1464 
1465   // Check that the vector holding indices is supported by architecture
1466   if (!arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
1467       if (C->print_intrinsics()) {
1468         tty->print_cr("  ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1469                       is_scatter, is_scatter ? "scatter" : "gather",
1470                       num_elem, is_masked_op ? 1 : 0);
1471       }
1472       return false; // not supported
1473   }
1474 
1475   Node* base = argument(5);
1476   Node* offset = ConvL2X(argument(6));
1477 
1478   // Save state and restore on bailout
1479   uint old_sp = sp();
1480   SafePointNode* old_map = clone_map();
1481 
1482   Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1483 
1484   const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1485   const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1486 
1487   // The array must be consistent with vector type
1488   if (arr_type == NULL || (arr_type != NULL && !elem_consistent_with_arr(elem_bt, arr_type))) {
1489     if (C->print_intrinsics()) {
1490       tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1491                     is_scatter, is_scatter ? "scatter" : "gather",
1492                     num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1493     }
1494     set_map(old_map);
1495     set_sp(old_sp);
1496     return false;
1497   }
1498 
1499   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1500   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1501   ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1502   if (vbox_idx_klass == NULL) {
1503     set_map(old_map);
1504     set_sp(old_sp);
1505     return false;
1506   }
1507 
1508   const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1509   Node* index_vect = unbox_vector(argument(8), vbox_idx_type, T_INT, num_elem);
1510   if (index_vect == NULL) {
1511     set_map(old_map);
1512     set_sp(old_sp);
1513     return false;
1514   }
1515 
1516   Node* mask = NULL;
1517   if (is_masked_op) {
1518     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1519     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1520     mask = unbox_vector(is_scatter ? argument(10) : argument(9), mbox_type, elem_bt, num_elem);
1521     if (mask == NULL) {
1522       if (C->print_intrinsics()) {
1523         tty->print_cr("  ** unbox failed mask=%s",
1524                     is_scatter ? NodeClassNames[argument(10)->Opcode()]
1525                                : NodeClassNames[argument(9)->Opcode()]);
1526       }
1527       set_map(old_map);
1528       set_sp(old_sp);
1529       return false;
1530     }
1531   }
1532 
1533   const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1534   if (is_scatter) {
1535     Node* val = unbox_vector(argument(9), vbox_type, elem_bt, num_elem);
1536     if (val == NULL) {
1537       set_map(old_map);
1538       set_sp(old_sp);
1539       return false; // operand unboxing failed
1540     }
1541     set_all_memory(reset_memory());
1542 
1543     Node* vstore = NULL;
1544     if (mask != NULL) {
1545       vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, index_vect, mask));
1546     } else {
1547       vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1548     }
1549     set_memory(vstore, addr_type);
1550   } else {
1551     Node* vload = NULL;
1552     if (mask != NULL) {
1553       vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_vect, mask));
1554     } else {
1555       vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1556     }
1557     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1558     set_result(box);
1559   }
1560 
1561   old_map->destruct(&_gvn);
1562 
1563   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1564   return true;
1565 }
1566 
1567 // public static
1568 // <V extends Vector<E>,
1569 //  M extends VectorMask<E>,
1570 //  E>
1571 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1572 //                       Class<E> elementType, int length, V v, M m,
1573 //                       ReductionOperation<V, M> defaultImpl)
1574 bool LibraryCallKit::inline_vector_reduction() {
1575   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
1576   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1577   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
1578   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
1579   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
1580 
1581   if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
1582       !opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1583     if (C->print_intrinsics()) {
1584       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1585                     NodeClassNames[argument(0)->Opcode()],
1586                     NodeClassNames[argument(1)->Opcode()],
1587                     NodeClassNames[argument(3)->Opcode()],
1588                     NodeClassNames[argument(4)->Opcode()]);
1589     }
1590     return false; // not enough info for intrinsification
1591   }
1592   if (!is_klass_initialized(vector_klass)) {
1593     if (C->print_intrinsics()) {
1594       tty->print_cr("  ** klass argument not initialized");
1595     }
1596     return false;
1597   }
1598   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1599   if (!elem_type->is_primitive_type()) {
1600     if (C->print_intrinsics()) {
1601       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1602     }
1603     return false; // should be primitive type
1604   }
1605 
1606   const Type* vmask_type = gvn().type(argument(6));
1607   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1608   if (is_masked_op) {
1609     if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
1610       if (C->print_intrinsics()) {
1611         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1612       }
1613       return false; // not enough info for intrinsification
1614     }
1615 
1616     if (!is_klass_initialized(mask_klass)) {
1617       if (C->print_intrinsics()) {
1618         tty->print_cr("  ** mask klass argument not initialized");
1619       }
1620       return false;
1621     }
1622 
1623     if (vmask_type->maybe_null()) {
1624       if (C->print_intrinsics()) {
1625         tty->print_cr("  ** null mask values are not allowed for masked op");
1626       }
1627       return false;
1628     }
1629   }
1630 
1631   BasicType elem_bt = elem_type->basic_type();
1632   int num_elem = vlen->get_con();
1633   int opc  = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1634   int sopc = ReductionNode::opcode(opc, elem_bt);
1635 
1636   // When using mask, mask use type needs to be VecMaskUseLoad.
1637   if (!arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1638     if (C->print_intrinsics()) {
1639       tty->print_cr("  ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1640                     sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1641     }
1642     return false;
1643   }
1644 
1645   // Return true if current platform has implemented the masked operation with predicate feature.
1646   bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1647   if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1648     if (C->print_intrinsics()) {
1649       tty->print_cr("  ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1650                     sopc, num_elem, type2name(elem_bt));
1651     }
1652     return false;
1653   }
1654 
1655   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1656   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1657 
1658   Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1659   if (opd == NULL) {
1660     return false; // operand unboxing failed
1661   }
1662 
1663   Node* mask = NULL;
1664   if (is_masked_op) {
1665     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1666     assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1667     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1668     mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1669     if (mask == NULL) {
1670       if (C->print_intrinsics()) {
1671         tty->print_cr("  ** unbox failed mask=%s",
1672                       NodeClassNames[argument(6)->Opcode()]);
1673       }
1674       return false;
1675     }
1676   }
1677 
1678   Node* init = ReductionNode::make_reduction_input(gvn(), opc, elem_bt);
1679   Node* value = NULL;
1680   if (mask == NULL) {
1681     assert(!is_masked_op, "Masked op needs the mask value never null");
1682     value = ReductionNode::make(opc, NULL, init, opd, elem_bt);
1683   } else {
1684     if (use_predicate) {
1685       value = ReductionNode::make(opc, NULL, init, opd, elem_bt);
1686       value->add_req(mask);
1687       value->add_flag(Node::Flag_is_predicated_vector);
1688     } else {
1689       Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, Type::get_const_basic_type(elem_bt)));
1690       value = gvn().transform(new VectorBlendNode(reduce_identity, opd, mask));
1691       value = ReductionNode::make(opc, NULL, init, value, elem_bt);
1692     }
1693   }
1694   value = gvn().transform(value);
1695 
1696   Node* bits = NULL;
1697   switch (elem_bt) {
1698     case T_BYTE:
1699     case T_SHORT:
1700     case T_INT: {
1701       bits = gvn().transform(new ConvI2LNode(value));
1702       break;
1703     }
1704     case T_FLOAT: {
1705       value = gvn().transform(new MoveF2INode(value));
1706       bits  = gvn().transform(new ConvI2LNode(value));
1707       break;
1708     }
1709     case T_DOUBLE: {
1710       bits = gvn().transform(new MoveD2LNode(value));
1711       break;
1712     }
1713     case T_LONG: {
1714       bits = value; // no conversion needed
1715       break;
1716     }
1717     default: fatal("%s", type2name(elem_bt));
1718   }
1719   set_result(bits);
1720   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1721   return true;
1722 }
1723 
1724 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1725 //                                V v1, V v2,
1726 //                                BiFunction<V, V, Boolean> defaultImpl)
1727 //
1728 bool LibraryCallKit::inline_vector_test() {
1729   const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1730   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1731   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1732   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1733 
1734   if (cond == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
1735       !cond->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1736     if (C->print_intrinsics()) {
1737       tty->print_cr("  ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1738                     NodeClassNames[argument(0)->Opcode()],
1739                     NodeClassNames[argument(1)->Opcode()],
1740                     NodeClassNames[argument(2)->Opcode()],
1741                     NodeClassNames[argument(3)->Opcode()]);
1742     }
1743     return false; // not enough info for intrinsification
1744   }
1745   if (!is_klass_initialized(vector_klass)) {
1746     if (C->print_intrinsics()) {
1747       tty->print_cr("  ** klass argument not initialized");
1748     }
1749     return false;
1750   }
1751   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1752   if (!elem_type->is_primitive_type()) {
1753     if (C->print_intrinsics()) {
1754       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1755     }
1756     return false; // should be primitive type
1757   }
1758   BasicType elem_bt = elem_type->basic_type();
1759   int num_elem = vlen->get_con();
1760   BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1761   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1762   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1763 
1764   if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1765     if (C->print_intrinsics()) {
1766       tty->print_cr("  ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1767                     cond->get_con(), num_elem, type2name(elem_bt),
1768                     is_vector_mask(vbox_klass));
1769     }
1770     return false;
1771   }
1772 
1773   Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1774   Node* opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1775   if (opd1 == NULL || opd2 == NULL) {
1776     return false; // operand unboxing failed
1777   }
1778   Node* test = new VectorTestNode(opd1, opd2, booltest);
1779   test = gvn().transform(test);
1780 
1781   set_result(test);
1782   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1783   return true;
1784 }
1785 
1786 // public static
1787 // <V extends Vector<E>,
1788 //  M extends VectorMask<E>,
1789 //  E>
1790 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1791 //         V v1, V v2, M m,
1792 //         VectorBlendOp<V, M, E> defaultImpl)
1793 bool LibraryCallKit::inline_vector_blend() {
1794   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1795   const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1796   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1797   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1798 
1799   if (mask_klass == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL) {
1800     return false; // dead code
1801   }
1802   if (mask_klass->const_oop() == NULL || vector_klass->const_oop() == NULL ||
1803       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1804     if (C->print_intrinsics()) {
1805       tty->print_cr("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1806                     NodeClassNames[argument(0)->Opcode()],
1807                     NodeClassNames[argument(1)->Opcode()],
1808                     NodeClassNames[argument(2)->Opcode()],
1809                     NodeClassNames[argument(3)->Opcode()]);
1810     }
1811     return false; // not enough info for intrinsification
1812   }
1813   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1814     if (C->print_intrinsics()) {
1815       tty->print_cr("  ** klass argument not initialized");
1816     }
1817     return false;
1818   }
1819   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1820   if (!elem_type->is_primitive_type()) {
1821     if (C->print_intrinsics()) {
1822       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1823     }
1824     return false; // should be primitive type
1825   }
1826   BasicType elem_bt = elem_type->basic_type();
1827   BasicType mask_bt = elem_bt;
1828   int num_elem = vlen->get_con();
1829 
1830   if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1831     if (C->print_intrinsics()) {
1832       tty->print_cr("  ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1833                     num_elem, type2name(elem_bt));
1834     }
1835     return false; // not supported
1836   }
1837   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1838   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1839 
1840   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1841   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1842 
1843   Node* v1   = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1844   Node* v2   = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1845   Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1846 
1847   if (v1 == NULL || v2 == NULL || mask == NULL) {
1848     return false; // operand unboxing failed
1849   }
1850 
1851   Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1852 
1853   Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1854   set_result(box);
1855   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1856   return true;
1857 }
1858 
1859 //  public static
1860 //  <V extends Vector<E>,
1861 //   M extends VectorMask<E>,
1862 //   E>
1863 //  M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1864 //            V v1, V v2, M m,
1865 //            VectorCompareOp<V,M> defaultImpl)
1866 bool LibraryCallKit::inline_vector_compare() {
1867   const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1868   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1869   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
1870   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
1871   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
1872 
1873   if (cond == NULL || vector_klass == NULL || mask_klass == NULL || elem_klass == NULL || vlen == NULL) {
1874     return false; // dead code
1875   }
1876   if (!cond->is_con() || vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
1877       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1878     if (C->print_intrinsics()) {
1879       tty->print_cr("  ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1880                     NodeClassNames[argument(0)->Opcode()],
1881                     NodeClassNames[argument(1)->Opcode()],
1882                     NodeClassNames[argument(2)->Opcode()],
1883                     NodeClassNames[argument(3)->Opcode()],
1884                     NodeClassNames[argument(4)->Opcode()]);
1885     }
1886     return false; // not enough info for intrinsification
1887   }
1888   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1889     if (C->print_intrinsics()) {
1890       tty->print_cr("  ** klass argument not initialized");
1891     }
1892     return false;
1893   }
1894   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1895   if (!elem_type->is_primitive_type()) {
1896     if (C->print_intrinsics()) {
1897       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1898     }
1899     return false; // should be primitive type
1900   }
1901 
1902   int num_elem = vlen->get_con();
1903   BasicType elem_bt = elem_type->basic_type();
1904   BasicType mask_bt = elem_bt;
1905 
1906   if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1907     if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1908       if (C->print_intrinsics()) {
1909         tty->print_cr("  ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1910                       cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1911       }
1912       return false;
1913     }
1914   }
1915 
1916   if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1917     if (C->print_intrinsics()) {
1918       tty->print_cr("  ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1919                     cond->get_con(), num_elem, type2name(elem_bt));
1920     }
1921     return false;
1922   }
1923 
1924   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1925   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1926 
1927   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1928   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1929 
1930   Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1931   Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1932 
1933   bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1934   Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : NULL;
1935   if (is_masked_op && mask == NULL) {
1936     if (C->print_intrinsics()) {
1937       tty->print_cr("  ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1938                     cond->get_con(), num_elem, type2name(elem_bt));
1939     }
1940     return false;
1941   }
1942 
1943   bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1944   if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1945     if (C->print_intrinsics()) {
1946       tty->print_cr("  ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1947                     cond->get_con(), num_elem, type2name(elem_bt));
1948     }
1949     return false;
1950   }
1951 
1952   if (v1 == NULL || v2 == NULL) {
1953     return false; // operand unboxing failed
1954   }
1955   BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1956   ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1957 
1958   const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1959   Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1960 
1961   if (is_masked_op) {
1962     if (use_predicate) {
1963       operation->add_req(mask);
1964       operation->add_flag(Node::Flag_is_predicated_vector);
1965     } else {
1966       operation = gvn().transform(operation);
1967       operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1968     }
1969   }
1970 
1971   operation = gvn().transform(operation);
1972 
1973   Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1974   set_result(box);
1975   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1976   return true;
1977 }
1978 
1979 // public static
1980 // <V extends Vector<E>,
1981 //  Sh extends VectorShuffle<E>,
1982 //  M extends VectorMask<E>,
1983 //  E>
1984 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1985 //               V v1, Sh sh, M m,
1986 //               VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1987 bool LibraryCallKit::inline_vector_rearrange() {
1988   const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
1989   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1990   const TypeInstPtr* mask_klass    = gvn().type(argument(2))->isa_instptr();
1991   const TypeInstPtr* elem_klass    = gvn().type(argument(3))->isa_instptr();
1992   const TypeInt*     vlen          = gvn().type(argument(4))->isa_int();
1993 
1994   if (vector_klass == NULL  || shuffle_klass == NULL ||  elem_klass == NULL || vlen == NULL) {
1995     return false; // dead code
1996   }
1997   if (shuffle_klass->const_oop() == NULL ||
1998       vector_klass->const_oop()  == NULL ||
1999       elem_klass->const_oop()    == NULL ||
2000       !vlen->is_con()) {
2001     if (C->print_intrinsics()) {
2002       tty->print_cr("  ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
2003                     NodeClassNames[argument(0)->Opcode()],
2004                     NodeClassNames[argument(1)->Opcode()],
2005                     NodeClassNames[argument(3)->Opcode()],
2006                     NodeClassNames[argument(4)->Opcode()]);
2007     }
2008     return false; // not enough info for intrinsification
2009   }
2010   if (!is_klass_initialized(vector_klass)  ||
2011       !is_klass_initialized(shuffle_klass)) {
2012     if (C->print_intrinsics()) {
2013       tty->print_cr("  ** klass argument not initialized");
2014     }
2015     return false;
2016   }
2017   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2018   if (!elem_type->is_primitive_type()) {
2019     if (C->print_intrinsics()) {
2020       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2021     }
2022     return false; // should be primitive type
2023   }
2024   BasicType elem_bt = elem_type->basic_type();
2025   BasicType shuffle_bt = elem_bt;
2026   int num_elem = vlen->get_con();
2027 
2028   if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
2029     if (C->print_intrinsics()) {
2030       tty->print_cr("  ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
2031                     num_elem, type2name(elem_bt));
2032     }
2033     return false; // not supported
2034   }
2035 
2036   bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
2037   bool use_predicate = is_masked_op;
2038   if (is_masked_op &&
2039       (mask_klass == NULL ||
2040        mask_klass->const_oop() == NULL ||
2041        !is_klass_initialized(mask_klass))) {
2042     if (C->print_intrinsics()) {
2043       tty->print_cr("  ** mask_klass argument not initialized");
2044     }
2045   }
2046   VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2047   if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
2048     use_predicate = false;
2049     if(!is_masked_op ||
2050        (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2051         !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)     ||
2052         !arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed))) {
2053       if (C->print_intrinsics()) {
2054         tty->print_cr("  ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
2055                       num_elem, type2name(elem_bt));
2056       }
2057       return false; // not supported
2058     }
2059   }
2060   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2061   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2062 
2063   ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
2064   const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
2065 
2066   Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2067   Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
2068 
2069   if (v1 == NULL || shuffle == NULL) {
2070     return false; // operand unboxing failed
2071   }
2072 
2073   Node* mask = NULL;
2074   if (is_masked_op) {
2075     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2076     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2077     mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2078     if (mask == NULL) {
2079       if (C->print_intrinsics()) {
2080         tty->print_cr("  ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
2081                       num_elem, type2name(elem_bt));
2082       }
2083       return false;
2084     }
2085   }
2086 
2087   Node* rearrange = new VectorRearrangeNode(v1, shuffle);
2088   if (is_masked_op) {
2089     if (use_predicate) {
2090       rearrange->add_req(mask);
2091       rearrange->add_flag(Node::Flag_is_predicated_vector);
2092     } else {
2093       const TypeVect* vt = v1->bottom_type()->is_vect();
2094       rearrange = gvn().transform(rearrange);
2095       Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2096       Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, Type::get_const_basic_type(elem_bt)));
2097       rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2098     }
2099   }
2100   rearrange = gvn().transform(rearrange);
2101 
2102   Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2103   set_result(box);
2104   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2105   return true;
2106 }
2107 
2108 static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
2109   address addr = NULL;
2110   assert(UseVectorStubs, "sanity");
2111   assert(name_ptr != NULL, "unexpected");
2112   assert((vop >= VectorSupport::VECTOR_OP_SVML_START) && (vop <= VectorSupport::VECTOR_OP_SVML_END), "unexpected");
2113   int op = vop - VectorSupport::VECTOR_OP_SVML_START;
2114 
2115   switch(bits) {
2116     case 64:  //fallthough
2117     case 128: //fallthough
2118     case 256: //fallthough
2119     case 512:
2120       if (bt == T_FLOAT) {
2121         snprintf(name_ptr, name_len, "vector_%s_float%d", VectorSupport::svmlname[op], bits);
2122         addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op];
2123       } else {
2124         assert(bt == T_DOUBLE, "must be FP type only");
2125         snprintf(name_ptr, name_len, "vector_%s_double%d", VectorSupport::svmlname[op], bits);
2126         addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op];
2127       }
2128       break;
2129     default:
2130       snprintf(name_ptr, name_len, "invalid");
2131       addr = NULL;
2132       Unimplemented();
2133       break;
2134   }
2135 
2136   return addr;
2137 }
2138 
2139 Node* LibraryCallKit::gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
2140   assert(UseVectorStubs, "sanity");
2141   assert(vector_api_op_id >= VectorSupport::VECTOR_OP_SVML_START && vector_api_op_id <= VectorSupport::VECTOR_OP_SVML_END, "need valid op id");
2142   assert(opd1 != NULL, "must not be null");
2143   const TypeVect* vt = TypeVect::make(bt, num_elem);
2144   const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != NULL ? 2 : 1, vt, vt);
2145   char name[100] = "";
2146 
2147   // Get address for svml method.
2148   address addr = get_svml_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
2149 
2150   if (addr == NULL) {
2151     return NULL;
2152   }
2153 
2154   assert(name != NULL, "name must not be null");
2155   Node* operation = make_runtime_call(RC_VECTOR,
2156                                       call_type,
2157                                       addr,
2158                                       name,
2159                                       TypePtr::BOTTOM,
2160                                       opd1,
2161                                       opd2);
2162   return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
2163 }
2164 
2165 //  public static
2166 //  <V extends Vector<E>,
2167 //   M extends VectorMask<E>,
2168 //   E>
2169 //  V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2170 //                 Class<E> elementType, int length,
2171 //                 V v, int n, M m,
2172 //                 VectorBroadcastIntOp<V, M> defaultImpl)
2173 bool LibraryCallKit::inline_vector_broadcast_int() {
2174   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
2175   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2176   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
2177   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
2178   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
2179 
2180   if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL) {
2181     return false; // dead code
2182   }
2183   if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
2184     if (C->print_intrinsics()) {
2185       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2186                     NodeClassNames[argument(0)->Opcode()],
2187                     NodeClassNames[argument(1)->Opcode()],
2188                     NodeClassNames[argument(3)->Opcode()],
2189                     NodeClassNames[argument(4)->Opcode()]);
2190     }
2191     return false; // not enough info for intrinsification
2192   }
2193   if (!is_klass_initialized(vector_klass)) {
2194     if (C->print_intrinsics()) {
2195       tty->print_cr("  ** klass argument not initialized");
2196     }
2197     return false;
2198   }
2199 
2200   const Type* vmask_type = gvn().type(argument(7));
2201   bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2202   if (is_masked_op) {
2203     if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
2204       if (C->print_intrinsics()) {
2205         tty->print_cr("  ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2206       }
2207       return false; // not enough info for intrinsification
2208     }
2209 
2210     if (!is_klass_initialized(mask_klass)) {
2211       if (C->print_intrinsics()) {
2212         tty->print_cr("  ** mask klass argument not initialized");
2213       }
2214       return false;
2215     }
2216 
2217     if (vmask_type->maybe_null()) {
2218       if (C->print_intrinsics()) {
2219         tty->print_cr("  ** null mask values are not allowed for masked op");
2220       }
2221       return false;
2222     }
2223   }
2224 
2225   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2226   if (!elem_type->is_primitive_type()) {
2227     if (C->print_intrinsics()) {
2228       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2229     }
2230     return false; // should be primitive type
2231   }
2232 
2233   int num_elem = vlen->get_con();
2234   BasicType elem_bt = elem_type->basic_type();
2235   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2236 
2237   bool is_shift  = VectorNode::is_shift_opcode(opc);
2238   bool is_rotate = VectorNode::is_rotate_opcode(opc);
2239 
2240   if (opc == 0 || (!is_shift && !is_rotate)) {
2241     if (C->print_intrinsics()) {
2242       tty->print_cr("  ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2243     }
2244     return false; // operation not supported
2245   }
2246 
2247   int sopc = VectorNode::opcode(opc, elem_bt);
2248   if (sopc == 0) {
2249     if (C->print_intrinsics()) {
2250       tty->print_cr("  ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2251     }
2252     return false; // operation not supported
2253   }
2254 
2255   Node* cnt  = argument(6);
2256   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2257   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2258   const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2259 
2260   // If CPU supports vector constant rotate instructions pass it directly
2261   bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2262                          Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2263   bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2264 
2265   VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2266   bool use_predicate = is_masked_op;
2267 
2268   if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2269     use_predicate = false;
2270     if (!is_masked_op ||
2271         (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2272          !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2273 
2274       if (C->print_intrinsics()) {
2275         tty->print_cr("  ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2276                       sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2277       }
2278       return false; // not supported
2279     }
2280   }
2281 
2282   Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2283   Node* opd2 = NULL;
2284   if (is_shift) {
2285     opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2286   } else {
2287     assert(is_rotate, "unexpected operation");
2288     if (!is_const_rotate) {
2289       const Type * type_bt = Type::get_const_basic_type(elem_bt);
2290       cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2291       opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, type_bt));
2292     } else {
2293       // Constant shift value.
2294       opd2 = cnt;
2295     }
2296   }
2297 
2298   if (opd1 == NULL || opd2 == NULL) {
2299     return false;
2300   }
2301 
2302   Node* mask = NULL;
2303   if (is_masked_op) {
2304     ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2305     const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2306     mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2307     if (mask == NULL) {
2308       if (C->print_intrinsics()) {
2309         tty->print_cr("  ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2310       }
2311       return false;
2312     }
2313   }
2314 
2315   Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2316   if (is_masked_op && mask != NULL) {
2317     if (use_predicate) {
2318       operation->add_req(mask);
2319       operation->add_flag(Node::Flag_is_predicated_vector);
2320     } else {
2321       operation = gvn().transform(operation);
2322       operation = new VectorBlendNode(opd1, operation, mask);
2323     }
2324   }
2325   operation = gvn().transform(operation);
2326   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2327   set_result(vbox);
2328   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2329   return true;
2330 }
2331 
2332 // public static <VOUT extends VectorPayload,
2333 //                 VIN extends VectorPayload,
2334 //                   S extends VectorSpecies>
2335 // VOUT convert(int oprId,
2336 //           Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2337 //           Class<?>   toVectorClass, Class<?>   toElementType, int   toVLen,
2338 //           VIN v, S s,
2339 //           VectorConvertOp<VOUT, VIN, S> defaultImpl)
2340 //
2341 bool LibraryCallKit::inline_vector_convert() {
2342   const TypeInt*     opr               = gvn().type(argument(0))->isa_int();
2343 
2344   const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2345   const TypeInstPtr* elem_klass_from   = gvn().type(argument(2))->isa_instptr();
2346   const TypeInt*     vlen_from         = gvn().type(argument(3))->isa_int();
2347 
2348   const TypeInstPtr* vector_klass_to   = gvn().type(argument(4))->isa_instptr();
2349   const TypeInstPtr* elem_klass_to     = gvn().type(argument(5))->isa_instptr();
2350   const TypeInt*     vlen_to           = gvn().type(argument(6))->isa_int();
2351 
2352   if (opr == NULL ||
2353       vector_klass_from == NULL || elem_klass_from == NULL || vlen_from == NULL ||
2354       vector_klass_to   == NULL || elem_klass_to   == NULL || vlen_to   == NULL) {
2355     return false; // dead code
2356   }
2357   if (!opr->is_con() ||
2358       vector_klass_from->const_oop() == NULL || elem_klass_from->const_oop() == NULL || !vlen_from->is_con() ||
2359       vector_klass_to->const_oop() == NULL || elem_klass_to->const_oop() == NULL || !vlen_to->is_con()) {
2360     if (C->print_intrinsics()) {
2361       tty->print_cr("  ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2362                     NodeClassNames[argument(0)->Opcode()],
2363                     NodeClassNames[argument(1)->Opcode()],
2364                     NodeClassNames[argument(2)->Opcode()],
2365                     NodeClassNames[argument(3)->Opcode()],
2366                     NodeClassNames[argument(4)->Opcode()],
2367                     NodeClassNames[argument(5)->Opcode()],
2368                     NodeClassNames[argument(6)->Opcode()]);
2369     }
2370     return false; // not enough info for intrinsification
2371   }
2372   if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2373     if (C->print_intrinsics()) {
2374       tty->print_cr("  ** klass argument not initialized");
2375     }
2376     return false;
2377   }
2378 
2379   assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST  ||
2380          opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
2381          opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2382   bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2383   bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2384 
2385   ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2386   ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2387   if (is_vector_shuffle(vbox_klass_from)) {
2388     return false; // vector shuffles aren't supported
2389   }
2390   bool is_mask = is_vector_mask(vbox_klass_from);
2391 
2392   ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2393   if (!elem_type_from->is_primitive_type()) {
2394     return false; // should be primitive type
2395   }
2396   BasicType elem_bt_from = elem_type_from->basic_type();
2397   ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2398   if (!elem_type_to->is_primitive_type()) {
2399     return false; // should be primitive type
2400   }
2401   BasicType elem_bt_to = elem_type_to->basic_type();
2402 
2403   int num_elem_from = vlen_from->get_con();
2404   int num_elem_to = vlen_to->get_con();
2405 
2406   // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2407   // since we may need to change size.
2408   if (!arch_supports_vector(Op_VectorReinterpret,
2409                             num_elem_from,
2410                             elem_bt_from,
2411                             is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2412     if (C->print_intrinsics()) {
2413       tty->print_cr("  ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2414                     is_cast ? "cast" : "reinterpret",
2415                     num_elem_from, type2name(elem_bt_from), is_mask);
2416     }
2417     return false;
2418   }
2419 
2420   // Check whether we can support resizing/reinterpreting to the new size.
2421   if (!arch_supports_vector(Op_VectorReinterpret,
2422                             num_elem_to,
2423                             elem_bt_to,
2424                             is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2425     if (C->print_intrinsics()) {
2426       tty->print_cr("  ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2427                     is_cast ? "cast" : "reinterpret",
2428                     num_elem_to, type2name(elem_bt_to), is_mask);
2429     }
2430     return false;
2431   }
2432 
2433   // At this point, we know that both input and output vector registers are supported
2434   // by the architecture. Next check if the casted type is simply to same type - which means
2435   // that it is actually a resize and not a cast.
2436   if (is_cast && elem_bt_from == elem_bt_to) {
2437     is_cast = false;
2438   }
2439 
2440   const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2441 
2442   Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2443   if (opd1 == NULL) {
2444     return false;
2445   }
2446 
2447   const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2448   const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2449 
2450   // Safety check to prevent casting if source mask is of type vector
2451   // and destination mask of type predicate vector and vice-versa.
2452   // From X86 standpoint, this case will only arise over KNL target,
2453   // where certain masks (depending on the species) are either propagated
2454   // through a vector or predicate register.
2455   if (is_mask &&
2456       ((src_type->isa_vectmask() == NULL && dst_type->isa_vectmask()) ||
2457        (dst_type->isa_vectmask() == NULL && src_type->isa_vectmask()))) {
2458     return false;
2459   }
2460 
2461   Node* op = opd1;
2462   if (is_cast) {
2463     BasicType new_elem_bt_to = elem_bt_to;
2464     BasicType new_elem_bt_from = elem_bt_from;
2465     if (is_mask && is_floating_point_type(elem_bt_from)) {
2466       new_elem_bt_from = elem_bt_from == T_FLOAT ? T_INT : T_LONG;
2467     }
2468     int cast_vopc = VectorCastNode::opcode(new_elem_bt_from, !is_ucast);
2469     // Make sure that cast is implemented to particular type/size combination.
2470     if (!arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2471       if (C->print_intrinsics()) {
2472         tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2473                       cast_vopc,
2474                       num_elem_to, type2name(elem_bt_to), is_mask);
2475       }
2476       return false;
2477     }
2478 
2479     if (num_elem_from < num_elem_to) {
2480       // Since input and output number of elements are not consistent, we need to make sure we
2481       // properly size. Thus, first make a cast that retains the number of elements from source.
2482       int num_elem_for_cast = num_elem_from;
2483 
2484       // It is possible that arch does not support this intermediate vector size
2485       // TODO More complex logic required here to handle this corner case for the sizes.
2486       if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2487         if (C->print_intrinsics()) {
2488           tty->print_cr("  ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2489                         cast_vopc,
2490                         num_elem_for_cast, type2name(elem_bt_to), is_mask);
2491         }
2492         return false;
2493       }
2494 
2495       op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2496       // Now ensure that the destination gets properly resized to needed size.
2497       op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2498     } else if (num_elem_from > num_elem_to) {
2499       // Since number elements from input is larger than output, simply reduce size of input (we are supposed to
2500       // drop top elements anyway).
2501       int num_elem_for_resize = num_elem_to;
2502 
2503       // It is possible that arch does not support this intermediate vector size
2504       // TODO More complex logic required here to handle this corner case for the sizes.
2505       if (!arch_supports_vector(Op_VectorReinterpret,
2506                                 num_elem_for_resize,
2507                                 elem_bt_from,
2508                                 VecMaskNotUsed)) {
2509         if (C->print_intrinsics()) {
2510           tty->print_cr("  ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2511                         num_elem_for_resize, type2name(elem_bt_from), is_mask);
2512         }
2513         return false;
2514       }
2515 
2516       op = gvn().transform(new VectorReinterpretNode(op,
2517                                                      src_type,
2518                                                      TypeVect::make(elem_bt_from,
2519                                                                     num_elem_for_resize)));
2520       op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2521     } else {
2522       if (is_mask) {
2523         if ((dst_type->isa_vectmask() && src_type->isa_vectmask()) ||
2524             (type2aelembytes(elem_bt_from) == type2aelembytes(elem_bt_to))) {
2525           op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2526         } else {
2527           op = VectorMaskCastNode::makeCastNode(&gvn(), op, dst_type);
2528         }
2529       } else {
2530         // Since input and output number of elements match, and since we know this vector size is
2531         // supported, simply do a cast with no resize needed.
2532         op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2533       }
2534     }
2535   } else if (Type::cmp(src_type, dst_type) != 0) {
2536     assert(!is_cast, "must be reinterpret");
2537     op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2538   }
2539 
2540   const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2541   Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2542   set_result(vbox);
2543   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2544   return true;
2545 }
2546 
2547 //  public static
2548 //  <V extends Vector<E>,
2549 //   E>
2550 //  V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2551 //           V vec, int ix, long val,
2552 //           VecInsertOp<V> defaultImpl)
2553 bool LibraryCallKit::inline_vector_insert() {
2554   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2555   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2556   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2557   const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
2558 
2559   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || idx == NULL) {
2560     return false; // dead code
2561   }
2562   if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
2563     if (C->print_intrinsics()) {
2564       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2565                     NodeClassNames[argument(0)->Opcode()],
2566                     NodeClassNames[argument(1)->Opcode()],
2567                     NodeClassNames[argument(2)->Opcode()],
2568                     NodeClassNames[argument(4)->Opcode()]);
2569     }
2570     return false; // not enough info for intrinsification
2571   }
2572   if (!is_klass_initialized(vector_klass)) {
2573     if (C->print_intrinsics()) {
2574       tty->print_cr("  ** klass argument not initialized");
2575     }
2576     return false;
2577   }
2578   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2579   if (!elem_type->is_primitive_type()) {
2580     if (C->print_intrinsics()) {
2581       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2582     }
2583     return false; // should be primitive type
2584   }
2585   BasicType elem_bt = elem_type->basic_type();
2586   int num_elem = vlen->get_con();
2587   if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2588     if (C->print_intrinsics()) {
2589       tty->print_cr("  ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2590                     num_elem, type2name(elem_bt));
2591     }
2592     return false; // not supported
2593   }
2594 
2595   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2596   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2597 
2598   Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2599   if (opd == NULL) {
2600     return false;
2601   }
2602 
2603   Node* insert_val = argument(5);
2604   assert(gvn().type(insert_val)->isa_long() != NULL, "expected to be long");
2605 
2606   // Convert insert value back to its appropriate type.
2607   switch (elem_bt) {
2608     case T_BYTE:
2609       insert_val = gvn().transform(new ConvL2INode(insert_val));
2610       insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::BYTE));
2611       break;
2612     case T_SHORT:
2613       insert_val = gvn().transform(new ConvL2INode(insert_val));
2614       insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::SHORT));
2615       break;
2616     case T_INT:
2617       insert_val = gvn().transform(new ConvL2INode(insert_val));
2618       break;
2619     case T_FLOAT:
2620       insert_val = gvn().transform(new ConvL2INode(insert_val));
2621       insert_val = gvn().transform(new MoveI2FNode(insert_val));
2622       break;
2623     case T_DOUBLE:
2624       insert_val = gvn().transform(new MoveL2DNode(insert_val));
2625       break;
2626     case T_LONG:
2627       // no conversion needed
2628       break;
2629     default: fatal("%s", type2name(elem_bt)); break;
2630   }
2631 
2632   Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con()));
2633 
2634   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2635   set_result(vbox);
2636   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2637   return true;
2638 }
2639 
2640 //  public static
2641 //  <V extends Vector<E>,
2642 //   E>
2643 //  long extract(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2644 //               V vec, int ix,
2645 //               VecExtractOp<V> defaultImpl)
2646 bool LibraryCallKit::inline_vector_extract() {
2647   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2648   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2649   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2650   const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
2651 
2652   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || idx == NULL) {
2653     return false; // dead code
2654   }
2655   if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
2656     if (C->print_intrinsics()) {
2657       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2658                     NodeClassNames[argument(0)->Opcode()],
2659                     NodeClassNames[argument(1)->Opcode()],
2660                     NodeClassNames[argument(2)->Opcode()],
2661                     NodeClassNames[argument(4)->Opcode()]);
2662     }
2663     return false; // not enough info for intrinsification
2664   }
2665   if (!is_klass_initialized(vector_klass)) {
2666     if (C->print_intrinsics()) {
2667       tty->print_cr("  ** klass argument not initialized");
2668     }
2669     return false;
2670   }
2671   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2672   if (!elem_type->is_primitive_type()) {
2673     if (C->print_intrinsics()) {
2674       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2675     }
2676     return false; // should be primitive type
2677   }
2678   BasicType elem_bt = elem_type->basic_type();
2679   int num_elem = vlen->get_con();
2680   int vopc = ExtractNode::opcode(elem_bt);
2681   if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2682     if (C->print_intrinsics()) {
2683       tty->print_cr("  ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2684                     num_elem, type2name(elem_bt));
2685     }
2686     return false; // not supported
2687   }
2688 
2689   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2690   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2691 
2692   Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2693   if (opd == NULL) {
2694     return false;
2695   }
2696 
2697   Node* operation = gvn().transform(ExtractNode::make(opd, idx->get_con(), elem_bt));
2698 
2699   Node* bits = NULL;
2700   switch (elem_bt) {
2701     case T_BYTE:
2702     case T_SHORT:
2703     case T_INT: {
2704       bits = gvn().transform(new ConvI2LNode(operation));
2705       break;
2706     }
2707     case T_FLOAT: {
2708       bits = gvn().transform(new MoveF2INode(operation));
2709       bits = gvn().transform(new ConvI2LNode(bits));
2710       break;
2711     }
2712     case T_DOUBLE: {
2713       bits = gvn().transform(new MoveD2LNode(operation));
2714       break;
2715     }
2716     case T_LONG: {
2717       bits = operation; // no conversion needed
2718       break;
2719     }
2720     default: fatal("%s", type2name(elem_bt));
2721   }
2722 
2723   set_result(bits);
2724   return true;
2725 }
2726 
2727 // public static
2728 // <V extends Vector<E>,
2729 //  M extends VectorMask<E>,
2730 //  E>
2731 //  V comExpOp(int opr,
2732 //             Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2733 //             int length, V v, M m,
2734 //             CmpExpOperation<V, M> defaultImpl)
2735 bool LibraryCallKit::inline_vector_compress_expand() {
2736   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
2737   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2738   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
2739   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
2740   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
2741 
2742   if (vector_klass == NULL || elem_klass == NULL || mask_klass == NULL || vlen == NULL ||
2743       vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
2744       elem_klass->const_oop() == NULL || !vlen->is_con()) {
2745     if (C->print_intrinsics()) {
2746       tty->print_cr("  ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2747                     NodeClassNames[argument(0)->Opcode()],
2748                     NodeClassNames[argument(1)->Opcode()],
2749                     NodeClassNames[argument(2)->Opcode()],
2750                     NodeClassNames[argument(3)->Opcode()],
2751                     NodeClassNames[argument(4)->Opcode()]);
2752     }
2753     return false; // not enough info for intrinsification
2754   }
2755 
2756   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2757     if (C->print_intrinsics()) {
2758       tty->print_cr("  ** klass argument not initialized");
2759     }
2760     return false;
2761   }
2762 
2763   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2764   if (!elem_type->is_primitive_type()) {
2765     if (C->print_intrinsics()) {
2766       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2767     }
2768     return false; // should be primitive type
2769   }
2770 
2771   int num_elem = vlen->get_con();
2772   BasicType elem_bt = elem_type->basic_type();
2773   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2774 
2775   if (!arch_supports_vector(opc, num_elem, elem_bt, VecMaskUseLoad)) {
2776     if (C->print_intrinsics()) {
2777       tty->print_cr("  ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2778                     opc, num_elem, type2name(elem_bt));
2779     }
2780     return false; // not supported
2781   }
2782 
2783   Node* opd1 = NULL;
2784   const TypeInstPtr* vbox_type = NULL;
2785   if (opc != Op_CompressM) {
2786     ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2787     vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2788     opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2789     if (opd1 == NULL) {
2790       if (C->print_intrinsics()) {
2791         tty->print_cr("  ** unbox failed vector=%s",
2792                       NodeClassNames[argument(5)->Opcode()]);
2793       }
2794       return false;
2795     }
2796   }
2797 
2798   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2799   assert(is_vector_mask(mbox_klass), "argument(6) should be a mask class");
2800   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2801 
2802   Node* mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2803   if (mask == NULL) {
2804     if (C->print_intrinsics()) {
2805       tty->print_cr("  ** unbox failed mask=%s",
2806                     NodeClassNames[argument(6)->Opcode()]);
2807     }
2808     return false;
2809   }
2810 
2811   const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
2812   Node* operation = gvn().transform(VectorNode::make(opc, opd1, mask, vt));
2813 
2814   // Wrap it up in VectorBox to keep object type information.
2815   const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2816   Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2817   set_result(vbox);
2818   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2819   return true;
2820 }
--- EOF ---