< prev index next >

src/hotspot/share/opto/vectorIntrinsics.cpp

Print this page

 340 //  E>
 341 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 342 //            int length, V v1, V v2, M m,
 343 //            BinaryOperation<V, M> defaultImpl)
 344 //
 345 // public static
 346 // <V extends Vector<E>,
 347 //  M extends VectorMask<E>,
 348 //  E>
 349 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 350 //             int length, V v1, V v2, V v3, M m,
 351 //             TernaryOperation<V, M> defaultImpl)
 352 //
 353 bool LibraryCallKit::inline_vector_nary_operation(int n) {
 354   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
 355   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
 356   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
 357   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
 358   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
 359 
 360   if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
 361       !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {


 362     if (C->print_intrinsics()) {
 363       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
 364                     NodeClassNames[argument(0)->Opcode()],
 365                     NodeClassNames[argument(1)->Opcode()],
 366                     NodeClassNames[argument(3)->Opcode()],
 367                     NodeClassNames[argument(4)->Opcode()]);
 368     }
 369     return false; // not enough info for intrinsification
 370   }
 371 
 372   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 373   if (!elem_type->is_primitive_type()) {
 374     if (C->print_intrinsics()) {
 375       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 376     }
 377     return false; // should be primitive type
 378   }
 379   if (!is_klass_initialized(vector_klass)) {
 380     if (C->print_intrinsics()) {
 381       tty->print_cr("  ** klass argument not initialized");

 575   operation = gvn().transform(operation);
 576 
 577   // Wrap it up in VectorBox to keep object type information.
 578   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
 579   set_result(vbox);
 580   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 581   return true;
 582 }
 583 
 584 // <Sh extends VectorShuffle<E>,  E>
 585 //  Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
 586 //                  int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
 587 bool LibraryCallKit::inline_vector_shuffle_iota() {
 588   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
 589   const TypeInt*     vlen          = gvn().type(argument(3))->isa_int();
 590   const TypeInt*     start_val     = gvn().type(argument(4))->isa_int();
 591   const TypeInt*     step_val      = gvn().type(argument(5))->isa_int();
 592   const TypeInt*     wrap          = gvn().type(argument(6))->isa_int();
 593 
 594   if (shuffle_klass == nullptr || shuffle_klass->const_oop() == nullptr ||
 595       vlen == nullptr || !vlen->is_con() || start_val == nullptr || step_val == nullptr ||
 596       wrap == nullptr || !wrap->is_con()) {


 597     return false; // not enough info for intrinsification
 598   }
 599 
 600   if (!is_klass_initialized(shuffle_klass)) {
 601     if (C->print_intrinsics()) {
 602       tty->print_cr("  ** klass argument not initialized");
 603     }
 604     return false;
 605   }
 606 
 607   int do_wrap = wrap->get_con();
 608   int num_elem = vlen->get_con();
 609   BasicType elem_bt = T_BYTE;
 610 
 611   bool effective_indices_in_range = false;
 612   if (start_val->is_con() && step_val->is_con()) {
 613     int effective_min_index = start_val->get_con();
 614     int effective_max_index = start_val->get_con() + step_val->get_con() * (num_elem - 1);
 615     effective_indices_in_range = effective_max_index >= effective_min_index && effective_min_index >= -128 && effective_max_index <= 127;
 616   }

 685   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 686   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 687 
 688   // Wrap it up in VectorBox to keep object type information.
 689   res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
 690   set_result(res);
 691   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 692   return true;
 693 }
 694 
 695 // <E, M>
 696 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
 697 //                          int length, M m, VectorMaskOp<M> defaultImpl)
 698 bool LibraryCallKit::inline_vector_mask_operation() {
 699   const TypeInt*     oper       = gvn().type(argument(0))->isa_int();
 700   const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
 701   const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
 702   const TypeInt*     vlen       = gvn().type(argument(3))->isa_int();
 703   Node*              mask       = argument(4);
 704 
 705   if (mask_klass == nullptr || elem_klass == nullptr || mask->is_top() || vlen == nullptr) {




 706     return false; // dead code
 707   }
 708 
 709   if (!is_klass_initialized(mask_klass)) {
 710     if (C->print_intrinsics()) {
 711       tty->print_cr("  ** klass argument not initialized");
 712     }
 713     return false;
 714   }
 715 
 716   int num_elem = vlen->get_con();
 717   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 718   BasicType elem_bt = elem_type->basic_type();
 719 
 720   int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
 721   if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
 722     if (C->print_intrinsics()) {
 723       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 724                     mopc, num_elem, type2name(elem_bt));
 725     }

 752   return true;
 753 }
 754 
 755 // public static
 756 // <V,
 757 //  Sh extends VectorShuffle<E>,
 758 //  E>
 759 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
 760 //                   Class<? extends Sh> shuffleClass, Sh s, int length,
 761 //                   ShuffleToVectorOperation<V, Sh, E> defaultImpl)
 762 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
 763   const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
 764   const TypeInstPtr* elem_klass    = gvn().type(argument(1))->isa_instptr();
 765   const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
 766   Node*              shuffle       = argument(3);
 767   const TypeInt*     vlen          = gvn().type(argument(4))->isa_int();
 768 
 769   if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) {
 770     return false; // dead code
 771   }
 772   if (!vlen->is_con() || vector_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr) {
 773     return false; // not enough info for intrinsification
 774   }
 775   if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
 776     if (C->print_intrinsics()) {
 777       tty->print_cr("  ** klass argument not initialized");
 778     }
 779     return false;
 780   }
 781 
 782   int num_elem = vlen->get_con();
 783   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 784   BasicType elem_bt = elem_type->basic_type();
 785 
 786   if (num_elem < 4) {
 787     return false;
 788   }
 789 
 790   int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE
 791   // Make sure that cast is implemented to particular type/size combination.
 792   if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {

 817   return true;
 818 }
 819 
 820 // public static
 821 // <M,
 822 //  S extends VectorSpecies<E>,
 823 //  E>
 824 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
 825 //                    long bits, int mode, S s,
 826 //                    BroadcastOperation<M, E, S> defaultImpl)
 827 bool LibraryCallKit::inline_vector_frombits_coerced() {
 828   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 829   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 830   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 831   const TypeLong*    bits_type    = gvn().type(argument(3))->isa_long();
 832   // Mode argument determines the mode of operation it can take following values:-
 833   // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
 834   // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
 835   const TypeInt*     mode         = gvn().type(argument(5))->isa_int();
 836 
 837   if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || mode == nullptr ||
 838       bits_type == nullptr || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr ||
 839       !vlen->is_con() || !mode->is_con()) {


 840     if (C->print_intrinsics()) {
 841       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
 842                     NodeClassNames[argument(0)->Opcode()],
 843                     NodeClassNames[argument(1)->Opcode()],
 844                     NodeClassNames[argument(2)->Opcode()],
 845                     NodeClassNames[argument(5)->Opcode()]);
 846     }
 847     return false; // not enough info for intrinsification
 848   }
 849 
 850   if (!is_klass_initialized(vector_klass)) {
 851     if (C->print_intrinsics()) {
 852       tty->print_cr("  ** klass argument not initialized");
 853     }
 854     return false;
 855   }
 856   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 857   if (!elem_type->is_primitive_type()) {
 858     if (C->print_intrinsics()) {
 859       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());

 953 //          int length,
 954 //          Object base, long offset,            // Unsafe addressing
 955 //          boolean fromSegment,
 956 //          C container, long index, S s,        // Arguments for default implementation
 957 //          LoadOperation<C, VM, S> defaultImpl) {
 958 //  public static
 959 //  <C,
 960 //   V extends VectorPayload>
 961 //  void store(Class<?> vClass, Class<?> eClass,
 962 //             int length,
 963 //             Object base, long offset,        // Unsafe addressing
 964 //             boolean fromSegment,
 965 //             V v, C container, long index,    // Arguments for default implementation
 966 //             StoreVectorOperation<C, V> defaultImpl) {
 967 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
 968   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 969   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 970   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 971   const TypeInt*     from_ms      = gvn().type(argument(6))->isa_int();
 972 
 973   if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || !from_ms->is_con() ||
 974       vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {


 975     if (C->print_intrinsics()) {
 976       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
 977                     NodeClassNames[argument(0)->Opcode()],
 978                     NodeClassNames[argument(1)->Opcode()],
 979                     NodeClassNames[argument(2)->Opcode()],
 980                     NodeClassNames[argument(6)->Opcode()]);
 981     }
 982     return false; // not enough info for intrinsification
 983   }
 984   if (!is_klass_initialized(vector_klass)) {
 985     if (C->print_intrinsics()) {
 986       tty->print_cr("  ** klass argument not initialized");
 987     }
 988     return false;
 989   }
 990 
 991   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 992   if (!elem_type->is_primitive_type()) {
 993     if (C->print_intrinsics()) {
 994       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());

1165 //               LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1166 //  public static
1167 //  <C,
1168 //   V extends Vector<E>,
1169 //   M extends VectorMask<E>,
1170 //   E>
1171 //  void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1172 //                   int length,
1173 //                   Object base, long offset,                  // Unsafe addressing
1174 //                   boolean fromSegment,
1175 //                   V v, M m, C container, long index,         // Arguments for default implementation
1176 //                   StoreVectorMaskedOperation<C, V, M> defaultImpl) {
1177 
1178 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1179   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1180   const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1181   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1182   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1183   const TypeInt*     from_ms      = gvn().type(argument(7))->isa_int();
1184 
1185   if (vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1186       vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr || from_ms == nullptr ||
1187       elem_klass->const_oop() == nullptr || !vlen->is_con() || !from_ms->is_con()) {


1188     if (C->print_intrinsics()) {
1189       tty->print_cr("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1190                     NodeClassNames[argument(0)->Opcode()],
1191                     NodeClassNames[argument(1)->Opcode()],
1192                     NodeClassNames[argument(2)->Opcode()],
1193                     NodeClassNames[argument(3)->Opcode()],
1194                     NodeClassNames[argument(7)->Opcode()]);
1195     }
1196     return false; // not enough info for intrinsification
1197   }
1198   if (!is_klass_initialized(vector_klass)) {
1199     if (C->print_intrinsics()) {
1200       tty->print_cr("  ** klass argument not initialized");
1201     }
1202     return false;
1203   }
1204 
1205   if (!is_klass_initialized(mask_klass)) {
1206     if (C->print_intrinsics()) {
1207       tty->print_cr("  ** mask klass argument not initialized");

1428 //               LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1429 //
1430 //  <C,
1431 //   V extends Vector<E>,
1432 //   W extends Vector<Integer>,
1433 //   M extends VectorMask<E>,
1434 //   E>
1435 //  void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1436 //                    int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset,    // Unsafe addressing
1437 //                    W index_vector, V v, M m,
1438 //                    C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1439 //                    StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1440 //
1441 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1442   const TypeInstPtr* vector_klass     = gvn().type(argument(0))->isa_instptr();
1443   const TypeInstPtr* mask_klass       = gvn().type(argument(1))->isa_instptr();
1444   const TypeInstPtr* elem_klass       = gvn().type(argument(2))->isa_instptr();
1445   const TypeInt*     vlen             = gvn().type(argument(3))->isa_int();
1446   const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1447 
1448   if (vector_klass == nullptr || elem_klass == nullptr || vector_idx_klass == nullptr || vlen == nullptr ||
1449       vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || vector_idx_klass->const_oop() == nullptr || !vlen->is_con()) {



1450     if (C->print_intrinsics()) {
1451       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1452                     NodeClassNames[argument(0)->Opcode()],
1453                     NodeClassNames[argument(2)->Opcode()],
1454                     NodeClassNames[argument(3)->Opcode()],
1455                     NodeClassNames[argument(4)->Opcode()]);
1456     }
1457     return false; // not enough info for intrinsification
1458   }
1459 
1460   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1461     if (C->print_intrinsics()) {
1462       tty->print_cr("  ** klass argument not initialized");
1463     }
1464     return false;
1465   }
1466 
1467   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1468   if (!elem_type->is_primitive_type()) {
1469     if (C->print_intrinsics()) {

1620   destruct_map_clone(old_map);
1621 
1622   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1623   return true;
1624 }
1625 
1626 // public static
1627 // <V extends Vector<E>,
1628 //  M extends VectorMask<E>,
1629 //  E>
1630 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1631 //                       Class<E> elementType, int length, V v, M m,
1632 //                       ReductionOperation<V, M> defaultImpl)
1633 bool LibraryCallKit::inline_vector_reduction() {
1634   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
1635   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1636   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
1637   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
1638   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
1639 
1640   if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1641       !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {



1642     if (C->print_intrinsics()) {
1643       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1644                     NodeClassNames[argument(0)->Opcode()],
1645                     NodeClassNames[argument(1)->Opcode()],
1646                     NodeClassNames[argument(3)->Opcode()],
1647                     NodeClassNames[argument(4)->Opcode()]);
1648     }
1649     return false; // not enough info for intrinsification
1650   }
1651   if (!is_klass_initialized(vector_klass)) {
1652     if (C->print_intrinsics()) {
1653       tty->print_cr("  ** klass argument not initialized");
1654     }
1655     return false;
1656   }
1657   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1658   if (!elem_type->is_primitive_type()) {
1659     if (C->print_intrinsics()) {
1660       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1661     }

1773       bits = value; // no conversion needed
1774       break;
1775     }
1776     default: fatal("%s", type2name(elem_bt));
1777   }
1778   set_result(bits);
1779   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1780   return true;
1781 }
1782 
1783 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1784 //                                V v1, V v2,
1785 //                                BiFunction<V, V, Boolean> defaultImpl)
1786 //
1787 bool LibraryCallKit::inline_vector_test() {
1788   const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1789   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1790   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1791   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1792 
1793   if (cond == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1794       !cond->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {


1795     if (C->print_intrinsics()) {
1796       tty->print_cr("  ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1797                     NodeClassNames[argument(0)->Opcode()],
1798                     NodeClassNames[argument(1)->Opcode()],
1799                     NodeClassNames[argument(2)->Opcode()],
1800                     NodeClassNames[argument(3)->Opcode()]);
1801     }
1802     return false; // not enough info for intrinsification
1803   }
1804   if (!is_klass_initialized(vector_klass)) {
1805     if (C->print_intrinsics()) {
1806       tty->print_cr("  ** klass argument not initialized");
1807     }
1808     return false;
1809   }
1810   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1811   if (!elem_type->is_primitive_type()) {
1812     if (C->print_intrinsics()) {
1813       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1814     }

2703   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2704   return true;
2705 }
2706 
2707 //  public static
2708 //  <VM extends VectorPayload,
2709 //   E>
2710 //  long extract(Class<? extends VM> vClass, Class<E> eClass,
2711 //               int length,
2712 //               VM vm, int i,
2713 //               VecExtractOp<VM> defaultImpl)
2714 bool LibraryCallKit::inline_vector_extract() {
2715   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2716   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2717   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2718   const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
2719 
2720   if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2721     return false; // dead code
2722   }
2723   if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2724     if (C->print_intrinsics()) {
2725       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
2726                     NodeClassNames[argument(0)->Opcode()],
2727                     NodeClassNames[argument(1)->Opcode()],
2728                     NodeClassNames[argument(2)->Opcode()]);
2729     }
2730     return false; // not enough info for intrinsification
2731   }
2732   if (!is_klass_initialized(vector_klass)) {
2733     if (C->print_intrinsics()) {
2734       tty->print_cr("  ** klass argument not initialized");
2735     }
2736     return false;
2737   }
2738   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2739   if (!elem_type->is_primitive_type()) {
2740     if (C->print_intrinsics()) {
2741       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2742     }
2743     return false; // should be primitive type

2832   }
2833   set_result(opd);
2834   return true;
2835 }
2836 
2837 // public static
2838 // <V extends Vector<E>,
2839 //  M extends VectorMask<E>,
2840 //  E>
2841 //  V compressExpandOp(int opr,
2842 //                    Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2843 //                    int length, V v, M m,
2844 //                    CompressExpandOperation<V, M> defaultImpl)
2845 bool LibraryCallKit::inline_vector_compress_expand() {
2846   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
2847   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2848   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
2849   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
2850   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
2851 
2852   if (vector_klass == nullptr || elem_klass == nullptr || mask_klass == nullptr || vlen == nullptr ||
2853       vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
2854       elem_klass->const_oop() == nullptr || !vlen->is_con() || !opr->is_con()) {


2855     if (C->print_intrinsics()) {
2856       tty->print_cr("  ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2857                     NodeClassNames[argument(0)->Opcode()],
2858                     NodeClassNames[argument(1)->Opcode()],
2859                     NodeClassNames[argument(2)->Opcode()],
2860                     NodeClassNames[argument(3)->Opcode()],
2861                     NodeClassNames[argument(4)->Opcode()]);
2862     }
2863     return false; // not enough info for intrinsification
2864   }
2865 
2866   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2867     if (C->print_intrinsics()) {
2868       tty->print_cr("  ** klass argument not initialized");
2869     }
2870     return false;
2871   }
2872 
2873   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2874   if (!elem_type->is_primitive_type()) {

2925   const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2926   Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2927   set_result(vbox);
2928   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2929   return true;
2930 }
2931 
2932 // public static
2933 // <V extends Vector<E>,
2934 //  E,
2935 //  S extends VectorSpecies<E>>
2936 //  V indexVector(Class<? extends V> vClass, Class<E> eClass,
2937 //                int length,
2938 //                V v, int step, S s,
2939 //                IndexOperation<V, S> defaultImpl)
2940 bool LibraryCallKit::inline_index_vector() {
2941   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2942   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2943   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2944 
2945   if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2946       vector_klass->const_oop() == nullptr || !vlen->is_con() ||
2947       elem_klass->const_oop() == nullptr) {
2948     if (C->print_intrinsics()) {
2949       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
2950                     NodeClassNames[argument(0)->Opcode()],
2951                     NodeClassNames[argument(1)->Opcode()],
2952                     NodeClassNames[argument(2)->Opcode()]);
2953     }
2954     return false; // not enough info for intrinsification
2955   }
2956 
2957   if (!is_klass_initialized(vector_klass)) {
2958     if (C->print_intrinsics()) {
2959       tty->print_cr("  ** klass argument not initialized");
2960     }
2961     return false;
2962   }
2963 
2964   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2965   if (!elem_type->is_primitive_type()) {
2966     if (C->print_intrinsics()) {
2967       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());

3075   if (needs_add) {
3076     index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3077   }
3078   Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3079   set_result(vbox);
3080   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3081   return true;
3082 }
3083 
3084 // public static
3085 // <E,
3086 //  M extends VectorMask<E>>
3087 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3088 //                              long offset, long limit,
3089 //                              IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3090 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3091   const TypeInstPtr* mask_klass   = gvn().type(argument(0))->isa_instptr();
3092   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
3093   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
3094 
3095   if (mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
3096       mask_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {

3097     if (C->print_intrinsics()) {
3098       tty->print_cr("  ** missing constant: mclass=%s etype=%s vlen=%s",
3099                     NodeClassNames[argument(0)->Opcode()],
3100                     NodeClassNames[argument(1)->Opcode()],
3101                     NodeClassNames[argument(2)->Opcode()]);
3102     }
3103     return false; // not enough info for intrinsification
3104   }
3105 
3106   if (!is_klass_initialized(mask_klass)) {
3107     if (C->print_intrinsics()) {
3108       tty->print_cr("  ** klass argument not initialized");
3109     }
3110     return false;
3111   }
3112 
3113   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3114   if (!elem_type->is_primitive_type()) {
3115     if (C->print_intrinsics()) {
3116       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());

 340 //  E>
 341 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 342 //            int length, V v1, V v2, M m,
 343 //            BinaryOperation<V, M> defaultImpl)
 344 //
 345 // public static
 346 // <V extends Vector<E>,
 347 //  M extends VectorMask<E>,
 348 //  E>
 349 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
 350 //             int length, V v1, V v2, V v3, M m,
 351 //             TernaryOperation<V, M> defaultImpl)
 352 //
 353 bool LibraryCallKit::inline_vector_nary_operation(int n) {
 354   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
 355   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
 356   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
 357   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
 358   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
 359 
 360   if (opr          == nullptr || !opr->is_con() ||
 361       vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
 362       elem_klass   == nullptr || elem_klass->const_oop()   == nullptr ||
 363       vlen         == nullptr || !vlen->is_con()) {
 364     if (C->print_intrinsics()) {
 365       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
 366                     NodeClassNames[argument(0)->Opcode()],
 367                     NodeClassNames[argument(1)->Opcode()],
 368                     NodeClassNames[argument(3)->Opcode()],
 369                     NodeClassNames[argument(4)->Opcode()]);
 370     }
 371     return false; // not enough info for intrinsification
 372   }
 373 
 374   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 375   if (!elem_type->is_primitive_type()) {
 376     if (C->print_intrinsics()) {
 377       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
 378     }
 379     return false; // should be primitive type
 380   }
 381   if (!is_klass_initialized(vector_klass)) {
 382     if (C->print_intrinsics()) {
 383       tty->print_cr("  ** klass argument not initialized");

 577   operation = gvn().transform(operation);
 578 
 579   // Wrap it up in VectorBox to keep object type information.
 580   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
 581   set_result(vbox);
 582   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 583   return true;
 584 }
 585 
 586 // <Sh extends VectorShuffle<E>,  E>
 587 //  Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
 588 //                  int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
 589 bool LibraryCallKit::inline_vector_shuffle_iota() {
 590   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
 591   const TypeInt*     vlen          = gvn().type(argument(3))->isa_int();
 592   const TypeInt*     start_val     = gvn().type(argument(4))->isa_int();
 593   const TypeInt*     step_val      = gvn().type(argument(5))->isa_int();
 594   const TypeInt*     wrap          = gvn().type(argument(6))->isa_int();
 595 
 596   if (shuffle_klass == nullptr || shuffle_klass->const_oop() == nullptr ||
 597       vlen          == nullptr || !vlen->is_con() ||
 598       start_val     == nullptr ||
 599       step_val      == nullptr ||
 600       wrap          == nullptr || !wrap->is_con()) {
 601     return false; // not enough info for intrinsification
 602   }
 603 
 604   if (!is_klass_initialized(shuffle_klass)) {
 605     if (C->print_intrinsics()) {
 606       tty->print_cr("  ** klass argument not initialized");
 607     }
 608     return false;
 609   }
 610 
 611   int do_wrap = wrap->get_con();
 612   int num_elem = vlen->get_con();
 613   BasicType elem_bt = T_BYTE;
 614 
 615   bool effective_indices_in_range = false;
 616   if (start_val->is_con() && step_val->is_con()) {
 617     int effective_min_index = start_val->get_con();
 618     int effective_max_index = start_val->get_con() + step_val->get_con() * (num_elem - 1);
 619     effective_indices_in_range = effective_max_index >= effective_min_index && effective_min_index >= -128 && effective_max_index <= 127;
 620   }

 689   ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
 690   const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
 691 
 692   // Wrap it up in VectorBox to keep object type information.
 693   res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
 694   set_result(res);
 695   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 696   return true;
 697 }
 698 
 699 // <E, M>
 700 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
 701 //                          int length, M m, VectorMaskOp<M> defaultImpl)
 702 bool LibraryCallKit::inline_vector_mask_operation() {
 703   const TypeInt*     oper       = gvn().type(argument(0))->isa_int();
 704   const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
 705   const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
 706   const TypeInt*     vlen       = gvn().type(argument(3))->isa_int();
 707   Node*              mask       = argument(4);
 708 
 709   if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
 710       elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
 711       vlen       == nullptr || !vlen->is_con() ||
 712       oper       == nullptr || !oper->is_con() ||
 713       mask->is_top()) {
 714     return false; // dead code
 715   }
 716 
 717   if (!is_klass_initialized(mask_klass)) {
 718     if (C->print_intrinsics()) {
 719       tty->print_cr("  ** klass argument not initialized");
 720     }
 721     return false;
 722   }
 723 
 724   int num_elem = vlen->get_con();
 725   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 726   BasicType elem_bt = elem_type->basic_type();
 727 
 728   int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
 729   if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
 730     if (C->print_intrinsics()) {
 731       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 732                     mopc, num_elem, type2name(elem_bt));
 733     }

 760   return true;
 761 }
 762 
 763 // public static
 764 // <V,
 765 //  Sh extends VectorShuffle<E>,
 766 //  E>
 767 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
 768 //                   Class<? extends Sh> shuffleClass, Sh s, int length,
 769 //                   ShuffleToVectorOperation<V, Sh, E> defaultImpl)
 770 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
 771   const TypeInstPtr* vector_klass  = gvn().type(argument(0))->isa_instptr();
 772   const TypeInstPtr* elem_klass    = gvn().type(argument(1))->isa_instptr();
 773   const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
 774   Node*              shuffle       = argument(3);
 775   const TypeInt*     vlen          = gvn().type(argument(4))->isa_int();
 776 
 777   if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) {
 778     return false; // dead code
 779   }
 780   if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr || !vlen->is_con()) {
 781     return false; // not enough info for intrinsification
 782   }
 783   if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
 784     if (C->print_intrinsics()) {
 785       tty->print_cr("  ** klass argument not initialized");
 786     }
 787     return false;
 788   }
 789 
 790   int num_elem = vlen->get_con();
 791   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 792   BasicType elem_bt = elem_type->basic_type();
 793 
 794   if (num_elem < 4) {
 795     return false;
 796   }
 797 
 798   int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE
 799   // Make sure that cast is implemented to particular type/size combination.
 800   if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {

 825   return true;
 826 }
 827 
 828 // public static
 829 // <M,
 830 //  S extends VectorSpecies<E>,
 831 //  E>
 832 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
 833 //                    long bits, int mode, S s,
 834 //                    BroadcastOperation<M, E, S> defaultImpl)
 835 bool LibraryCallKit::inline_vector_frombits_coerced() {
 836   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 837   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 838   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 839   const TypeLong*    bits_type    = gvn().type(argument(3))->isa_long();
 840   // Mode argument determines the mode of operation it can take following values:-
 841   // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
 842   // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
 843   const TypeInt*     mode         = gvn().type(argument(5))->isa_int();
 844 
 845   if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
 846       elem_klass   == nullptr || elem_klass->const_oop()   == nullptr ||
 847       vlen         == nullptr || !vlen->is_con() ||
 848       bits_type    == nullptr ||
 849       mode         == nullptr || !mode->is_con()) {
 850     if (C->print_intrinsics()) {
 851       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
 852                     NodeClassNames[argument(0)->Opcode()],
 853                     NodeClassNames[argument(1)->Opcode()],
 854                     NodeClassNames[argument(2)->Opcode()],
 855                     NodeClassNames[argument(5)->Opcode()]);
 856     }
 857     return false; // not enough info for intrinsification
 858   }
 859 
 860   if (!is_klass_initialized(vector_klass)) {
 861     if (C->print_intrinsics()) {
 862       tty->print_cr("  ** klass argument not initialized");
 863     }
 864     return false;
 865   }
 866   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 867   if (!elem_type->is_primitive_type()) {
 868     if (C->print_intrinsics()) {
 869       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());

 963 //          int length,
 964 //          Object base, long offset,            // Unsafe addressing
 965 //          boolean fromSegment,
 966 //          C container, long index, S s,        // Arguments for default implementation
 967 //          LoadOperation<C, VM, S> defaultImpl) {
 968 //  public static
 969 //  <C,
 970 //   V extends VectorPayload>
 971 //  void store(Class<?> vClass, Class<?> eClass,
 972 //             int length,
 973 //             Object base, long offset,        // Unsafe addressing
 974 //             boolean fromSegment,
 975 //             V v, C container, long index,    // Arguments for default implementation
 976 //             StoreVectorOperation<C, V> defaultImpl) {
 977 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
 978   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 979   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 980   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 981   const TypeInt*     from_ms      = gvn().type(argument(6))->isa_int();
 982 
 983   if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
 984       elem_klass   == nullptr || elem_klass->const_oop()   == nullptr ||
 985       vlen         == nullptr || !vlen->is_con() ||
 986       from_ms      == nullptr || !from_ms->is_con()) {
 987     if (C->print_intrinsics()) {
 988       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
 989                     NodeClassNames[argument(0)->Opcode()],
 990                     NodeClassNames[argument(1)->Opcode()],
 991                     NodeClassNames[argument(2)->Opcode()],
 992                     NodeClassNames[argument(6)->Opcode()]);
 993     }
 994     return false; // not enough info for intrinsification
 995   }
 996   if (!is_klass_initialized(vector_klass)) {
 997     if (C->print_intrinsics()) {
 998       tty->print_cr("  ** klass argument not initialized");
 999     }
1000     return false;
1001   }
1002 
1003   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1004   if (!elem_type->is_primitive_type()) {
1005     if (C->print_intrinsics()) {
1006       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());

1177 //               LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1178 //  public static
1179 //  <C,
1180 //   V extends Vector<E>,
1181 //   M extends VectorMask<E>,
1182 //   E>
1183 //  void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1184 //                   int length,
1185 //                   Object base, long offset,                  // Unsafe addressing
1186 //                   boolean fromSegment,
1187 //                   V v, M m, C container, long index,         // Arguments for default implementation
1188 //                   StoreVectorMaskedOperation<C, V, M> defaultImpl) {
1189 
1190 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1191   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1192   const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1193   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1194   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1195   const TypeInt*     from_ms      = gvn().type(argument(7))->isa_int();
1196 
1197   if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1198       mask_klass   == nullptr || mask_klass->const_oop()   == nullptr ||
1199       elem_klass   == nullptr || elem_klass->const_oop()   == nullptr ||
1200       vlen         == nullptr || !vlen->is_con() ||
1201       from_ms      == nullptr || !from_ms->is_con()) {
1202     if (C->print_intrinsics()) {
1203       tty->print_cr("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1204                     NodeClassNames[argument(0)->Opcode()],
1205                     NodeClassNames[argument(1)->Opcode()],
1206                     NodeClassNames[argument(2)->Opcode()],
1207                     NodeClassNames[argument(3)->Opcode()],
1208                     NodeClassNames[argument(7)->Opcode()]);
1209     }
1210     return false; // not enough info for intrinsification
1211   }
1212   if (!is_klass_initialized(vector_klass)) {
1213     if (C->print_intrinsics()) {
1214       tty->print_cr("  ** klass argument not initialized");
1215     }
1216     return false;
1217   }
1218 
1219   if (!is_klass_initialized(mask_klass)) {
1220     if (C->print_intrinsics()) {
1221       tty->print_cr("  ** mask klass argument not initialized");

1442 //               LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1443 //
1444 //  <C,
1445 //   V extends Vector<E>,
1446 //   W extends Vector<Integer>,
1447 //   M extends VectorMask<E>,
1448 //   E>
1449 //  void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1450 //                    int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset,    // Unsafe addressing
1451 //                    W index_vector, V v, M m,
1452 //                    C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1453 //                    StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1454 //
1455 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1456   const TypeInstPtr* vector_klass     = gvn().type(argument(0))->isa_instptr();
1457   const TypeInstPtr* mask_klass       = gvn().type(argument(1))->isa_instptr();
1458   const TypeInstPtr* elem_klass       = gvn().type(argument(2))->isa_instptr();
1459   const TypeInt*     vlen             = gvn().type(argument(3))->isa_int();
1460   const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1461 
1462   if (vector_klass     == nullptr || vector_klass->const_oop()     == nullptr ||
1463 //      mask_klass       == nullptr || mask_klass->const_oop()       == nullptr ||
1464       elem_klass       == nullptr || elem_klass->const_oop()       == nullptr ||
1465       vlen             == nullptr || !vlen->is_con() ||
1466       vector_idx_klass == nullptr || vector_idx_klass->const_oop() == nullptr) {
1467     if (C->print_intrinsics()) {
1468       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1469                     NodeClassNames[argument(0)->Opcode()],
1470                     NodeClassNames[argument(2)->Opcode()],
1471                     NodeClassNames[argument(3)->Opcode()],
1472                     NodeClassNames[argument(4)->Opcode()]);
1473     }
1474     return false; // not enough info for intrinsification
1475   }
1476 
1477   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1478     if (C->print_intrinsics()) {
1479       tty->print_cr("  ** klass argument not initialized");
1480     }
1481     return false;
1482   }
1483 
1484   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1485   if (!elem_type->is_primitive_type()) {
1486     if (C->print_intrinsics()) {

1637   destruct_map_clone(old_map);
1638 
1639   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1640   return true;
1641 }
1642 
1643 // public static
1644 // <V extends Vector<E>,
1645 //  M extends VectorMask<E>,
1646 //  E>
1647 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1648 //                       Class<E> elementType, int length, V v, M m,
1649 //                       ReductionOperation<V, M> defaultImpl)
1650 bool LibraryCallKit::inline_vector_reduction() {
1651   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
1652   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1653   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
1654   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
1655   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
1656 
1657   if (opr          == nullptr || !opr->is_con() ||
1658       vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1659 //      mask_klass   == nullptr || mask_klass->const_oop()   == nullptr ||
1660       elem_klass   == nullptr || elem_klass->const_oop()   == nullptr ||
1661       vlen         == nullptr || !vlen->is_con()) {
1662     if (C->print_intrinsics()) {
1663       tty->print_cr("  ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1664                     NodeClassNames[argument(0)->Opcode()],
1665                     NodeClassNames[argument(1)->Opcode()],
1666                     NodeClassNames[argument(3)->Opcode()],
1667                     NodeClassNames[argument(4)->Opcode()]);
1668     }
1669     return false; // not enough info for intrinsification
1670   }
1671   if (!is_klass_initialized(vector_klass)) {
1672     if (C->print_intrinsics()) {
1673       tty->print_cr("  ** klass argument not initialized");
1674     }
1675     return false;
1676   }
1677   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1678   if (!elem_type->is_primitive_type()) {
1679     if (C->print_intrinsics()) {
1680       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1681     }

1793       bits = value; // no conversion needed
1794       break;
1795     }
1796     default: fatal("%s", type2name(elem_bt));
1797   }
1798   set_result(bits);
1799   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1800   return true;
1801 }
1802 
1803 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1804 //                                V v1, V v2,
1805 //                                BiFunction<V, V, Boolean> defaultImpl)
1806 //
1807 bool LibraryCallKit::inline_vector_test() {
1808   const TypeInt*     cond         = gvn().type(argument(0))->isa_int();
1809   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1810   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1811   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1812 
1813   if (cond         == nullptr || !cond->is_con() ||
1814       vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1815       elem_klass   == nullptr || elem_klass->const_oop()   == nullptr ||
1816       vlen         == nullptr || !vlen->is_con()) {
1817     if (C->print_intrinsics()) {
1818       tty->print_cr("  ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1819                     NodeClassNames[argument(0)->Opcode()],
1820                     NodeClassNames[argument(1)->Opcode()],
1821                     NodeClassNames[argument(2)->Opcode()],
1822                     NodeClassNames[argument(3)->Opcode()]);
1823     }
1824     return false; // not enough info for intrinsification
1825   }
1826   if (!is_klass_initialized(vector_klass)) {
1827     if (C->print_intrinsics()) {
1828       tty->print_cr("  ** klass argument not initialized");
1829     }
1830     return false;
1831   }
1832   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1833   if (!elem_type->is_primitive_type()) {
1834     if (C->print_intrinsics()) {
1835       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
1836     }

2725   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2726   return true;
2727 }
2728 
2729 //  public static
2730 //  <VM extends VectorPayload,
2731 //   E>
2732 //  long extract(Class<? extends VM> vClass, Class<E> eClass,
2733 //               int length,
2734 //               VM vm, int i,
2735 //               VecExtractOp<VM> defaultImpl)
2736 bool LibraryCallKit::inline_vector_extract() {
2737   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2738   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2739   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2740   const TypeInt*     idx          = gvn().type(argument(4))->isa_int();
2741 
2742   if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2743     return false; // dead code
2744   }
2745   if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2746     if (C->print_intrinsics()) {
2747       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
2748                     NodeClassNames[argument(0)->Opcode()],
2749                     NodeClassNames[argument(1)->Opcode()],
2750                     NodeClassNames[argument(2)->Opcode()]);
2751     }
2752     return false; // not enough info for intrinsification
2753   }
2754   if (!is_klass_initialized(vector_klass)) {
2755     if (C->print_intrinsics()) {
2756       tty->print_cr("  ** klass argument not initialized");
2757     }
2758     return false;
2759   }
2760   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2761   if (!elem_type->is_primitive_type()) {
2762     if (C->print_intrinsics()) {
2763       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2764     }
2765     return false; // should be primitive type

2854   }
2855   set_result(opd);
2856   return true;
2857 }
2858 
2859 // public static
2860 // <V extends Vector<E>,
2861 //  M extends VectorMask<E>,
2862 //  E>
2863 //  V compressExpandOp(int opr,
2864 //                    Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2865 //                    int length, V v, M m,
2866 //                    CompressExpandOperation<V, M> defaultImpl)
2867 bool LibraryCallKit::inline_vector_compress_expand() {
2868   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
2869   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2870   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
2871   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
2872   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
2873 
2874   if (opr          == nullptr || !opr->is_con() ||
2875       vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2876       mask_klass   == nullptr || mask_klass->const_oop()   == nullptr ||
2877       elem_klass   == nullptr || elem_klass->const_oop()   == nullptr ||
2878       vlen         == nullptr || !vlen->is_con()) {
2879     if (C->print_intrinsics()) {
2880       tty->print_cr("  ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2881                     NodeClassNames[argument(0)->Opcode()],
2882                     NodeClassNames[argument(1)->Opcode()],
2883                     NodeClassNames[argument(2)->Opcode()],
2884                     NodeClassNames[argument(3)->Opcode()],
2885                     NodeClassNames[argument(4)->Opcode()]);
2886     }
2887     return false; // not enough info for intrinsification
2888   }
2889 
2890   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2891     if (C->print_intrinsics()) {
2892       tty->print_cr("  ** klass argument not initialized");
2893     }
2894     return false;
2895   }
2896 
2897   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2898   if (!elem_type->is_primitive_type()) {

2949   const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2950   Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2951   set_result(vbox);
2952   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2953   return true;
2954 }
2955 
2956 // public static
2957 // <V extends Vector<E>,
2958 //  E,
2959 //  S extends VectorSpecies<E>>
2960 //  V indexVector(Class<? extends V> vClass, Class<E> eClass,
2961 //                int length,
2962 //                V v, int step, S s,
2963 //                IndexOperation<V, S> defaultImpl)
2964 bool LibraryCallKit::inline_index_vector() {
2965   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2966   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
2967   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
2968 
2969   if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2970       elem_klass   == nullptr || elem_klass->const_oop()   == nullptr ||
2971       vlen         == nullptr || !vlen->is_con() ) {
2972     if (C->print_intrinsics()) {
2973       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
2974                     NodeClassNames[argument(0)->Opcode()],
2975                     NodeClassNames[argument(1)->Opcode()],
2976                     NodeClassNames[argument(2)->Opcode()]);
2977     }
2978     return false; // not enough info for intrinsification
2979   }
2980 
2981   if (!is_klass_initialized(vector_klass)) {
2982     if (C->print_intrinsics()) {
2983       tty->print_cr("  ** klass argument not initialized");
2984     }
2985     return false;
2986   }
2987 
2988   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2989   if (!elem_type->is_primitive_type()) {
2990     if (C->print_intrinsics()) {
2991       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());

3099   if (needs_add) {
3100     index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3101   }
3102   Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3103   set_result(vbox);
3104   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3105   return true;
3106 }
3107 
3108 // public static
3109 // <E,
3110 //  M extends VectorMask<E>>
3111 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3112 //                              long offset, long limit,
3113 //                              IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3114 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3115   const TypeInstPtr* mask_klass   = gvn().type(argument(0))->isa_instptr();
3116   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
3117   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
3118 
3119   if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
3120       elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
3121       vlen       == nullptr || !vlen->is_con()) {
3122     if (C->print_intrinsics()) {
3123       tty->print_cr("  ** missing constant: mclass=%s etype=%s vlen=%s",
3124                     NodeClassNames[argument(0)->Opcode()],
3125                     NodeClassNames[argument(1)->Opcode()],
3126                     NodeClassNames[argument(2)->Opcode()]);
3127     }
3128     return false; // not enough info for intrinsification
3129   }
3130 
3131   if (!is_klass_initialized(mask_klass)) {
3132     if (C->print_intrinsics()) {
3133       tty->print_cr("  ** klass argument not initialized");
3134     }
3135     return false;
3136   }
3137 
3138   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3139   if (!elem_type->is_primitive_type()) {
3140     if (C->print_intrinsics()) {
3141       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
< prev index next >