306 // E>
307 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
308 // int length, V v1, V v2, M m,
309 // BinaryOperation<V, M> defaultImpl)
310 //
311 // public static
312 // <V extends Vector<E>,
313 // M extends VectorMask<E>,
314 // E>
315 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
316 // int length, V v1, V v2, V v3, M m,
317 // TernaryOperation<V, M> defaultImpl)
318 //
319 bool LibraryCallKit::inline_vector_nary_operation(int n) {
320 const TypeInt* opr = gvn().type(argument(0))->isa_int();
321 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
322 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
323 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
324 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
325
326 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
327 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
328 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
329 NodeClassNames[argument(0)->Opcode()],
330 NodeClassNames[argument(1)->Opcode()],
331 NodeClassNames[argument(3)->Opcode()],
332 NodeClassNames[argument(4)->Opcode()]);
333 return false; // not enough info for intrinsification
334 }
335
336 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
337 if (!elem_type->is_primitive_type()) {
338 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
339 return false; // should be primitive type
340 }
341 if (!is_klass_initialized(vector_klass)) {
342 log_if_needed(" ** klass argument not initialized");
343 return false;
344 }
345
346 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
347 // is used to control this operation.
536 const TypeVect* vmask_type = TypeVect::makemask(type_bt, num_elem);
537 Node* mask = gvn().transform(new VectorMaskCmpNode(pred, bcast_lane_cnt, index_vec, pred_node, vmask_type));
538
539 // Make the indices greater than lane count as -ve values to match the java side implementation.
540 index_vec = gvn().transform(VectorNode::make(Op_AndV, index_vec, bcast_mod_mask, vt));
541 Node* biased_val = gvn().transform(VectorNode::make(Op_SubVB, index_vec, bcast_lane_cnt, vt));
542 return gvn().transform(new VectorBlendNode(biased_val, index_vec, mask));
543 }
544
545 // <Sh extends VectorShuffle<E>, E>
546 // Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
547 // int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
548 bool LibraryCallKit::inline_vector_shuffle_iota() {
549 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
550 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
551 const TypeInt* start_val = gvn().type(argument(4))->isa_int();
552 const TypeInt* step_val = gvn().type(argument(5))->isa_int();
553 const TypeInt* wrap = gvn().type(argument(6))->isa_int();
554
555 if (shuffle_klass == nullptr || shuffle_klass->const_oop() == nullptr ||
556 vlen == nullptr || !vlen->is_con() || start_val == nullptr || step_val == nullptr ||
557 wrap == nullptr || !wrap->is_con()) {
558 return false; // not enough info for intrinsification
559 }
560
561 if (!is_klass_initialized(shuffle_klass)) {
562 log_if_needed(" ** klass argument not initialized");
563 return false;
564 }
565
566 int do_wrap = wrap->get_con();
567 int num_elem = vlen->get_con();
568 BasicType elem_bt = T_BYTE;
569
570 bool effective_indices_in_range = false;
571 if (start_val->is_con() && step_val->is_con()) {
572 int effective_min_index = start_val->get_con();
573 int effective_max_index = start_val->get_con() + step_val->get_con() * (num_elem - 1);
574 effective_indices_in_range = effective_max_index >= effective_min_index && effective_min_index >= -128 && effective_max_index <= 127;
575 }
576
577 if (!do_wrap && !effective_indices_in_range) {
635 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
636 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
637
638 // Wrap it up in VectorBox to keep object type information.
639 res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
640 set_result(res);
641 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
642 return true;
643 }
644
645 // <E, M>
646 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
647 // int length, M m, VectorMaskOp<M> defaultImpl)
648 bool LibraryCallKit::inline_vector_mask_operation() {
649 const TypeInt* oper = gvn().type(argument(0))->isa_int();
650 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
651 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
652 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
653 Node* mask = argument(4);
654
655 if (mask_klass == nullptr || elem_klass == nullptr || mask->is_top() || vlen == nullptr) {
656 return false; // dead code
657 }
658
659 if (!is_klass_initialized(mask_klass)) {
660 log_if_needed(" ** klass argument not initialized");
661 return false;
662 }
663
664 int num_elem = vlen->get_con();
665 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
666 BasicType elem_bt = elem_type->basic_type();
667
668 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
669 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
670 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
671 mopc, num_elem, type2name(elem_bt));
672 return false; // not supported
673 }
674
675 const Type* elem_ty = Type::get_const_basic_type(elem_bt);
696 return true;
697 }
698
699 // public static
700 // <V,
701 // Sh extends VectorShuffle<E>,
702 // E>
703 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
704 // Class<? extends Sh> shuffleClass, Sh s, int length,
705 // ShuffleToVectorOperation<V, Sh, E> defaultImpl)
706 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
707 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
708 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
709 const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
710 Node* shuffle = argument(3);
711 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
712
713 if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) {
714 return false; // dead code
715 }
716 if (!vlen->is_con() || vector_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr) {
717 return false; // not enough info for intrinsification
718 }
719 if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
720 log_if_needed(" ** klass argument not initialized");
721 return false;
722 }
723
724 int num_elem = vlen->get_con();
725 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
726 BasicType elem_bt = elem_type->basic_type();
727
728 if (num_elem < 4) {
729 return false;
730 }
731
732 int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE
733 // Make sure that cast is implemented to particular type/size combination.
734 if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
735 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
736 cast_vopc, num_elem, type2name(elem_bt));
815 return true;
816 }
817
818 // public static
819 // <M,
820 // S extends VectorSpecies<E>,
821 // E>
822 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
823 // long bits, int mode, S s,
824 // BroadcastOperation<M, E, S> defaultImpl)
825 bool LibraryCallKit::inline_vector_frombits_coerced() {
826 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
827 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
828 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
829 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
830 // Mode argument determines the mode of operation it can take following values:-
831 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
832 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
833 const TypeInt* mode = gvn().type(argument(5))->isa_int();
834
835 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || mode == nullptr ||
836 bits_type == nullptr || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr ||
837 !vlen->is_con() || !mode->is_con()) {
838 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
839 NodeClassNames[argument(0)->Opcode()],
840 NodeClassNames[argument(1)->Opcode()],
841 NodeClassNames[argument(2)->Opcode()],
842 NodeClassNames[argument(5)->Opcode()]);
843 return false; // not enough info for intrinsification
844 }
845
846 if (!is_klass_initialized(vector_klass)) {
847 log_if_needed(" ** klass argument not initialized");
848 return false;
849 }
850 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
851 if (!elem_type->is_primitive_type()) {
852 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
853 return false; // should be primitive type
854 }
855 BasicType elem_bt = elem_type->basic_type();
856 int num_elem = vlen->get_con();
857 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
943 // int length,
944 // Object base, long offset, // Unsafe addressing
945 // boolean fromSegment,
946 // C container, long index, S s, // Arguments for default implementation
947 // LoadOperation<C, VM, S> defaultImpl) {
948 // public static
949 // <C,
950 // V extends VectorPayload>
951 // void store(Class<?> vClass, Class<?> eClass,
952 // int length,
953 // Object base, long offset, // Unsafe addressing
954 // boolean fromSegment,
955 // V v, C container, long index, // Arguments for default implementation
956 // StoreVectorOperation<C, V> defaultImpl) {
957 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
958 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
959 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
960 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
961 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
962
963 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || !from_ms->is_con() ||
964 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
965 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
966 NodeClassNames[argument(0)->Opcode()],
967 NodeClassNames[argument(1)->Opcode()],
968 NodeClassNames[argument(2)->Opcode()],
969 NodeClassNames[argument(6)->Opcode()]);
970 return false; // not enough info for intrinsification
971 }
972 if (!is_klass_initialized(vector_klass)) {
973 log_if_needed(" ** klass argument not initialized");
974 return false;
975 }
976
977 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
978 if (!elem_type->is_primitive_type()) {
979 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
980 return false; // should be primitive type
981 }
982 BasicType elem_bt = elem_type->basic_type();
983 int num_elem = vlen->get_con();
984
1151 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1152 // public static
1153 // <C,
1154 // V extends Vector<E>,
1155 // M extends VectorMask<E>,
1156 // E>
1157 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1158 // int length,
1159 // Object base, long offset, // Unsafe addressing
1160 // boolean fromSegment,
1161 // V v, M m, C container, long index, // Arguments for default implementation
1162 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
1163
1164 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1165 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1166 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1167 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1168 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1169 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
1170
1171 if (vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1172 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr || from_ms == nullptr ||
1173 elem_klass->const_oop() == nullptr || !vlen->is_con() || !from_ms->is_con()) {
1174 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1175 NodeClassNames[argument(0)->Opcode()],
1176 NodeClassNames[argument(1)->Opcode()],
1177 NodeClassNames[argument(2)->Opcode()],
1178 NodeClassNames[argument(3)->Opcode()],
1179 NodeClassNames[argument(7)->Opcode()]);
1180 return false; // not enough info for intrinsification
1181 }
1182 if (!is_klass_initialized(vector_klass)) {
1183 log_if_needed(" ** klass argument not initialized");
1184 return false;
1185 }
1186
1187 if (!is_klass_initialized(mask_klass)) {
1188 log_if_needed(" ** mask klass argument not initialized");
1189 return false;
1190 }
1191
1192 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1193 if (!elem_type->is_primitive_type()) {
1390 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1391 //
1392 // <C,
1393 // V extends Vector<E>,
1394 // W extends Vector<Integer>,
1395 // M extends VectorMask<E>,
1396 // E>
1397 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1398 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1399 // W index_vector, V v, M m,
1400 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1401 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1402 //
1403 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1404 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1405 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1406 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1407 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1408 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1409
1410 if (vector_klass == nullptr || elem_klass == nullptr || vector_idx_klass == nullptr || vlen == nullptr ||
1411 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || vector_idx_klass->const_oop() == nullptr || !vlen->is_con()) {
1412 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1413 NodeClassNames[argument(0)->Opcode()],
1414 NodeClassNames[argument(2)->Opcode()],
1415 NodeClassNames[argument(3)->Opcode()],
1416 NodeClassNames[argument(4)->Opcode()]);
1417 return false; // not enough info for intrinsification
1418 }
1419
1420 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1421 log_if_needed(" ** klass argument not initialized");
1422 return false;
1423 }
1424
1425 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1426 if (!elem_type->is_primitive_type()) {
1427 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1428 return false; // should be primitive type
1429 }
1430
1431 BasicType elem_bt = elem_type->basic_type();
1577 destruct_map_clone(old_map);
1578
1579 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1580 return true;
1581 }
1582
1583 // public static
1584 // <V extends Vector<E>,
1585 // M extends VectorMask<E>,
1586 // E>
1587 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1588 // Class<E> elementType, int length, V v, M m,
1589 // ReductionOperation<V, M> defaultImpl)
1590 bool LibraryCallKit::inline_vector_reduction() {
1591 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1592 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1593 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1594 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1595 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1596
1597 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1598 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1599 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1600 NodeClassNames[argument(0)->Opcode()],
1601 NodeClassNames[argument(1)->Opcode()],
1602 NodeClassNames[argument(3)->Opcode()],
1603 NodeClassNames[argument(4)->Opcode()]);
1604 return false; // not enough info for intrinsification
1605 }
1606 if (!is_klass_initialized(vector_klass)) {
1607 log_if_needed(" ** klass argument not initialized");
1608 return false;
1609 }
1610 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1611 if (!elem_type->is_primitive_type()) {
1612 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1613 return false; // should be primitive type
1614 }
1615
1616 const Type* vmask_type = gvn().type(argument(6));
1617 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1618 if (is_masked_op) {
1714 bits = value; // no conversion needed
1715 break;
1716 }
1717 default: fatal("%s", type2name(elem_bt));
1718 }
1719 set_result(bits);
1720 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1721 return true;
1722 }
1723
1724 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1725 // V v1, V v2,
1726 // BiFunction<V, V, Boolean> defaultImpl)
1727 //
1728 bool LibraryCallKit::inline_vector_test() {
1729 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1730 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1731 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1732 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1733
1734 if (cond == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1735 !cond->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1736 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1737 NodeClassNames[argument(0)->Opcode()],
1738 NodeClassNames[argument(1)->Opcode()],
1739 NodeClassNames[argument(2)->Opcode()],
1740 NodeClassNames[argument(3)->Opcode()]);
1741 return false; // not enough info for intrinsification
1742 }
1743 if (!is_klass_initialized(vector_klass)) {
1744 log_if_needed(" ** klass argument not initialized");
1745 return false;
1746 }
1747 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1748 if (!elem_type->is_primitive_type()) {
1749 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1750 return false; // should be primitive type
1751 }
1752 BasicType elem_bt = elem_type->basic_type();
1753 int num_elem = vlen->get_con();
1754 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1755 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2727 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2728 return true;
2729 }
2730
2731 // public static
2732 // <VM extends VectorPayload,
2733 // E>
2734 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2735 // int length,
2736 // VM vm, int i,
2737 // VecExtractOp<VM> defaultImpl)
2738 bool LibraryCallKit::inline_vector_extract() {
2739 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2740 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2741 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2742 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2743
2744 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2745 return false; // dead code
2746 }
2747 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2748 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2749 NodeClassNames[argument(0)->Opcode()],
2750 NodeClassNames[argument(1)->Opcode()],
2751 NodeClassNames[argument(2)->Opcode()]);
2752 return false; // not enough info for intrinsification
2753 }
2754 if (!is_klass_initialized(vector_klass)) {
2755 log_if_needed(" ** klass argument not initialized");
2756 return false;
2757 }
2758 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2759 if (!elem_type->is_primitive_type()) {
2760 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2761 return false; // should be primitive type
2762 }
2763 BasicType elem_bt = elem_type->basic_type();
2764 int num_elem = vlen->get_con();
2765
2766 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2767 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2844 }
2845 set_result(opd);
2846 return true;
2847 }
2848
2849 // public static
2850 // <V extends Vector<E>,
2851 // M extends VectorMask<E>,
2852 // E>
2853 // V compressExpandOp(int opr,
2854 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2855 // int length, V v, M m,
2856 // CompressExpandOperation<V, M> defaultImpl)
2857 bool LibraryCallKit::inline_vector_compress_expand() {
2858 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2859 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2860 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2861 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2862 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2863
2864 if (vector_klass == nullptr || elem_klass == nullptr || mask_klass == nullptr || vlen == nullptr ||
2865 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
2866 elem_klass->const_oop() == nullptr || !vlen->is_con() || !opr->is_con()) {
2867 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2868 NodeClassNames[argument(0)->Opcode()],
2869 NodeClassNames[argument(1)->Opcode()],
2870 NodeClassNames[argument(2)->Opcode()],
2871 NodeClassNames[argument(3)->Opcode()],
2872 NodeClassNames[argument(4)->Opcode()]);
2873 return false; // not enough info for intrinsification
2874 }
2875
2876 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2877 log_if_needed(" ** klass argument not initialized");
2878 return false;
2879 }
2880
2881 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2882 if (!elem_type->is_primitive_type()) {
2883 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2884 return false; // should be primitive type
2885 }
2886
2925 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2926 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2927 set_result(vbox);
2928 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2929 return true;
2930 }
2931
2932 // public static
2933 // <V extends Vector<E>,
2934 // E,
2935 // S extends VectorSpecies<E>>
2936 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2937 // int length,
2938 // V v, int step, S s,
2939 // IndexOperation<V, S> defaultImpl)
2940 bool LibraryCallKit::inline_index_vector() {
2941 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2942 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2943 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2944
2945 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2946 vector_klass->const_oop() == nullptr || !vlen->is_con() ||
2947 elem_klass->const_oop() == nullptr) {
2948 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2949 NodeClassNames[argument(0)->Opcode()],
2950 NodeClassNames[argument(1)->Opcode()],
2951 NodeClassNames[argument(2)->Opcode()]);
2952 return false; // not enough info for intrinsification
2953 }
2954
2955 if (!is_klass_initialized(vector_klass)) {
2956 log_if_needed(" ** klass argument not initialized");
2957 return false;
2958 }
2959
2960 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2961 if (!elem_type->is_primitive_type()) {
2962 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2963 return false; // should be primitive type
2964 }
2965
2966 int num_elem = vlen->get_con();
2967 BasicType elem_bt = elem_type->basic_type();
3059 if (needs_add) {
3060 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3061 }
3062 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3063 set_result(vbox);
3064 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3065 return true;
3066 }
3067
3068 // public static
3069 // <E,
3070 // M extends VectorMask<E>>
3071 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3072 // long offset, long limit,
3073 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3074 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3075 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
3076 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3077 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3078
3079 if (mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
3080 mask_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
3081 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
3082 NodeClassNames[argument(0)->Opcode()],
3083 NodeClassNames[argument(1)->Opcode()],
3084 NodeClassNames[argument(2)->Opcode()]);
3085 return false; // not enough info for intrinsification
3086 }
3087
3088 if (!is_klass_initialized(mask_klass)) {
3089 log_if_needed(" ** klass argument not initialized");
3090 return false;
3091 }
3092
3093 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3094 if (!elem_type->is_primitive_type()) {
3095 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3096 return false; // should be primitive type
3097 }
3098
3099 int num_elem = vlen->get_con();
3100 BasicType elem_bt = elem_type->basic_type();
|
306 // E>
307 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
308 // int length, V v1, V v2, M m,
309 // BinaryOperation<V, M> defaultImpl)
310 //
311 // public static
312 // <V extends Vector<E>,
313 // M extends VectorMask<E>,
314 // E>
315 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
316 // int length, V v1, V v2, V v3, M m,
317 // TernaryOperation<V, M> defaultImpl)
318 //
319 bool LibraryCallKit::inline_vector_nary_operation(int n) {
320 const TypeInt* opr = gvn().type(argument(0))->isa_int();
321 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
322 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
323 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
324 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
325
326 if (opr == nullptr || !opr->is_con() ||
327 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
328 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
329 vlen == nullptr || !vlen->is_con()) {
330 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
331 NodeClassNames[argument(0)->Opcode()],
332 NodeClassNames[argument(1)->Opcode()],
333 NodeClassNames[argument(3)->Opcode()],
334 NodeClassNames[argument(4)->Opcode()]);
335 return false; // not enough info for intrinsification
336 }
337
338 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
339 if (!elem_type->is_primitive_type()) {
340 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
341 return false; // should be primitive type
342 }
343 if (!is_klass_initialized(vector_klass)) {
344 log_if_needed(" ** klass argument not initialized");
345 return false;
346 }
347
348 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
349 // is used to control this operation.
538 const TypeVect* vmask_type = TypeVect::makemask(type_bt, num_elem);
539 Node* mask = gvn().transform(new VectorMaskCmpNode(pred, bcast_lane_cnt, index_vec, pred_node, vmask_type));
540
541 // Make the indices greater than lane count as -ve values to match the java side implementation.
542 index_vec = gvn().transform(VectorNode::make(Op_AndV, index_vec, bcast_mod_mask, vt));
543 Node* biased_val = gvn().transform(VectorNode::make(Op_SubVB, index_vec, bcast_lane_cnt, vt));
544 return gvn().transform(new VectorBlendNode(biased_val, index_vec, mask));
545 }
546
547 // <Sh extends VectorShuffle<E>, E>
548 // Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
549 // int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
550 bool LibraryCallKit::inline_vector_shuffle_iota() {
551 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
552 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
553 const TypeInt* start_val = gvn().type(argument(4))->isa_int();
554 const TypeInt* step_val = gvn().type(argument(5))->isa_int();
555 const TypeInt* wrap = gvn().type(argument(6))->isa_int();
556
557 if (shuffle_klass == nullptr || shuffle_klass->const_oop() == nullptr ||
558 vlen == nullptr || !vlen->is_con() ||
559 start_val == nullptr ||
560 step_val == nullptr ||
561 wrap == nullptr || !wrap->is_con()) {
562 return false; // not enough info for intrinsification
563 }
564
565 if (!is_klass_initialized(shuffle_klass)) {
566 log_if_needed(" ** klass argument not initialized");
567 return false;
568 }
569
570 int do_wrap = wrap->get_con();
571 int num_elem = vlen->get_con();
572 BasicType elem_bt = T_BYTE;
573
574 bool effective_indices_in_range = false;
575 if (start_val->is_con() && step_val->is_con()) {
576 int effective_min_index = start_val->get_con();
577 int effective_max_index = start_val->get_con() + step_val->get_con() * (num_elem - 1);
578 effective_indices_in_range = effective_max_index >= effective_min_index && effective_min_index >= -128 && effective_max_index <= 127;
579 }
580
581 if (!do_wrap && !effective_indices_in_range) {
639 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
640 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
641
642 // Wrap it up in VectorBox to keep object type information.
643 res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
644 set_result(res);
645 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
646 return true;
647 }
648
649 // <E, M>
650 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
651 // int length, M m, VectorMaskOp<M> defaultImpl)
652 bool LibraryCallKit::inline_vector_mask_operation() {
653 const TypeInt* oper = gvn().type(argument(0))->isa_int();
654 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
655 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
656 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
657 Node* mask = argument(4);
658
659 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
660 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
661 vlen == nullptr || !vlen->is_con() ||
662 oper == nullptr || !oper->is_con() ||
663 mask->is_top()) {
664 return false; // dead code
665 }
666
667 if (!is_klass_initialized(mask_klass)) {
668 log_if_needed(" ** klass argument not initialized");
669 return false;
670 }
671
672 int num_elem = vlen->get_con();
673 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
674 BasicType elem_bt = elem_type->basic_type();
675
676 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
677 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
678 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
679 mopc, num_elem, type2name(elem_bt));
680 return false; // not supported
681 }
682
683 const Type* elem_ty = Type::get_const_basic_type(elem_bt);
704 return true;
705 }
706
707 // public static
708 // <V,
709 // Sh extends VectorShuffle<E>,
710 // E>
711 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
712 // Class<? extends Sh> shuffleClass, Sh s, int length,
713 // ShuffleToVectorOperation<V, Sh, E> defaultImpl)
714 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
715 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
716 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
717 const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
718 Node* shuffle = argument(3);
719 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
720
721 if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) {
722 return false; // dead code
723 }
724 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr || !vlen->is_con()) {
725 return false; // not enough info for intrinsification
726 }
727 if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
728 log_if_needed(" ** klass argument not initialized");
729 return false;
730 }
731
732 int num_elem = vlen->get_con();
733 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
734 BasicType elem_bt = elem_type->basic_type();
735
736 if (num_elem < 4) {
737 return false;
738 }
739
740 int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE
741 // Make sure that cast is implemented to particular type/size combination.
742 if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
743 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
744 cast_vopc, num_elem, type2name(elem_bt));
823 return true;
824 }
825
826 // public static
827 // <M,
828 // S extends VectorSpecies<E>,
829 // E>
830 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
831 // long bits, int mode, S s,
832 // BroadcastOperation<M, E, S> defaultImpl)
833 bool LibraryCallKit::inline_vector_frombits_coerced() {
834 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
835 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
836 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
837 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
838 // Mode argument determines the mode of operation it can take following values:-
839 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
840 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
841 const TypeInt* mode = gvn().type(argument(5))->isa_int();
842
843 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
844 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
845 vlen == nullptr || !vlen->is_con() ||
846 bits_type == nullptr ||
847 mode == nullptr || !mode->is_con()) {
848 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
849 NodeClassNames[argument(0)->Opcode()],
850 NodeClassNames[argument(1)->Opcode()],
851 NodeClassNames[argument(2)->Opcode()],
852 NodeClassNames[argument(5)->Opcode()]);
853 return false; // not enough info for intrinsification
854 }
855
856 if (!is_klass_initialized(vector_klass)) {
857 log_if_needed(" ** klass argument not initialized");
858 return false;
859 }
860 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
861 if (!elem_type->is_primitive_type()) {
862 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
863 return false; // should be primitive type
864 }
865 BasicType elem_bt = elem_type->basic_type();
866 int num_elem = vlen->get_con();
867 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
953 // int length,
954 // Object base, long offset, // Unsafe addressing
955 // boolean fromSegment,
956 // C container, long index, S s, // Arguments for default implementation
957 // LoadOperation<C, VM, S> defaultImpl) {
958 // public static
959 // <C,
960 // V extends VectorPayload>
961 // void store(Class<?> vClass, Class<?> eClass,
962 // int length,
963 // Object base, long offset, // Unsafe addressing
964 // boolean fromSegment,
965 // V v, C container, long index, // Arguments for default implementation
966 // StoreVectorOperation<C, V> defaultImpl) {
967 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
968 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
969 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
970 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
971 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
972
973 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
974 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
975 vlen == nullptr || !vlen->is_con() ||
976 from_ms == nullptr || !from_ms->is_con()) {
977 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
978 NodeClassNames[argument(0)->Opcode()],
979 NodeClassNames[argument(1)->Opcode()],
980 NodeClassNames[argument(2)->Opcode()],
981 NodeClassNames[argument(6)->Opcode()]);
982 return false; // not enough info for intrinsification
983 }
984 if (!is_klass_initialized(vector_klass)) {
985 log_if_needed(" ** klass argument not initialized");
986 return false;
987 }
988
989 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
990 if (!elem_type->is_primitive_type()) {
991 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
992 return false; // should be primitive type
993 }
994 BasicType elem_bt = elem_type->basic_type();
995 int num_elem = vlen->get_con();
996
1163 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1164 // public static
1165 // <C,
1166 // V extends Vector<E>,
1167 // M extends VectorMask<E>,
1168 // E>
1169 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1170 // int length,
1171 // Object base, long offset, // Unsafe addressing
1172 // boolean fromSegment,
1173 // V v, M m, C container, long index, // Arguments for default implementation
1174 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
1175
1176 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1177 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1178 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1179 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1180 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1181 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
1182
1183 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1184 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1185 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1186 vlen == nullptr || !vlen->is_con() ||
1187 from_ms == nullptr || !from_ms->is_con()) {
1188 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1189 NodeClassNames[argument(0)->Opcode()],
1190 NodeClassNames[argument(1)->Opcode()],
1191 NodeClassNames[argument(2)->Opcode()],
1192 NodeClassNames[argument(3)->Opcode()],
1193 NodeClassNames[argument(7)->Opcode()]);
1194 return false; // not enough info for intrinsification
1195 }
1196 if (!is_klass_initialized(vector_klass)) {
1197 log_if_needed(" ** klass argument not initialized");
1198 return false;
1199 }
1200
1201 if (!is_klass_initialized(mask_klass)) {
1202 log_if_needed(" ** mask klass argument not initialized");
1203 return false;
1204 }
1205
1206 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1207 if (!elem_type->is_primitive_type()) {
1404 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1405 //
1406 // <C,
1407 // V extends Vector<E>,
1408 // W extends Vector<Integer>,
1409 // M extends VectorMask<E>,
1410 // E>
1411 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1412 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1413 // W index_vector, V v, M m,
1414 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1415 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1416 //
1417 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1418 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1419 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1420 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1421 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1422 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1423
1424 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1425 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1426 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1427 vlen == nullptr || !vlen->is_con() ||
1428 vector_idx_klass == nullptr || vector_idx_klass->const_oop() == nullptr) {
1429 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1430 NodeClassNames[argument(0)->Opcode()],
1431 NodeClassNames[argument(2)->Opcode()],
1432 NodeClassNames[argument(3)->Opcode()],
1433 NodeClassNames[argument(4)->Opcode()]);
1434 return false; // not enough info for intrinsification
1435 }
1436
1437 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1438 log_if_needed(" ** klass argument not initialized");
1439 return false;
1440 }
1441
1442 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1443 if (!elem_type->is_primitive_type()) {
1444 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1445 return false; // should be primitive type
1446 }
1447
1448 BasicType elem_bt = elem_type->basic_type();
1594 destruct_map_clone(old_map);
1595
1596 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1597 return true;
1598 }
1599
1600 // public static
1601 // <V extends Vector<E>,
1602 // M extends VectorMask<E>,
1603 // E>
1604 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1605 // Class<E> elementType, int length, V v, M m,
1606 // ReductionOperation<V, M> defaultImpl)
1607 bool LibraryCallKit::inline_vector_reduction() {
1608 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1609 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1610 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1611 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1612 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1613
1614 if (opr == nullptr || !opr->is_con() ||
1615 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1616 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1617 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1618 vlen == nullptr || !vlen->is_con()) {
1619 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1620 NodeClassNames[argument(0)->Opcode()],
1621 NodeClassNames[argument(1)->Opcode()],
1622 NodeClassNames[argument(3)->Opcode()],
1623 NodeClassNames[argument(4)->Opcode()]);
1624 return false; // not enough info for intrinsification
1625 }
1626 if (!is_klass_initialized(vector_klass)) {
1627 log_if_needed(" ** klass argument not initialized");
1628 return false;
1629 }
1630 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1631 if (!elem_type->is_primitive_type()) {
1632 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1633 return false; // should be primitive type
1634 }
1635
1636 const Type* vmask_type = gvn().type(argument(6));
1637 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1638 if (is_masked_op) {
1734 bits = value; // no conversion needed
1735 break;
1736 }
1737 default: fatal("%s", type2name(elem_bt));
1738 }
1739 set_result(bits);
1740 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1741 return true;
1742 }
1743
1744 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1745 // V v1, V v2,
1746 // BiFunction<V, V, Boolean> defaultImpl)
1747 //
1748 bool LibraryCallKit::inline_vector_test() {
1749 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1750 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1751 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1752 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1753
1754 if (cond == nullptr || !cond->is_con() ||
1755 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1756 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1757 vlen == nullptr || !vlen->is_con()) {
1758 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1759 NodeClassNames[argument(0)->Opcode()],
1760 NodeClassNames[argument(1)->Opcode()],
1761 NodeClassNames[argument(2)->Opcode()],
1762 NodeClassNames[argument(3)->Opcode()]);
1763 return false; // not enough info for intrinsification
1764 }
1765 if (!is_klass_initialized(vector_klass)) {
1766 log_if_needed(" ** klass argument not initialized");
1767 return false;
1768 }
1769 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1770 if (!elem_type->is_primitive_type()) {
1771 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1772 return false; // should be primitive type
1773 }
1774 BasicType elem_bt = elem_type->basic_type();
1775 int num_elem = vlen->get_con();
1776 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1777 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2749 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2750 return true;
2751 }
2752
2753 // public static
2754 // <VM extends VectorPayload,
2755 // E>
2756 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2757 // int length,
2758 // VM vm, int i,
2759 // VecExtractOp<VM> defaultImpl)
2760 bool LibraryCallKit::inline_vector_extract() {
2761 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2762 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2763 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2764 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2765
2766 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2767 return false; // dead code
2768 }
2769 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2770 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2771 NodeClassNames[argument(0)->Opcode()],
2772 NodeClassNames[argument(1)->Opcode()],
2773 NodeClassNames[argument(2)->Opcode()]);
2774 return false; // not enough info for intrinsification
2775 }
2776 if (!is_klass_initialized(vector_klass)) {
2777 log_if_needed(" ** klass argument not initialized");
2778 return false;
2779 }
2780 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2781 if (!elem_type->is_primitive_type()) {
2782 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2783 return false; // should be primitive type
2784 }
2785 BasicType elem_bt = elem_type->basic_type();
2786 int num_elem = vlen->get_con();
2787
2788 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2789 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2866 }
2867 set_result(opd);
2868 return true;
2869 }
2870
2871 // public static
2872 // <V extends Vector<E>,
2873 // M extends VectorMask<E>,
2874 // E>
2875 // V compressExpandOp(int opr,
2876 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2877 // int length, V v, M m,
2878 // CompressExpandOperation<V, M> defaultImpl)
2879 bool LibraryCallKit::inline_vector_compress_expand() {
2880 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2881 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2882 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2883 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2884 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2885
2886 if (opr == nullptr || !opr->is_con() ||
2887 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2888 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
2889 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2890 vlen == nullptr || !vlen->is_con()) {
2891 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2892 NodeClassNames[argument(0)->Opcode()],
2893 NodeClassNames[argument(1)->Opcode()],
2894 NodeClassNames[argument(2)->Opcode()],
2895 NodeClassNames[argument(3)->Opcode()],
2896 NodeClassNames[argument(4)->Opcode()]);
2897 return false; // not enough info for intrinsification
2898 }
2899
2900 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2901 log_if_needed(" ** klass argument not initialized");
2902 return false;
2903 }
2904
2905 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2906 if (!elem_type->is_primitive_type()) {
2907 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2908 return false; // should be primitive type
2909 }
2910
2949 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2950 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2951 set_result(vbox);
2952 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2953 return true;
2954 }
2955
2956 // public static
2957 // <V extends Vector<E>,
2958 // E,
2959 // S extends VectorSpecies<E>>
2960 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2961 // int length,
2962 // V v, int step, S s,
2963 // IndexOperation<V, S> defaultImpl)
2964 bool LibraryCallKit::inline_index_vector() {
2965 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2966 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2967 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2968
2969 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2970 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2971 vlen == nullptr || !vlen->is_con() ) {
2972 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2973 NodeClassNames[argument(0)->Opcode()],
2974 NodeClassNames[argument(1)->Opcode()],
2975 NodeClassNames[argument(2)->Opcode()]);
2976 return false; // not enough info for intrinsification
2977 }
2978
2979 if (!is_klass_initialized(vector_klass)) {
2980 log_if_needed(" ** klass argument not initialized");
2981 return false;
2982 }
2983
2984 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2985 if (!elem_type->is_primitive_type()) {
2986 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2987 return false; // should be primitive type
2988 }
2989
2990 int num_elem = vlen->get_con();
2991 BasicType elem_bt = elem_type->basic_type();
3083 if (needs_add) {
3084 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3085 }
3086 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3087 set_result(vbox);
3088 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3089 return true;
3090 }
3091
3092 // public static
3093 // <E,
3094 // M extends VectorMask<E>>
3095 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3096 // long offset, long limit,
3097 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3098 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3099 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
3100 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3101 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3102
3103 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
3104 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
3105 vlen == nullptr || !vlen->is_con()) {
3106 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
3107 NodeClassNames[argument(0)->Opcode()],
3108 NodeClassNames[argument(1)->Opcode()],
3109 NodeClassNames[argument(2)->Opcode()]);
3110 return false; // not enough info for intrinsification
3111 }
3112
3113 if (!is_klass_initialized(mask_klass)) {
3114 log_if_needed(" ** klass argument not initialized");
3115 return false;
3116 }
3117
3118 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3119 if (!elem_type->is_primitive_type()) {
3120 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3121 return false; // should be primitive type
3122 }
3123
3124 int num_elem = vlen->get_con();
3125 BasicType elem_bt = elem_type->basic_type();
|