306 // E>
307 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
308 // int length, V v1, V v2, M m,
309 // BinaryOperation<V, M> defaultImpl)
310 //
311 // public static
312 // <V extends Vector<E>,
313 // M extends VectorMask<E>,
314 // E>
315 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
316 // int length, V v1, V v2, V v3, M m,
317 // TernaryOperation<V, M> defaultImpl)
318 //
319 bool LibraryCallKit::inline_vector_nary_operation(int n) {
320 const TypeInt* opr = gvn().type(argument(0))->isa_int();
321 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
322 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
323 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
324 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
325
326 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
327 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
328 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
329 NodeClassNames[argument(0)->Opcode()],
330 NodeClassNames[argument(1)->Opcode()],
331 NodeClassNames[argument(3)->Opcode()],
332 NodeClassNames[argument(4)->Opcode()]);
333 return false; // not enough info for intrinsification
334 }
335
336 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
337 if (!elem_type->is_primitive_type()) {
338 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
339 return false; // should be primitive type
340 }
341 if (!is_klass_initialized(vector_klass)) {
342 log_if_needed(" ** klass argument not initialized");
343 return false;
344 }
345
346 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
347 // is used to control this operation.
538 const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
539 Node* mask = gvn().transform(new VectorMaskCmpNode(pred, bcast_lane_cnt, index_vec, pred_node, vmask_type));
540
541 // Make the indices greater than lane count as -ve values to match the java side implementation.
542 index_vec = gvn().transform(VectorNode::make(Op_AndV, index_vec, bcast_mod_mask, vt));
543 Node* biased_val = gvn().transform(VectorNode::make(Op_SubVB, index_vec, bcast_lane_cnt, vt));
544 return gvn().transform(new VectorBlendNode(biased_val, index_vec, mask));
545 }
546
547 // <Sh extends VectorShuffle<E>, E>
548 // Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
549 // int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
550 bool LibraryCallKit::inline_vector_shuffle_iota() {
551 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
552 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
553 const TypeInt* start_val = gvn().type(argument(4))->isa_int();
554 const TypeInt* step_val = gvn().type(argument(5))->isa_int();
555 const TypeInt* wrap = gvn().type(argument(6))->isa_int();
556
557 if (shuffle_klass == nullptr || shuffle_klass->const_oop() == nullptr ||
558 vlen == nullptr || !vlen->is_con() || start_val == nullptr || step_val == nullptr ||
559 wrap == nullptr || !wrap->is_con()) {
560 return false; // not enough info for intrinsification
561 }
562
563 if (!is_klass_initialized(shuffle_klass)) {
564 log_if_needed(" ** klass argument not initialized");
565 return false;
566 }
567
568 int do_wrap = wrap->get_con();
569 int num_elem = vlen->get_con();
570 BasicType elem_bt = T_BYTE;
571
572 bool effective_indices_in_range = false;
573 if (start_val->is_con() && step_val->is_con()) {
574 int effective_min_index = start_val->get_con();
575 int effective_max_index = start_val->get_con() + step_val->get_con() * (num_elem - 1);
576 effective_indices_in_range = effective_max_index >= effective_min_index && effective_min_index >= -128 && effective_max_index <= 127;
577 }
578
579 if (!do_wrap && !effective_indices_in_range) {
636 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
637 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
638
639 // Wrap it up in VectorBox to keep object type information.
640 res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
641 set_result(res);
642 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
643 return true;
644 }
645
646 // <E, M>
647 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
648 // int length, M m, VectorMaskOp<M> defaultImpl)
649 bool LibraryCallKit::inline_vector_mask_operation() {
650 const TypeInt* oper = gvn().type(argument(0))->isa_int();
651 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
652 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
653 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
654 Node* mask = argument(4);
655
656 if (mask_klass == nullptr || elem_klass == nullptr || mask->is_top() || vlen == nullptr) {
657 return false; // dead code
658 }
659
660 if (!is_klass_initialized(mask_klass)) {
661 log_if_needed(" ** klass argument not initialized");
662 return false;
663 }
664
665 int num_elem = vlen->get_con();
666 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
667 BasicType elem_bt = elem_type->basic_type();
668
669 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
670 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
671 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
672 mopc, num_elem, type2name(elem_bt));
673 return false; // not supported
674 }
675
676 const Type* elem_ty = Type::get_const_basic_type(elem_bt);
697 return true;
698 }
699
700 // public static
701 // <V,
702 // Sh extends VectorShuffle<E>,
703 // E>
704 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
705 // Class<? extends Sh> shuffleClass, Sh s, int length,
706 // ShuffleToVectorOperation<V, Sh, E> defaultImpl)
707 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
708 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
709 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
710 const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
711 Node* shuffle = argument(3);
712 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
713
714 if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) {
715 return false; // dead code
716 }
717 if (!vlen->is_con() || vector_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr) {
718 return false; // not enough info for intrinsification
719 }
720 if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
721 log_if_needed(" ** klass argument not initialized");
722 return false;
723 }
724
725 int num_elem = vlen->get_con();
726 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
727 BasicType elem_bt = elem_type->basic_type();
728
729 if (num_elem < 4) {
730 return false;
731 }
732
733 int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE
734 // Make sure that cast is implemented to particular type/size combination.
735 if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
736 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
737 cast_vopc, num_elem, type2name(elem_bt));
815 return true;
816 }
817
818 // public static
819 // <M,
820 // S extends VectorSpecies<E>,
821 // E>
822 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
823 // long bits, int mode, S s,
824 // BroadcastOperation<M, E, S> defaultImpl)
825 bool LibraryCallKit::inline_vector_frombits_coerced() {
826 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
827 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
828 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
829 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
830 // Mode argument determines the mode of operation it can take following values:-
831 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
832 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
833 const TypeInt* mode = gvn().type(argument(5))->isa_int();
834
835 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || mode == nullptr ||
836 bits_type == nullptr || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr ||
837 !vlen->is_con() || !mode->is_con()) {
838 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
839 NodeClassNames[argument(0)->Opcode()],
840 NodeClassNames[argument(1)->Opcode()],
841 NodeClassNames[argument(2)->Opcode()],
842 NodeClassNames[argument(5)->Opcode()]);
843 return false; // not enough info for intrinsification
844 }
845
846 if (!is_klass_initialized(vector_klass)) {
847 log_if_needed(" ** klass argument not initialized");
848 return false;
849 }
850 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
851 if (!elem_type->is_primitive_type()) {
852 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
853 return false; // should be primitive type
854 }
855 BasicType elem_bt = elem_type->basic_type();
856 int num_elem = vlen->get_con();
857 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
943 // int length,
944 // Object base, long offset, // Unsafe addressing
945 // boolean fromSegment,
946 // C container, long index, S s, // Arguments for default implementation
947 // LoadOperation<C, VM, S> defaultImpl) {
948 // public static
949 // <C,
950 // V extends VectorPayload>
951 // void store(Class<?> vClass, Class<?> eClass,
952 // int length,
953 // Object base, long offset, // Unsafe addressing
954 // boolean fromSegment,
955 // V v, C container, long index, // Arguments for default implementation
956 // StoreVectorOperation<C, V> defaultImpl) {
957 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
958 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
959 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
960 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
961 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
962
963 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || !from_ms->is_con() ||
964 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
965 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
966 NodeClassNames[argument(0)->Opcode()],
967 NodeClassNames[argument(1)->Opcode()],
968 NodeClassNames[argument(2)->Opcode()],
969 NodeClassNames[argument(6)->Opcode()]);
970 return false; // not enough info for intrinsification
971 }
972 if (!is_klass_initialized(vector_klass)) {
973 log_if_needed(" ** klass argument not initialized");
974 return false;
975 }
976
977 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
978 if (!elem_type->is_primitive_type()) {
979 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
980 return false; // should be primitive type
981 }
982 BasicType elem_bt = elem_type->basic_type();
983 int num_elem = vlen->get_con();
984
1151 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1152 // public static
1153 // <C,
1154 // V extends Vector<E>,
1155 // M extends VectorMask<E>,
1156 // E>
1157 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1158 // int length,
1159 // Object base, long offset, // Unsafe addressing
1160 // boolean fromSegment,
1161 // V v, M m, C container, long index, // Arguments for default implementation
1162 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
1163
1164 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1165 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1166 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1167 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1168 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1169 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
1170
1171 if (vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1172 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr || from_ms == nullptr ||
1173 elem_klass->const_oop() == nullptr || !vlen->is_con() || !from_ms->is_con()) {
1174 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1175 NodeClassNames[argument(0)->Opcode()],
1176 NodeClassNames[argument(1)->Opcode()],
1177 NodeClassNames[argument(2)->Opcode()],
1178 NodeClassNames[argument(3)->Opcode()],
1179 NodeClassNames[argument(7)->Opcode()]);
1180 return false; // not enough info for intrinsification
1181 }
1182 if (!is_klass_initialized(vector_klass)) {
1183 log_if_needed(" ** klass argument not initialized");
1184 return false;
1185 }
1186
1187 if (!is_klass_initialized(mask_klass)) {
1188 log_if_needed(" ** mask klass argument not initialized");
1189 return false;
1190 }
1191
1192 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1193 if (!elem_type->is_primitive_type()) {
1390 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1391 //
1392 // <C,
1393 // V extends Vector<E>,
1394 // W extends Vector<Integer>,
1395 // M extends VectorMask<E>,
1396 // E>
1397 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1398 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1399 // W index_vector, V v, M m,
1400 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1401 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1402 //
1403 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1404 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1405 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1406 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1407 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1408 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1409
1410 if (vector_klass == nullptr || elem_klass == nullptr || vector_idx_klass == nullptr || vlen == nullptr ||
1411 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || vector_idx_klass->const_oop() == nullptr || !vlen->is_con()) {
1412 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1413 NodeClassNames[argument(0)->Opcode()],
1414 NodeClassNames[argument(2)->Opcode()],
1415 NodeClassNames[argument(3)->Opcode()],
1416 NodeClassNames[argument(4)->Opcode()]);
1417 return false; // not enough info for intrinsification
1418 }
1419
1420 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1421 log_if_needed(" ** klass argument not initialized");
1422 return false;
1423 }
1424
1425 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1426 if (!elem_type->is_primitive_type()) {
1427 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1428 return false; // should be primitive type
1429 }
1430
1431 BasicType elem_bt = elem_type->basic_type();
1577 destruct_map_clone(old_map);
1578
1579 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1580 return true;
1581 }
1582
1583 // public static
1584 // <V extends Vector<E>,
1585 // M extends VectorMask<E>,
1586 // E>
1587 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1588 // Class<E> elementType, int length, V v, M m,
1589 // ReductionOperation<V, M> defaultImpl)
1590 bool LibraryCallKit::inline_vector_reduction() {
1591 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1592 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1593 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1594 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1595 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1596
1597 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1598 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1599 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1600 NodeClassNames[argument(0)->Opcode()],
1601 NodeClassNames[argument(1)->Opcode()],
1602 NodeClassNames[argument(3)->Opcode()],
1603 NodeClassNames[argument(4)->Opcode()]);
1604 return false; // not enough info for intrinsification
1605 }
1606 if (!is_klass_initialized(vector_klass)) {
1607 log_if_needed(" ** klass argument not initialized");
1608 return false;
1609 }
1610 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1611 if (!elem_type->is_primitive_type()) {
1612 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1613 return false; // should be primitive type
1614 }
1615
1616 const Type* vmask_type = gvn().type(argument(6));
1617 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1618 if (is_masked_op) {
1714 bits = value; // no conversion needed
1715 break;
1716 }
1717 default: fatal("%s", type2name(elem_bt));
1718 }
1719 set_result(bits);
1720 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1721 return true;
1722 }
1723
1724 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1725 // V v1, V v2,
1726 // BiFunction<V, V, Boolean> defaultImpl)
1727 //
1728 bool LibraryCallKit::inline_vector_test() {
1729 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1730 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1731 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1732 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1733
1734 if (cond == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1735 !cond->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1736 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1737 NodeClassNames[argument(0)->Opcode()],
1738 NodeClassNames[argument(1)->Opcode()],
1739 NodeClassNames[argument(2)->Opcode()],
1740 NodeClassNames[argument(3)->Opcode()]);
1741 return false; // not enough info for intrinsification
1742 }
1743 if (!is_klass_initialized(vector_klass)) {
1744 log_if_needed(" ** klass argument not initialized");
1745 return false;
1746 }
1747 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1748 if (!elem_type->is_primitive_type()) {
1749 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1750 return false; // should be primitive type
1751 }
1752 BasicType elem_bt = elem_type->basic_type();
1753 int num_elem = vlen->get_con();
1754 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1755 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2726 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2727 return true;
2728 }
2729
2730 // public static
2731 // <VM extends VectorPayload,
2732 // E>
2733 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2734 // int length,
2735 // VM vm, int i,
2736 // VecExtractOp<VM> defaultImpl)
2737 bool LibraryCallKit::inline_vector_extract() {
2738 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2739 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2740 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2741 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2742
2743 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2744 return false; // dead code
2745 }
2746 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2747 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2748 NodeClassNames[argument(0)->Opcode()],
2749 NodeClassNames[argument(1)->Opcode()],
2750 NodeClassNames[argument(2)->Opcode()]);
2751 return false; // not enough info for intrinsification
2752 }
2753 if (!is_klass_initialized(vector_klass)) {
2754 log_if_needed(" ** klass argument not initialized");
2755 return false;
2756 }
2757 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2758 if (!elem_type->is_primitive_type()) {
2759 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2760 return false; // should be primitive type
2761 }
2762 BasicType elem_bt = elem_type->basic_type();
2763 int num_elem = vlen->get_con();
2764
2765 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2766 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
3025 set_result(vbox);
3026 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3027 return true;
3028 }
3029
3030 // public static
3031 // <V extends Vector<E>,
3032 // M extends VectorMask<E>,
3033 // E>
3034 // V compressExpandOp(int opr,
3035 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
3036 // int length, V v, M m,
3037 // CompressExpandOperation<V, M> defaultImpl)
3038 bool LibraryCallKit::inline_vector_compress_expand() {
3039 const TypeInt* opr = gvn().type(argument(0))->isa_int();
3040 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
3041 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
3042 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
3043 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
3044
3045 if (vector_klass == nullptr || elem_klass == nullptr || mask_klass == nullptr || vlen == nullptr ||
3046 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
3047 elem_klass->const_oop() == nullptr || !vlen->is_con() || !opr->is_con()) {
3048 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
3049 NodeClassNames[argument(0)->Opcode()],
3050 NodeClassNames[argument(1)->Opcode()],
3051 NodeClassNames[argument(2)->Opcode()],
3052 NodeClassNames[argument(3)->Opcode()],
3053 NodeClassNames[argument(4)->Opcode()]);
3054 return false; // not enough info for intrinsification
3055 }
3056
3057 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
3058 log_if_needed(" ** klass argument not initialized");
3059 return false;
3060 }
3061
3062 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3063 if (!elem_type->is_primitive_type()) {
3064 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3065 return false; // should be primitive type
3066 }
3067
3106 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
3107 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
3108 set_result(vbox);
3109 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3110 return true;
3111 }
3112
3113 // public static
3114 // <V extends Vector<E>,
3115 // E,
3116 // S extends VectorSpecies<E>>
3117 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
3118 // int length,
3119 // V v, int step, S s,
3120 // IndexOperation<V, S> defaultImpl)
3121 bool LibraryCallKit::inline_index_vector() {
3122 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
3123 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3124 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3125
3126 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
3127 vector_klass->const_oop() == nullptr || !vlen->is_con() ||
3128 elem_klass->const_oop() == nullptr) {
3129 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
3130 NodeClassNames[argument(0)->Opcode()],
3131 NodeClassNames[argument(1)->Opcode()],
3132 NodeClassNames[argument(2)->Opcode()]);
3133 return false; // not enough info for intrinsification
3134 }
3135
3136 if (!is_klass_initialized(vector_klass)) {
3137 log_if_needed(" ** klass argument not initialized");
3138 return false;
3139 }
3140
3141 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3142 if (!elem_type->is_primitive_type()) {
3143 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3144 return false; // should be primitive type
3145 }
3146
3147 int num_elem = vlen->get_con();
3148 BasicType elem_bt = elem_type->basic_type();
3240 if (needs_add) {
3241 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3242 }
3243 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3244 set_result(vbox);
3245 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3246 return true;
3247 }
3248
3249 // public static
3250 // <E,
3251 // M extends VectorMask<E>>
3252 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3253 // long offset, long limit,
3254 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3255 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3256 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
3257 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3258 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3259
3260 if (mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
3261 mask_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
3262 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
3263 NodeClassNames[argument(0)->Opcode()],
3264 NodeClassNames[argument(1)->Opcode()],
3265 NodeClassNames[argument(2)->Opcode()]);
3266 return false; // not enough info for intrinsification
3267 }
3268
3269 if (!is_klass_initialized(mask_klass)) {
3270 log_if_needed(" ** klass argument not initialized");
3271 return false;
3272 }
3273
3274 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3275 if (!elem_type->is_primitive_type()) {
3276 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3277 return false; // should be primitive type
3278 }
3279
3280 int num_elem = vlen->get_con();
3281 BasicType elem_bt = elem_type->basic_type();
|
306 // E>
307 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
308 // int length, V v1, V v2, M m,
309 // BinaryOperation<V, M> defaultImpl)
310 //
311 // public static
312 // <V extends Vector<E>,
313 // M extends VectorMask<E>,
314 // E>
315 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
316 // int length, V v1, V v2, V v3, M m,
317 // TernaryOperation<V, M> defaultImpl)
318 //
319 bool LibraryCallKit::inline_vector_nary_operation(int n) {
320 const TypeInt* opr = gvn().type(argument(0))->isa_int();
321 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
322 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
323 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
324 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
325
326 if (opr == nullptr || !opr->is_con() ||
327 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
328 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
329 vlen == nullptr || !vlen->is_con()) {
330 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
331 NodeClassNames[argument(0)->Opcode()],
332 NodeClassNames[argument(1)->Opcode()],
333 NodeClassNames[argument(3)->Opcode()],
334 NodeClassNames[argument(4)->Opcode()]);
335 return false; // not enough info for intrinsification
336 }
337
338 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
339 if (!elem_type->is_primitive_type()) {
340 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
341 return false; // should be primitive type
342 }
343 if (!is_klass_initialized(vector_klass)) {
344 log_if_needed(" ** klass argument not initialized");
345 return false;
346 }
347
348 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
349 // is used to control this operation.
540 const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
541 Node* mask = gvn().transform(new VectorMaskCmpNode(pred, bcast_lane_cnt, index_vec, pred_node, vmask_type));
542
543 // Make the indices greater than lane count as -ve values to match the java side implementation.
544 index_vec = gvn().transform(VectorNode::make(Op_AndV, index_vec, bcast_mod_mask, vt));
545 Node* biased_val = gvn().transform(VectorNode::make(Op_SubVB, index_vec, bcast_lane_cnt, vt));
546 return gvn().transform(new VectorBlendNode(biased_val, index_vec, mask));
547 }
548
549 // <Sh extends VectorShuffle<E>, E>
550 // Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
551 // int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
552 bool LibraryCallKit::inline_vector_shuffle_iota() {
553 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
554 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
555 const TypeInt* start_val = gvn().type(argument(4))->isa_int();
556 const TypeInt* step_val = gvn().type(argument(5))->isa_int();
557 const TypeInt* wrap = gvn().type(argument(6))->isa_int();
558
559 if (shuffle_klass == nullptr || shuffle_klass->const_oop() == nullptr ||
560 vlen == nullptr || !vlen->is_con() ||
561 start_val == nullptr ||
562 step_val == nullptr ||
563 wrap == nullptr || !wrap->is_con()) {
564 return false; // not enough info for intrinsification
565 }
566
567 if (!is_klass_initialized(shuffle_klass)) {
568 log_if_needed(" ** klass argument not initialized");
569 return false;
570 }
571
572 int do_wrap = wrap->get_con();
573 int num_elem = vlen->get_con();
574 BasicType elem_bt = T_BYTE;
575
576 bool effective_indices_in_range = false;
577 if (start_val->is_con() && step_val->is_con()) {
578 int effective_min_index = start_val->get_con();
579 int effective_max_index = start_val->get_con() + step_val->get_con() * (num_elem - 1);
580 effective_indices_in_range = effective_max_index >= effective_min_index && effective_min_index >= -128 && effective_max_index <= 127;
581 }
582
583 if (!do_wrap && !effective_indices_in_range) {
640 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
641 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
642
643 // Wrap it up in VectorBox to keep object type information.
644 res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
645 set_result(res);
646 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
647 return true;
648 }
649
650 // <E, M>
651 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
652 // int length, M m, VectorMaskOp<M> defaultImpl)
653 bool LibraryCallKit::inline_vector_mask_operation() {
654 const TypeInt* oper = gvn().type(argument(0))->isa_int();
655 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
656 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
657 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
658 Node* mask = argument(4);
659
660 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
661 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
662 vlen == nullptr || !vlen->is_con() ||
663 oper == nullptr || !oper->is_con() ||
664 mask->is_top()) {
665 return false; // dead code
666 }
667
668 if (!is_klass_initialized(mask_klass)) {
669 log_if_needed(" ** klass argument not initialized");
670 return false;
671 }
672
673 int num_elem = vlen->get_con();
674 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
675 BasicType elem_bt = elem_type->basic_type();
676
677 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
678 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
679 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
680 mopc, num_elem, type2name(elem_bt));
681 return false; // not supported
682 }
683
684 const Type* elem_ty = Type::get_const_basic_type(elem_bt);
705 return true;
706 }
707
708 // public static
709 // <V,
710 // Sh extends VectorShuffle<E>,
711 // E>
712 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
713 // Class<? extends Sh> shuffleClass, Sh s, int length,
714 // ShuffleToVectorOperation<V, Sh, E> defaultImpl)
715 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
716 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
717 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
718 const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
719 Node* shuffle = argument(3);
720 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
721
722 if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) {
723 return false; // dead code
724 }
725 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr || !vlen->is_con()) {
726 return false; // not enough info for intrinsification
727 }
728 if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
729 log_if_needed(" ** klass argument not initialized");
730 return false;
731 }
732
733 int num_elem = vlen->get_con();
734 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
735 BasicType elem_bt = elem_type->basic_type();
736
737 if (num_elem < 4) {
738 return false;
739 }
740
741 int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE
742 // Make sure that cast is implemented to particular type/size combination.
743 if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
744 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
745 cast_vopc, num_elem, type2name(elem_bt));
823 return true;
824 }
825
826 // public static
827 // <M,
828 // S extends VectorSpecies<E>,
829 // E>
830 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
831 // long bits, int mode, S s,
832 // BroadcastOperation<M, E, S> defaultImpl)
833 bool LibraryCallKit::inline_vector_frombits_coerced() {
834 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
835 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
836 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
837 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
838 // Mode argument determines the mode of operation it can take following values:-
839 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
840 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
841 const TypeInt* mode = gvn().type(argument(5))->isa_int();
842
843 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
844 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
845 vlen == nullptr || !vlen->is_con() ||
846 bits_type == nullptr ||
847 mode == nullptr || !mode->is_con()) {
848 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
849 NodeClassNames[argument(0)->Opcode()],
850 NodeClassNames[argument(1)->Opcode()],
851 NodeClassNames[argument(2)->Opcode()],
852 NodeClassNames[argument(5)->Opcode()]);
853 return false; // not enough info for intrinsification
854 }
855
856 if (!is_klass_initialized(vector_klass)) {
857 log_if_needed(" ** klass argument not initialized");
858 return false;
859 }
860 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
861 if (!elem_type->is_primitive_type()) {
862 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
863 return false; // should be primitive type
864 }
865 BasicType elem_bt = elem_type->basic_type();
866 int num_elem = vlen->get_con();
867 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
953 // int length,
954 // Object base, long offset, // Unsafe addressing
955 // boolean fromSegment,
956 // C container, long index, S s, // Arguments for default implementation
957 // LoadOperation<C, VM, S> defaultImpl) {
958 // public static
959 // <C,
960 // V extends VectorPayload>
961 // void store(Class<?> vClass, Class<?> eClass,
962 // int length,
963 // Object base, long offset, // Unsafe addressing
964 // boolean fromSegment,
965 // V v, C container, long index, // Arguments for default implementation
966 // StoreVectorOperation<C, V> defaultImpl) {
967 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
968 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
969 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
970 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
971 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
972
973 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
974 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
975 vlen == nullptr || !vlen->is_con() ||
976 from_ms == nullptr || !from_ms->is_con()) {
977 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
978 NodeClassNames[argument(0)->Opcode()],
979 NodeClassNames[argument(1)->Opcode()],
980 NodeClassNames[argument(2)->Opcode()],
981 NodeClassNames[argument(6)->Opcode()]);
982 return false; // not enough info for intrinsification
983 }
984 if (!is_klass_initialized(vector_klass)) {
985 log_if_needed(" ** klass argument not initialized");
986 return false;
987 }
988
989 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
990 if (!elem_type->is_primitive_type()) {
991 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
992 return false; // should be primitive type
993 }
994 BasicType elem_bt = elem_type->basic_type();
995 int num_elem = vlen->get_con();
996
1163 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1164 // public static
1165 // <C,
1166 // V extends Vector<E>,
1167 // M extends VectorMask<E>,
1168 // E>
1169 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1170 // int length,
1171 // Object base, long offset, // Unsafe addressing
1172 // boolean fromSegment,
1173 // V v, M m, C container, long index, // Arguments for default implementation
1174 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
1175
1176 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1177 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1178 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1179 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1180 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1181 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
1182
1183 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1184 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1185 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1186 vlen == nullptr || !vlen->is_con() ||
1187 from_ms == nullptr || !from_ms->is_con()) {
1188 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1189 NodeClassNames[argument(0)->Opcode()],
1190 NodeClassNames[argument(1)->Opcode()],
1191 NodeClassNames[argument(2)->Opcode()],
1192 NodeClassNames[argument(3)->Opcode()],
1193 NodeClassNames[argument(7)->Opcode()]);
1194 return false; // not enough info for intrinsification
1195 }
1196 if (!is_klass_initialized(vector_klass)) {
1197 log_if_needed(" ** klass argument not initialized");
1198 return false;
1199 }
1200
1201 if (!is_klass_initialized(mask_klass)) {
1202 log_if_needed(" ** mask klass argument not initialized");
1203 return false;
1204 }
1205
1206 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1207 if (!elem_type->is_primitive_type()) {
1404 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1405 //
1406 // <C,
1407 // V extends Vector<E>,
1408 // W extends Vector<Integer>,
1409 // M extends VectorMask<E>,
1410 // E>
1411 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1412 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1413 // W index_vector, V v, M m,
1414 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1415 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1416 //
1417 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1418 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1419 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1420 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1421 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1422 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1423
1424 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1425 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1426 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1427 vlen == nullptr || !vlen->is_con() ||
1428 vector_idx_klass == nullptr || vector_idx_klass->const_oop() == nullptr) {
1429 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1430 NodeClassNames[argument(0)->Opcode()],
1431 NodeClassNames[argument(2)->Opcode()],
1432 NodeClassNames[argument(3)->Opcode()],
1433 NodeClassNames[argument(4)->Opcode()]);
1434 return false; // not enough info for intrinsification
1435 }
1436
1437 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1438 log_if_needed(" ** klass argument not initialized");
1439 return false;
1440 }
1441
1442 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1443 if (!elem_type->is_primitive_type()) {
1444 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1445 return false; // should be primitive type
1446 }
1447
1448 BasicType elem_bt = elem_type->basic_type();
1594 destruct_map_clone(old_map);
1595
1596 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1597 return true;
1598 }
1599
1600 // public static
1601 // <V extends Vector<E>,
1602 // M extends VectorMask<E>,
1603 // E>
1604 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1605 // Class<E> elementType, int length, V v, M m,
1606 // ReductionOperation<V, M> defaultImpl)
1607 bool LibraryCallKit::inline_vector_reduction() {
1608 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1609 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1610 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1611 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1612 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1613
1614 if (opr == nullptr || !opr->is_con() ||
1615 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1616 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1617 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1618 vlen == nullptr || !vlen->is_con()) {
1619 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1620 NodeClassNames[argument(0)->Opcode()],
1621 NodeClassNames[argument(1)->Opcode()],
1622 NodeClassNames[argument(3)->Opcode()],
1623 NodeClassNames[argument(4)->Opcode()]);
1624 return false; // not enough info for intrinsification
1625 }
1626 if (!is_klass_initialized(vector_klass)) {
1627 log_if_needed(" ** klass argument not initialized");
1628 return false;
1629 }
1630 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1631 if (!elem_type->is_primitive_type()) {
1632 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1633 return false; // should be primitive type
1634 }
1635
1636 const Type* vmask_type = gvn().type(argument(6));
1637 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1638 if (is_masked_op) {
1734 bits = value; // no conversion needed
1735 break;
1736 }
1737 default: fatal("%s", type2name(elem_bt));
1738 }
1739 set_result(bits);
1740 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1741 return true;
1742 }
1743
1744 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1745 // V v1, V v2,
1746 // BiFunction<V, V, Boolean> defaultImpl)
1747 //
1748 bool LibraryCallKit::inline_vector_test() {
1749 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1750 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1751 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1752 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1753
1754 if (cond == nullptr || !cond->is_con() ||
1755 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1756 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1757 vlen == nullptr || !vlen->is_con()) {
1758 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1759 NodeClassNames[argument(0)->Opcode()],
1760 NodeClassNames[argument(1)->Opcode()],
1761 NodeClassNames[argument(2)->Opcode()],
1762 NodeClassNames[argument(3)->Opcode()]);
1763 return false; // not enough info for intrinsification
1764 }
1765 if (!is_klass_initialized(vector_klass)) {
1766 log_if_needed(" ** klass argument not initialized");
1767 return false;
1768 }
1769 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1770 if (!elem_type->is_primitive_type()) {
1771 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1772 return false; // should be primitive type
1773 }
1774 BasicType elem_bt = elem_type->basic_type();
1775 int num_elem = vlen->get_con();
1776 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1777 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2748 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2749 return true;
2750 }
2751
2752 // public static
2753 // <VM extends VectorPayload,
2754 // E>
2755 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2756 // int length,
2757 // VM vm, int i,
2758 // VecExtractOp<VM> defaultImpl)
2759 bool LibraryCallKit::inline_vector_extract() {
2760 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2761 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2762 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2763 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2764
2765 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2766 return false; // dead code
2767 }
2768 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2769 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2770 NodeClassNames[argument(0)->Opcode()],
2771 NodeClassNames[argument(1)->Opcode()],
2772 NodeClassNames[argument(2)->Opcode()]);
2773 return false; // not enough info for intrinsification
2774 }
2775 if (!is_klass_initialized(vector_klass)) {
2776 log_if_needed(" ** klass argument not initialized");
2777 return false;
2778 }
2779 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2780 if (!elem_type->is_primitive_type()) {
2781 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2782 return false; // should be primitive type
2783 }
2784 BasicType elem_bt = elem_type->basic_type();
2785 int num_elem = vlen->get_con();
2786
2787 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2788 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
3047 set_result(vbox);
3048 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3049 return true;
3050 }
3051
3052 // public static
3053 // <V extends Vector<E>,
3054 // M extends VectorMask<E>,
3055 // E>
3056 // V compressExpandOp(int opr,
3057 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
3058 // int length, V v, M m,
3059 // CompressExpandOperation<V, M> defaultImpl)
3060 bool LibraryCallKit::inline_vector_compress_expand() {
3061 const TypeInt* opr = gvn().type(argument(0))->isa_int();
3062 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
3063 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
3064 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
3065 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
3066
3067 if (opr == nullptr || !opr->is_con() ||
3068 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
3069 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
3070 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
3071 vlen == nullptr || !vlen->is_con()) {
3072 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
3073 NodeClassNames[argument(0)->Opcode()],
3074 NodeClassNames[argument(1)->Opcode()],
3075 NodeClassNames[argument(2)->Opcode()],
3076 NodeClassNames[argument(3)->Opcode()],
3077 NodeClassNames[argument(4)->Opcode()]);
3078 return false; // not enough info for intrinsification
3079 }
3080
3081 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
3082 log_if_needed(" ** klass argument not initialized");
3083 return false;
3084 }
3085
3086 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3087 if (!elem_type->is_primitive_type()) {
3088 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3089 return false; // should be primitive type
3090 }
3091
3130 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
3131 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
3132 set_result(vbox);
3133 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3134 return true;
3135 }
3136
3137 // public static
3138 // <V extends Vector<E>,
3139 // E,
3140 // S extends VectorSpecies<E>>
3141 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
3142 // int length,
3143 // V v, int step, S s,
3144 // IndexOperation<V, S> defaultImpl)
3145 bool LibraryCallKit::inline_index_vector() {
3146 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
3147 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3148 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3149
3150 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
3151 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
3152 vlen == nullptr || !vlen->is_con() ) {
3153 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
3154 NodeClassNames[argument(0)->Opcode()],
3155 NodeClassNames[argument(1)->Opcode()],
3156 NodeClassNames[argument(2)->Opcode()]);
3157 return false; // not enough info for intrinsification
3158 }
3159
3160 if (!is_klass_initialized(vector_klass)) {
3161 log_if_needed(" ** klass argument not initialized");
3162 return false;
3163 }
3164
3165 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3166 if (!elem_type->is_primitive_type()) {
3167 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3168 return false; // should be primitive type
3169 }
3170
3171 int num_elem = vlen->get_con();
3172 BasicType elem_bt = elem_type->basic_type();
3264 if (needs_add) {
3265 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3266 }
3267 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3268 set_result(vbox);
3269 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3270 return true;
3271 }
3272
3273 // public static
3274 // <E,
3275 // M extends VectorMask<E>>
3276 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3277 // long offset, long limit,
3278 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3279 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3280 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
3281 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3282 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3283
3284 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
3285 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
3286 vlen == nullptr || !vlen->is_con()) {
3287 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
3288 NodeClassNames[argument(0)->Opcode()],
3289 NodeClassNames[argument(1)->Opcode()],
3290 NodeClassNames[argument(2)->Opcode()]);
3291 return false; // not enough info for intrinsification
3292 }
3293
3294 if (!is_klass_initialized(mask_klass)) {
3295 log_if_needed(" ** klass argument not initialized");
3296 return false;
3297 }
3298
3299 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3300 if (!elem_type->is_primitive_type()) {
3301 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3302 return false; // should be primitive type
3303 }
3304
3305 int num_elem = vlen->get_con();
3306 BasicType elem_bt = elem_type->basic_type();
|