306 // E>
307 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
308 // int length, V v1, V v2, M m,
309 // BinaryOperation<V, M> defaultImpl)
310 //
311 // public static
312 // <V extends Vector<E>,
313 // M extends VectorMask<E>,
314 // E>
315 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
316 // int length, V v1, V v2, V v3, M m,
317 // TernaryOperation<V, M> defaultImpl)
318 //
319 bool LibraryCallKit::inline_vector_nary_operation(int n) {
320 const TypeInt* opr = gvn().type(argument(0))->isa_int();
321 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
322 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
323 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
324 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
325
326 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
327 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
328 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
329 NodeClassNames[argument(0)->Opcode()],
330 NodeClassNames[argument(1)->Opcode()],
331 NodeClassNames[argument(3)->Opcode()],
332 NodeClassNames[argument(4)->Opcode()]);
333 return false; // not enough info for intrinsification
334 }
335
336 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
337 if (!elem_type->is_primitive_type()) {
338 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
339 return false; // should be primitive type
340 }
341 if (!is_klass_initialized(vector_klass)) {
342 log_if_needed(" ** klass argument not initialized");
343 return false;
344 }
345
346 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
347 // is used to control this operation.
536 const TypeVect* vmask_type = TypeVect::makemask(type_bt, num_elem);
537 Node* mask = gvn().transform(new VectorMaskCmpNode(pred, bcast_lane_cnt, index_vec, pred_node, vmask_type));
538
539 // Make the indices greater than lane count as -ve values to match the java side implementation.
540 index_vec = gvn().transform(VectorNode::make(Op_AndV, index_vec, bcast_mod_mask, vt));
541 Node* biased_val = gvn().transform(VectorNode::make(Op_SubVB, index_vec, bcast_lane_cnt, vt));
542 return gvn().transform(new VectorBlendNode(biased_val, index_vec, mask));
543 }
544
545 // <Sh extends VectorShuffle<E>, E>
546 // Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
547 // int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
548 bool LibraryCallKit::inline_vector_shuffle_iota() {
549 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
550 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
551 const TypeInt* start_val = gvn().type(argument(4))->isa_int();
552 const TypeInt* step_val = gvn().type(argument(5))->isa_int();
553 const TypeInt* wrap = gvn().type(argument(6))->isa_int();
554
555 if (shuffle_klass == nullptr || shuffle_klass->const_oop() == nullptr ||
556 vlen == nullptr || !vlen->is_con() || start_val == nullptr || step_val == nullptr ||
557 wrap == nullptr || !wrap->is_con()) {
558 return false; // not enough info for intrinsification
559 }
560
561 if (!is_klass_initialized(shuffle_klass)) {
562 log_if_needed(" ** klass argument not initialized");
563 return false;
564 }
565
566 int do_wrap = wrap->get_con();
567 int num_elem = vlen->get_con();
568 BasicType elem_bt = T_BYTE;
569
570 bool effective_indices_in_range = false;
571 if (start_val->is_con() && step_val->is_con()) {
572 int effective_min_index = start_val->get_con();
573 int effective_max_index = start_val->get_con() + step_val->get_con() * (num_elem - 1);
574 effective_indices_in_range = effective_max_index >= effective_min_index && effective_min_index >= -128 && effective_max_index <= 127;
575 }
576
577 if (!do_wrap && !effective_indices_in_range) {
635 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
636 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
637
638 // Wrap it up in VectorBox to keep object type information.
639 res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
640 set_result(res);
641 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
642 return true;
643 }
644
645 // <E, M>
646 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
647 // int length, M m, VectorMaskOp<M> defaultImpl)
648 bool LibraryCallKit::inline_vector_mask_operation() {
649 const TypeInt* oper = gvn().type(argument(0))->isa_int();
650 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
651 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
652 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
653 Node* mask = argument(4);
654
655 if (mask_klass == nullptr || elem_klass == nullptr || mask->is_top() || vlen == nullptr) {
656 return false; // dead code
657 }
658
659 if (!is_klass_initialized(mask_klass)) {
660 log_if_needed(" ** klass argument not initialized");
661 return false;
662 }
663
664 int num_elem = vlen->get_con();
665 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
666 BasicType elem_bt = elem_type->basic_type();
667
668 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
669 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
670 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
671 mopc, num_elem, type2name(elem_bt));
672 return false; // not supported
673 }
674
675 const Type* elem_ty = Type::get_const_basic_type(elem_bt);
696 return true;
697 }
698
699 // public static
700 // <V,
701 // Sh extends VectorShuffle<E>,
702 // E>
703 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
704 // Class<? extends Sh> shuffleClass, Sh s, int length,
705 // ShuffleToVectorOperation<V, Sh, E> defaultImpl)
706 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
707 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
708 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
709 const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
710 Node* shuffle = argument(3);
711 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
712
713 if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) {
714 return false; // dead code
715 }
716 if (!vlen->is_con() || vector_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr) {
717 return false; // not enough info for intrinsification
718 }
719 if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
720 log_if_needed(" ** klass argument not initialized");
721 return false;
722 }
723
724 int num_elem = vlen->get_con();
725 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
726 BasicType elem_bt = elem_type->basic_type();
727
728 if (num_elem < 4) {
729 return false;
730 }
731
732 int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE
733 // Make sure that cast is implemented to particular type/size combination.
734 if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
735 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
736 cast_vopc, num_elem, type2name(elem_bt));
757 return true;
758 }
759
760 // public static
761 // <M,
762 // S extends VectorSpecies<E>,
763 // E>
764 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
765 // long bits, int mode, S s,
766 // BroadcastOperation<M, E, S> defaultImpl)
767 bool LibraryCallKit::inline_vector_frombits_coerced() {
768 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
769 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
770 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
771 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
772 // Mode argument determines the mode of operation it can take following values:-
773 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
774 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
775 const TypeInt* mode = gvn().type(argument(5))->isa_int();
776
777 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || mode == nullptr ||
778 bits_type == nullptr || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr ||
779 !vlen->is_con() || !mode->is_con()) {
780 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
781 NodeClassNames[argument(0)->Opcode()],
782 NodeClassNames[argument(1)->Opcode()],
783 NodeClassNames[argument(2)->Opcode()],
784 NodeClassNames[argument(5)->Opcode()]);
785 return false; // not enough info for intrinsification
786 }
787
788 if (!is_klass_initialized(vector_klass)) {
789 log_if_needed(" ** klass argument not initialized");
790 return false;
791 }
792 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
793 if (!elem_type->is_primitive_type()) {
794 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
795 return false; // should be primitive type
796 }
797 BasicType elem_bt = elem_type->basic_type();
798 int num_elem = vlen->get_con();
799 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
885 // int length,
886 // Object base, long offset, // Unsafe addressing
887 // boolean fromSegment,
888 // C container, long index, S s, // Arguments for default implementation
889 // LoadOperation<C, VM, S> defaultImpl) {
890 // public static
891 // <C,
892 // V extends VectorPayload>
893 // void store(Class<?> vClass, Class<?> eClass,
894 // int length,
895 // Object base, long offset, // Unsafe addressing
896 // boolean fromSegment,
897 // V v, C container, long index, // Arguments for default implementation
898 // StoreVectorOperation<C, V> defaultImpl) {
899 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
900 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
901 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
902 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
903 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
904
905 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || !from_ms->is_con() ||
906 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
907 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
908 NodeClassNames[argument(0)->Opcode()],
909 NodeClassNames[argument(1)->Opcode()],
910 NodeClassNames[argument(2)->Opcode()],
911 NodeClassNames[argument(6)->Opcode()]);
912 return false; // not enough info for intrinsification
913 }
914 if (!is_klass_initialized(vector_klass)) {
915 log_if_needed(" ** klass argument not initialized");
916 return false;
917 }
918
919 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
920 if (!elem_type->is_primitive_type()) {
921 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
922 return false; // should be primitive type
923 }
924 BasicType elem_bt = elem_type->basic_type();
925 int num_elem = vlen->get_con();
926
1093 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1094 // public static
1095 // <C,
1096 // V extends Vector<E>,
1097 // M extends VectorMask<E>,
1098 // E>
1099 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1100 // int length,
1101 // Object base, long offset, // Unsafe addressing
1102 // boolean fromSegment,
1103 // V v, M m, C container, long index, // Arguments for default implementation
1104 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
1105
1106 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1107 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1108 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1109 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1110 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1111 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
1112
1113 if (vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1114 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr || from_ms == nullptr ||
1115 elem_klass->const_oop() == nullptr || !vlen->is_con() || !from_ms->is_con()) {
1116 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1117 NodeClassNames[argument(0)->Opcode()],
1118 NodeClassNames[argument(1)->Opcode()],
1119 NodeClassNames[argument(2)->Opcode()],
1120 NodeClassNames[argument(3)->Opcode()],
1121 NodeClassNames[argument(7)->Opcode()]);
1122 return false; // not enough info for intrinsification
1123 }
1124 if (!is_klass_initialized(vector_klass)) {
1125 log_if_needed(" ** klass argument not initialized");
1126 return false;
1127 }
1128
1129 if (!is_klass_initialized(mask_klass)) {
1130 log_if_needed(" ** mask klass argument not initialized");
1131 return false;
1132 }
1133
1134 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1135 if (!elem_type->is_primitive_type()) {
1332 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1333 //
1334 // <C,
1335 // V extends Vector<E>,
1336 // W extends Vector<Integer>,
1337 // M extends VectorMask<E>,
1338 // E>
1339 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1340 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1341 // W index_vector, V v, M m,
1342 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1343 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1344 //
1345 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1346 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1347 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1348 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1349 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1350 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1351
1352 if (vector_klass == nullptr || elem_klass == nullptr || vector_idx_klass == nullptr || vlen == nullptr ||
1353 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || vector_idx_klass->const_oop() == nullptr || !vlen->is_con()) {
1354 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1355 NodeClassNames[argument(0)->Opcode()],
1356 NodeClassNames[argument(2)->Opcode()],
1357 NodeClassNames[argument(3)->Opcode()],
1358 NodeClassNames[argument(4)->Opcode()]);
1359 return false; // not enough info for intrinsification
1360 }
1361
1362 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1363 log_if_needed(" ** klass argument not initialized");
1364 return false;
1365 }
1366
1367 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1368 if (!elem_type->is_primitive_type()) {
1369 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1370 return false; // should be primitive type
1371 }
1372
1373 BasicType elem_bt = elem_type->basic_type();
1519 destruct_map_clone(old_map);
1520
1521 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1522 return true;
1523 }
1524
1525 // public static
1526 // <V extends Vector<E>,
1527 // M extends VectorMask<E>,
1528 // E>
1529 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1530 // Class<E> elementType, int length, V v, M m,
1531 // ReductionOperation<V, M> defaultImpl)
1532 bool LibraryCallKit::inline_vector_reduction() {
1533 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1534 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1535 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1536 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1537 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1538
1539 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1540 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1541 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1542 NodeClassNames[argument(0)->Opcode()],
1543 NodeClassNames[argument(1)->Opcode()],
1544 NodeClassNames[argument(3)->Opcode()],
1545 NodeClassNames[argument(4)->Opcode()]);
1546 return false; // not enough info for intrinsification
1547 }
1548 if (!is_klass_initialized(vector_klass)) {
1549 log_if_needed(" ** klass argument not initialized");
1550 return false;
1551 }
1552 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1553 if (!elem_type->is_primitive_type()) {
1554 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1555 return false; // should be primitive type
1556 }
1557
1558 const Type* vmask_type = gvn().type(argument(6));
1559 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1560 if (is_masked_op) {
1656 bits = value; // no conversion needed
1657 break;
1658 }
1659 default: fatal("%s", type2name(elem_bt));
1660 }
1661 set_result(bits);
1662 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1663 return true;
1664 }
1665
1666 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1667 // V v1, V v2,
1668 // BiFunction<V, V, Boolean> defaultImpl)
1669 //
1670 bool LibraryCallKit::inline_vector_test() {
1671 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1672 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1673 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1674 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1675
1676 if (cond == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1677 !cond->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1678 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1679 NodeClassNames[argument(0)->Opcode()],
1680 NodeClassNames[argument(1)->Opcode()],
1681 NodeClassNames[argument(2)->Opcode()],
1682 NodeClassNames[argument(3)->Opcode()]);
1683 return false; // not enough info for intrinsification
1684 }
1685 if (!is_klass_initialized(vector_klass)) {
1686 log_if_needed(" ** klass argument not initialized");
1687 return false;
1688 }
1689 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1690 if (!elem_type->is_primitive_type()) {
1691 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1692 return false; // should be primitive type
1693 }
1694 BasicType elem_bt = elem_type->basic_type();
1695 int num_elem = vlen->get_con();
1696 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1697 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2514 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2515 return true;
2516 }
2517
2518 // public static
2519 // <VM extends VectorPayload,
2520 // E>
2521 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2522 // int length,
2523 // VM vm, int i,
2524 // VecExtractOp<VM> defaultImpl)
2525 bool LibraryCallKit::inline_vector_extract() {
2526 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2527 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2528 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2529 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2530
2531 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2532 return false; // dead code
2533 }
2534 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2535 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2536 NodeClassNames[argument(0)->Opcode()],
2537 NodeClassNames[argument(1)->Opcode()],
2538 NodeClassNames[argument(2)->Opcode()]);
2539 return false; // not enough info for intrinsification
2540 }
2541 if (!is_klass_initialized(vector_klass)) {
2542 log_if_needed(" ** klass argument not initialized");
2543 return false;
2544 }
2545 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2546 if (!elem_type->is_primitive_type()) {
2547 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2548 return false; // should be primitive type
2549 }
2550 BasicType elem_bt = elem_type->basic_type();
2551 int num_elem = vlen->get_con();
2552
2553 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2554 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2631 }
2632 set_result(opd);
2633 return true;
2634 }
2635
2636 // public static
2637 // <V extends Vector<E>,
2638 // M extends VectorMask<E>,
2639 // E>
2640 // V compressExpandOp(int opr,
2641 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2642 // int length, V v, M m,
2643 // CompressExpandOperation<V, M> defaultImpl)
2644 bool LibraryCallKit::inline_vector_compress_expand() {
2645 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2646 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2647 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2648 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2649 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2650
2651 if (vector_klass == nullptr || elem_klass == nullptr || mask_klass == nullptr || vlen == nullptr ||
2652 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
2653 elem_klass->const_oop() == nullptr || !vlen->is_con() || !opr->is_con()) {
2654 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2655 NodeClassNames[argument(0)->Opcode()],
2656 NodeClassNames[argument(1)->Opcode()],
2657 NodeClassNames[argument(2)->Opcode()],
2658 NodeClassNames[argument(3)->Opcode()],
2659 NodeClassNames[argument(4)->Opcode()]);
2660 return false; // not enough info for intrinsification
2661 }
2662
2663 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2664 log_if_needed(" ** klass argument not initialized");
2665 return false;
2666 }
2667
2668 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2669 if (!elem_type->is_primitive_type()) {
2670 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2671 return false; // should be primitive type
2672 }
2673
2712 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2713 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2714 set_result(vbox);
2715 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2716 return true;
2717 }
2718
2719 // public static
2720 // <V extends Vector<E>,
2721 // E,
2722 // S extends VectorSpecies<E>>
2723 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2724 // int length,
2725 // V v, int step, S s,
2726 // IndexOperation<V, S> defaultImpl)
2727 bool LibraryCallKit::inline_index_vector() {
2728 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2729 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2730 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2731
2732 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2733 vector_klass->const_oop() == nullptr || !vlen->is_con() ||
2734 elem_klass->const_oop() == nullptr) {
2735 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2736 NodeClassNames[argument(0)->Opcode()],
2737 NodeClassNames[argument(1)->Opcode()],
2738 NodeClassNames[argument(2)->Opcode()]);
2739 return false; // not enough info for intrinsification
2740 }
2741
2742 if (!is_klass_initialized(vector_klass)) {
2743 log_if_needed(" ** klass argument not initialized");
2744 return false;
2745 }
2746
2747 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2748 if (!elem_type->is_primitive_type()) {
2749 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2750 return false; // should be primitive type
2751 }
2752
2753 int num_elem = vlen->get_con();
2754 BasicType elem_bt = elem_type->basic_type();
2846 if (needs_add) {
2847 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
2848 }
2849 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
2850 set_result(vbox);
2851 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2852 return true;
2853 }
2854
2855 // public static
2856 // <E,
2857 // M extends VectorMask<E>>
2858 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
2859 // long offset, long limit,
2860 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
2861 bool LibraryCallKit::inline_index_partially_in_upper_range() {
2862 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
2863 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2864 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2865
2866 if (mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2867 mask_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2868 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
2869 NodeClassNames[argument(0)->Opcode()],
2870 NodeClassNames[argument(1)->Opcode()],
2871 NodeClassNames[argument(2)->Opcode()]);
2872 return false; // not enough info for intrinsification
2873 }
2874
2875 if (!is_klass_initialized(mask_klass)) {
2876 log_if_needed(" ** klass argument not initialized");
2877 return false;
2878 }
2879
2880 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2881 if (!elem_type->is_primitive_type()) {
2882 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2883 return false; // should be primitive type
2884 }
2885
2886 int num_elem = vlen->get_con();
2887 BasicType elem_bt = elem_type->basic_type();
|
306 // E>
307 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
308 // int length, V v1, V v2, M m,
309 // BinaryOperation<V, M> defaultImpl)
310 //
311 // public static
312 // <V extends Vector<E>,
313 // M extends VectorMask<E>,
314 // E>
315 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
316 // int length, V v1, V v2, V v3, M m,
317 // TernaryOperation<V, M> defaultImpl)
318 //
319 bool LibraryCallKit::inline_vector_nary_operation(int n) {
320 const TypeInt* opr = gvn().type(argument(0))->isa_int();
321 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
322 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
323 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
324 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
325
326 if (opr == nullptr || !opr->is_con() ||
327 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
328 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
329 vlen == nullptr || !vlen->is_con()) {
330 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
331 NodeClassNames[argument(0)->Opcode()],
332 NodeClassNames[argument(1)->Opcode()],
333 NodeClassNames[argument(3)->Opcode()],
334 NodeClassNames[argument(4)->Opcode()]);
335 return false; // not enough info for intrinsification
336 }
337
338 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
339 if (!elem_type->is_primitive_type()) {
340 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
341 return false; // should be primitive type
342 }
343 if (!is_klass_initialized(vector_klass)) {
344 log_if_needed(" ** klass argument not initialized");
345 return false;
346 }
347
348 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
349 // is used to control this operation.
538 const TypeVect* vmask_type = TypeVect::makemask(type_bt, num_elem);
539 Node* mask = gvn().transform(new VectorMaskCmpNode(pred, bcast_lane_cnt, index_vec, pred_node, vmask_type));
540
541 // Make the indices greater than lane count as -ve values to match the java side implementation.
542 index_vec = gvn().transform(VectorNode::make(Op_AndV, index_vec, bcast_mod_mask, vt));
543 Node* biased_val = gvn().transform(VectorNode::make(Op_SubVB, index_vec, bcast_lane_cnt, vt));
544 return gvn().transform(new VectorBlendNode(biased_val, index_vec, mask));
545 }
546
547 // <Sh extends VectorShuffle<E>, E>
548 // Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
549 // int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
550 bool LibraryCallKit::inline_vector_shuffle_iota() {
551 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
552 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
553 const TypeInt* start_val = gvn().type(argument(4))->isa_int();
554 const TypeInt* step_val = gvn().type(argument(5))->isa_int();
555 const TypeInt* wrap = gvn().type(argument(6))->isa_int();
556
557 if (shuffle_klass == nullptr || shuffle_klass->const_oop() == nullptr ||
558 vlen == nullptr || !vlen->is_con() ||
559 start_val == nullptr ||
560 step_val == nullptr ||
561 wrap == nullptr || !wrap->is_con()) {
562 return false; // not enough info for intrinsification
563 }
564
565 if (!is_klass_initialized(shuffle_klass)) {
566 log_if_needed(" ** klass argument not initialized");
567 return false;
568 }
569
570 int do_wrap = wrap->get_con();
571 int num_elem = vlen->get_con();
572 BasicType elem_bt = T_BYTE;
573
574 bool effective_indices_in_range = false;
575 if (start_val->is_con() && step_val->is_con()) {
576 int effective_min_index = start_val->get_con();
577 int effective_max_index = start_val->get_con() + step_val->get_con() * (num_elem - 1);
578 effective_indices_in_range = effective_max_index >= effective_min_index && effective_min_index >= -128 && effective_max_index <= 127;
579 }
580
581 if (!do_wrap && !effective_indices_in_range) {
639 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
640 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
641
642 // Wrap it up in VectorBox to keep object type information.
643 res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
644 set_result(res);
645 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
646 return true;
647 }
648
649 // <E, M>
650 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
651 // int length, M m, VectorMaskOp<M> defaultImpl)
652 bool LibraryCallKit::inline_vector_mask_operation() {
653 const TypeInt* oper = gvn().type(argument(0))->isa_int();
654 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
655 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
656 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
657 Node* mask = argument(4);
658
659 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
660 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
661 vlen == nullptr || !vlen->is_con() ||
662 oper == nullptr || !oper->is_con() ||
663 mask->is_top()) {
664 return false; // dead code
665 }
666
667 if (!is_klass_initialized(mask_klass)) {
668 log_if_needed(" ** klass argument not initialized");
669 return false;
670 }
671
672 int num_elem = vlen->get_con();
673 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
674 BasicType elem_bt = elem_type->basic_type();
675
676 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
677 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
678 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
679 mopc, num_elem, type2name(elem_bt));
680 return false; // not supported
681 }
682
683 const Type* elem_ty = Type::get_const_basic_type(elem_bt);
704 return true;
705 }
706
707 // public static
708 // <V,
709 // Sh extends VectorShuffle<E>,
710 // E>
711 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
712 // Class<? extends Sh> shuffleClass, Sh s, int length,
713 // ShuffleToVectorOperation<V, Sh, E> defaultImpl)
714 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
715 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
716 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
717 const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
718 Node* shuffle = argument(3);
719 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
720
721 if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) {
722 return false; // dead code
723 }
724 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr || !vlen->is_con()) {
725 return false; // not enough info for intrinsification
726 }
727 if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
728 log_if_needed(" ** klass argument not initialized");
729 return false;
730 }
731
732 int num_elem = vlen->get_con();
733 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
734 BasicType elem_bt = elem_type->basic_type();
735
736 if (num_elem < 4) {
737 return false;
738 }
739
740 int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE
741 // Make sure that cast is implemented to particular type/size combination.
742 if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
743 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
744 cast_vopc, num_elem, type2name(elem_bt));
765 return true;
766 }
767
768 // public static
769 // <M,
770 // S extends VectorSpecies<E>,
771 // E>
772 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
773 // long bits, int mode, S s,
774 // BroadcastOperation<M, E, S> defaultImpl)
775 bool LibraryCallKit::inline_vector_frombits_coerced() {
776 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
777 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
778 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
779 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
780 // Mode argument determines the mode of operation it can take following values:-
781 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
782 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
783 const TypeInt* mode = gvn().type(argument(5))->isa_int();
784
785 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
786 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
787 vlen == nullptr || !vlen->is_con() ||
788 bits_type == nullptr ||
789 mode == nullptr || !mode->is_con()) {
790 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
791 NodeClassNames[argument(0)->Opcode()],
792 NodeClassNames[argument(1)->Opcode()],
793 NodeClassNames[argument(2)->Opcode()],
794 NodeClassNames[argument(5)->Opcode()]);
795 return false; // not enough info for intrinsification
796 }
797
798 if (!is_klass_initialized(vector_klass)) {
799 log_if_needed(" ** klass argument not initialized");
800 return false;
801 }
802 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
803 if (!elem_type->is_primitive_type()) {
804 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
805 return false; // should be primitive type
806 }
807 BasicType elem_bt = elem_type->basic_type();
808 int num_elem = vlen->get_con();
809 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
895 // int length,
896 // Object base, long offset, // Unsafe addressing
897 // boolean fromSegment,
898 // C container, long index, S s, // Arguments for default implementation
899 // LoadOperation<C, VM, S> defaultImpl) {
900 // public static
901 // <C,
902 // V extends VectorPayload>
903 // void store(Class<?> vClass, Class<?> eClass,
904 // int length,
905 // Object base, long offset, // Unsafe addressing
906 // boolean fromSegment,
907 // V v, C container, long index, // Arguments for default implementation
908 // StoreVectorOperation<C, V> defaultImpl) {
909 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
910 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
911 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
912 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
913 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
914
915 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
916 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
917 vlen == nullptr || !vlen->is_con() ||
918 from_ms == nullptr || !from_ms->is_con()) {
919 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
920 NodeClassNames[argument(0)->Opcode()],
921 NodeClassNames[argument(1)->Opcode()],
922 NodeClassNames[argument(2)->Opcode()],
923 NodeClassNames[argument(6)->Opcode()]);
924 return false; // not enough info for intrinsification
925 }
926 if (!is_klass_initialized(vector_klass)) {
927 log_if_needed(" ** klass argument not initialized");
928 return false;
929 }
930
931 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
932 if (!elem_type->is_primitive_type()) {
933 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
934 return false; // should be primitive type
935 }
936 BasicType elem_bt = elem_type->basic_type();
937 int num_elem = vlen->get_con();
938
1105 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1106 // public static
1107 // <C,
1108 // V extends Vector<E>,
1109 // M extends VectorMask<E>,
1110 // E>
1111 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1112 // int length,
1113 // Object base, long offset, // Unsafe addressing
1114 // boolean fromSegment,
1115 // V v, M m, C container, long index, // Arguments for default implementation
1116 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
1117
1118 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1119 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1120 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1121 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1122 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1123 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
1124
1125 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1126 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1127 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1128 vlen == nullptr || !vlen->is_con() ||
1129 from_ms == nullptr || !from_ms->is_con()) {
1130 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1131 NodeClassNames[argument(0)->Opcode()],
1132 NodeClassNames[argument(1)->Opcode()],
1133 NodeClassNames[argument(2)->Opcode()],
1134 NodeClassNames[argument(3)->Opcode()],
1135 NodeClassNames[argument(7)->Opcode()]);
1136 return false; // not enough info for intrinsification
1137 }
1138 if (!is_klass_initialized(vector_klass)) {
1139 log_if_needed(" ** klass argument not initialized");
1140 return false;
1141 }
1142
1143 if (!is_klass_initialized(mask_klass)) {
1144 log_if_needed(" ** mask klass argument not initialized");
1145 return false;
1146 }
1147
1148 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1149 if (!elem_type->is_primitive_type()) {
1346 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1347 //
1348 // <C,
1349 // V extends Vector<E>,
1350 // W extends Vector<Integer>,
1351 // M extends VectorMask<E>,
1352 // E>
1353 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1354 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1355 // W index_vector, V v, M m,
1356 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1357 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1358 //
1359 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1360 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1361 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1362 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1363 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1364 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1365
1366 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1367 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1368 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1369 vlen == nullptr || !vlen->is_con() ||
1370 vector_idx_klass == nullptr || vector_idx_klass->const_oop() == nullptr) {
1371 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1372 NodeClassNames[argument(0)->Opcode()],
1373 NodeClassNames[argument(2)->Opcode()],
1374 NodeClassNames[argument(3)->Opcode()],
1375 NodeClassNames[argument(4)->Opcode()]);
1376 return false; // not enough info for intrinsification
1377 }
1378
1379 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1380 log_if_needed(" ** klass argument not initialized");
1381 return false;
1382 }
1383
1384 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1385 if (!elem_type->is_primitive_type()) {
1386 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1387 return false; // should be primitive type
1388 }
1389
1390 BasicType elem_bt = elem_type->basic_type();
1536 destruct_map_clone(old_map);
1537
1538 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1539 return true;
1540 }
1541
1542 // public static
1543 // <V extends Vector<E>,
1544 // M extends VectorMask<E>,
1545 // E>
1546 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1547 // Class<E> elementType, int length, V v, M m,
1548 // ReductionOperation<V, M> defaultImpl)
1549 bool LibraryCallKit::inline_vector_reduction() {
1550 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1551 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1552 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1553 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1554 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1555
1556 if (opr == nullptr || !opr->is_con() ||
1557 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1558 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1559 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1560 vlen == nullptr || !vlen->is_con()) {
1561 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1562 NodeClassNames[argument(0)->Opcode()],
1563 NodeClassNames[argument(1)->Opcode()],
1564 NodeClassNames[argument(3)->Opcode()],
1565 NodeClassNames[argument(4)->Opcode()]);
1566 return false; // not enough info for intrinsification
1567 }
1568 if (!is_klass_initialized(vector_klass)) {
1569 log_if_needed(" ** klass argument not initialized");
1570 return false;
1571 }
1572 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1573 if (!elem_type->is_primitive_type()) {
1574 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1575 return false; // should be primitive type
1576 }
1577
1578 const Type* vmask_type = gvn().type(argument(6));
1579 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1580 if (is_masked_op) {
1676 bits = value; // no conversion needed
1677 break;
1678 }
1679 default: fatal("%s", type2name(elem_bt));
1680 }
1681 set_result(bits);
1682 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1683 return true;
1684 }
1685
1686 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1687 // V v1, V v2,
1688 // BiFunction<V, V, Boolean> defaultImpl)
1689 //
1690 bool LibraryCallKit::inline_vector_test() {
1691 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1692 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1693 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1694 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1695
1696 if (cond == nullptr || !cond->is_con() ||
1697 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1698 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1699 vlen == nullptr || !vlen->is_con()) {
1700 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1701 NodeClassNames[argument(0)->Opcode()],
1702 NodeClassNames[argument(1)->Opcode()],
1703 NodeClassNames[argument(2)->Opcode()],
1704 NodeClassNames[argument(3)->Opcode()]);
1705 return false; // not enough info for intrinsification
1706 }
1707 if (!is_klass_initialized(vector_klass)) {
1708 log_if_needed(" ** klass argument not initialized");
1709 return false;
1710 }
1711 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1712 if (!elem_type->is_primitive_type()) {
1713 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1714 return false; // should be primitive type
1715 }
1716 BasicType elem_bt = elem_type->basic_type();
1717 int num_elem = vlen->get_con();
1718 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1719 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2536 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2537 return true;
2538 }
2539
2540 // public static
2541 // <VM extends VectorPayload,
2542 // E>
2543 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2544 // int length,
2545 // VM vm, int i,
2546 // VecExtractOp<VM> defaultImpl)
2547 bool LibraryCallKit::inline_vector_extract() {
2548 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2549 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2550 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2551 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2552
2553 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2554 return false; // dead code
2555 }
2556 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2557 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2558 NodeClassNames[argument(0)->Opcode()],
2559 NodeClassNames[argument(1)->Opcode()],
2560 NodeClassNames[argument(2)->Opcode()]);
2561 return false; // not enough info for intrinsification
2562 }
2563 if (!is_klass_initialized(vector_klass)) {
2564 log_if_needed(" ** klass argument not initialized");
2565 return false;
2566 }
2567 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2568 if (!elem_type->is_primitive_type()) {
2569 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2570 return false; // should be primitive type
2571 }
2572 BasicType elem_bt = elem_type->basic_type();
2573 int num_elem = vlen->get_con();
2574
2575 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2576 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2653 }
2654 set_result(opd);
2655 return true;
2656 }
2657
2658 // public static
2659 // <V extends Vector<E>,
2660 // M extends VectorMask<E>,
2661 // E>
2662 // V compressExpandOp(int opr,
2663 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2664 // int length, V v, M m,
2665 // CompressExpandOperation<V, M> defaultImpl)
2666 bool LibraryCallKit::inline_vector_compress_expand() {
2667 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2668 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2669 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2670 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2671 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2672
2673 if (opr == nullptr || !opr->is_con() ||
2674 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2675 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
2676 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2677 vlen == nullptr || !vlen->is_con()) {
2678 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2679 NodeClassNames[argument(0)->Opcode()],
2680 NodeClassNames[argument(1)->Opcode()],
2681 NodeClassNames[argument(2)->Opcode()],
2682 NodeClassNames[argument(3)->Opcode()],
2683 NodeClassNames[argument(4)->Opcode()]);
2684 return false; // not enough info for intrinsification
2685 }
2686
2687 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2688 log_if_needed(" ** klass argument not initialized");
2689 return false;
2690 }
2691
2692 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2693 if (!elem_type->is_primitive_type()) {
2694 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2695 return false; // should be primitive type
2696 }
2697
2736 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2737 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2738 set_result(vbox);
2739 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2740 return true;
2741 }
2742
2743 // public static
2744 // <V extends Vector<E>,
2745 // E,
2746 // S extends VectorSpecies<E>>
2747 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2748 // int length,
2749 // V v, int step, S s,
2750 // IndexOperation<V, S> defaultImpl)
2751 bool LibraryCallKit::inline_index_vector() {
2752 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2753 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2754 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2755
2756 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2757 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2758 vlen == nullptr || !vlen->is_con() ) {
2759 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2760 NodeClassNames[argument(0)->Opcode()],
2761 NodeClassNames[argument(1)->Opcode()],
2762 NodeClassNames[argument(2)->Opcode()]);
2763 return false; // not enough info for intrinsification
2764 }
2765
2766 if (!is_klass_initialized(vector_klass)) {
2767 log_if_needed(" ** klass argument not initialized");
2768 return false;
2769 }
2770
2771 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2772 if (!elem_type->is_primitive_type()) {
2773 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2774 return false; // should be primitive type
2775 }
2776
2777 int num_elem = vlen->get_con();
2778 BasicType elem_bt = elem_type->basic_type();
2870 if (needs_add) {
2871 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
2872 }
2873 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
2874 set_result(vbox);
2875 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2876 return true;
2877 }
2878
2879 // public static
2880 // <E,
2881 // M extends VectorMask<E>>
2882 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
2883 // long offset, long limit,
2884 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
2885 bool LibraryCallKit::inline_index_partially_in_upper_range() {
2886 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
2887 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2888 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2889
2890 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
2891 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2892 vlen == nullptr || !vlen->is_con()) {
2893 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
2894 NodeClassNames[argument(0)->Opcode()],
2895 NodeClassNames[argument(1)->Opcode()],
2896 NodeClassNames[argument(2)->Opcode()]);
2897 return false; // not enough info for intrinsification
2898 }
2899
2900 if (!is_klass_initialized(mask_klass)) {
2901 log_if_needed(" ** klass argument not initialized");
2902 return false;
2903 }
2904
2905 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2906 if (!elem_type->is_primitive_type()) {
2907 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2908 return false; // should be primitive type
2909 }
2910
2911 int num_elem = vlen->get_con();
2912 BasicType elem_bt = elem_type->basic_type();
|