301 // E>
302 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
303 // int length, V v1, V v2, M m,
304 // BinaryOperation<V, M> defaultImpl)
305 //
306 // public static
307 // <V extends Vector<E>,
308 // M extends VectorMask<E>,
309 // E>
310 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
311 // int length, V v1, V v2, V v3, M m,
312 // TernaryOperation<V, M> defaultImpl)
313 //
314 bool LibraryCallKit::inline_vector_nary_operation(int n) {
315 const TypeInt* opr = gvn().type(argument(0))->isa_int();
316 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
317 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
318 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
319 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
320
321 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
322 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
323 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
324 NodeClassNames[argument(0)->Opcode()],
325 NodeClassNames[argument(1)->Opcode()],
326 NodeClassNames[argument(3)->Opcode()],
327 NodeClassNames[argument(4)->Opcode()]);
328 return false; // not enough info for intrinsification
329 }
330
331 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
332 if (!elem_type->is_primitive_type()) {
333 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
334 return false; // should be primitive type
335 }
336 if (!is_klass_initialized(vector_klass)) {
337 log_if_needed(" ** klass argument not initialized");
338 return false;
339 }
340
341 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
342 // is used to control this operation.
570
571 vcall = gvn().transform(new ProjNode(gvn().transform(vcall), TypeFunc::Parms));
572
573 // Wrap it up in VectorBox to keep object type information.
574 Node* vbox = box_vector(vcall, vbox_type, elem_bt, num_elem);
575 set_result(vbox);
576 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
577 return true;
578 }
579
580 // <E, M>
581 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
582 // int length, M m, VectorMaskOp<M> defaultImpl)
583 bool LibraryCallKit::inline_vector_mask_operation() {
584 const TypeInt* oper = gvn().type(argument(0))->isa_int();
585 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
586 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
587 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
588 Node* mask = argument(4);
589
590 if (mask_klass == nullptr || elem_klass == nullptr || mask->is_top() || vlen == nullptr) {
591 return false; // dead code
592 }
593
594 if (!is_klass_initialized(mask_klass)) {
595 log_if_needed(" ** klass argument not initialized");
596 return false;
597 }
598
599 int num_elem = vlen->get_con();
600 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
601 BasicType elem_bt = elem_type->basic_type();
602
603 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
604 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
605 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
606 mopc, num_elem, type2name(elem_bt));
607 return false; // not supported
608 }
609
610 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
630 return true;
631 }
632
633 // public static
634 // <M,
635 // S extends VectorSpecies<E>,
636 // E>
637 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
638 // long bits, int mode, S s,
639 // BroadcastOperation<M, E, S> defaultImpl)
640 bool LibraryCallKit::inline_vector_frombits_coerced() {
641 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
642 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
643 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
644 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
645 // Mode argument determines the mode of operation it can take following values:-
646 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
647 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
648 const TypeInt* mode = gvn().type(argument(5))->isa_int();
649
650 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || mode == nullptr ||
651 bits_type == nullptr || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr ||
652 !vlen->is_con() || !mode->is_con()) {
653 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
654 NodeClassNames[argument(0)->Opcode()],
655 NodeClassNames[argument(1)->Opcode()],
656 NodeClassNames[argument(2)->Opcode()],
657 NodeClassNames[argument(5)->Opcode()]);
658 return false; // not enough info for intrinsification
659 }
660
661 if (!is_klass_initialized(vector_klass)) {
662 log_if_needed(" ** klass argument not initialized");
663 return false;
664 }
665 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
666 if (!elem_type->is_primitive_type()) {
667 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
668 return false; // should be primitive type
669 }
670 BasicType elem_bt = elem_type->basic_type();
671 int num_elem = vlen->get_con();
672 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
758 // int length,
759 // Object base, long offset, // Unsafe addressing
760 // boolean fromSegment,
761 // C container, long index, S s, // Arguments for default implementation
762 // LoadOperation<C, VM, S> defaultImpl) {
763 // public static
764 // <C,
765 // V extends VectorPayload>
766 // void store(Class<?> vClass, Class<?> eClass,
767 // int length,
768 // Object base, long offset, // Unsafe addressing
769 // boolean fromSegment,
770 // V v, C container, long index, // Arguments for default implementation
771 // StoreVectorOperation<C, V> defaultImpl) {
772 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
773 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
774 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
775 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
776 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
777
778 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || !from_ms->is_con() ||
779 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
780 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
781 NodeClassNames[argument(0)->Opcode()],
782 NodeClassNames[argument(1)->Opcode()],
783 NodeClassNames[argument(2)->Opcode()],
784 NodeClassNames[argument(6)->Opcode()]);
785 return false; // not enough info for intrinsification
786 }
787 if (!is_klass_initialized(vector_klass)) {
788 log_if_needed(" ** klass argument not initialized");
789 return false;
790 }
791
792 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
793 if (!elem_type->is_primitive_type()) {
794 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
795 return false; // should be primitive type
796 }
797 BasicType elem_bt = elem_type->basic_type();
798 int num_elem = vlen->get_con();
799
966 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
967 // public static
968 // <C,
969 // V extends Vector<E>,
970 // M extends VectorMask<E>,
971 // E>
972 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
973 // int length,
974 // Object base, long offset, // Unsafe addressing
975 // boolean fromSegment,
976 // V v, M m, C container, long index, // Arguments for default implementation
977 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
978
979 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
980 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
981 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
982 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
983 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
984 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
985
986 if (vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
987 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr || from_ms == nullptr ||
988 elem_klass->const_oop() == nullptr || !vlen->is_con() || !from_ms->is_con()) {
989 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
990 NodeClassNames[argument(0)->Opcode()],
991 NodeClassNames[argument(1)->Opcode()],
992 NodeClassNames[argument(2)->Opcode()],
993 NodeClassNames[argument(3)->Opcode()],
994 NodeClassNames[argument(7)->Opcode()]);
995 return false; // not enough info for intrinsification
996 }
997 if (!is_klass_initialized(vector_klass)) {
998 log_if_needed(" ** klass argument not initialized");
999 return false;
1000 }
1001
1002 if (!is_klass_initialized(mask_klass)) {
1003 log_if_needed(" ** mask klass argument not initialized");
1004 return false;
1005 }
1006
1007 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1008 if (!elem_type->is_primitive_type()) {
1205 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1206 //
1207 // <C,
1208 // V extends Vector<E>,
1209 // W extends Vector<Integer>,
1210 // M extends VectorMask<E>,
1211 // E>
1212 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1213 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1214 // W index_vector, V v, M m,
1215 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1216 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1217 //
1218 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1219 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1220 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1221 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1222 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1223 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1224
1225 if (vector_klass == nullptr || elem_klass == nullptr || vector_idx_klass == nullptr || vlen == nullptr ||
1226 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || vector_idx_klass->const_oop() == nullptr || !vlen->is_con()) {
1227 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1228 NodeClassNames[argument(0)->Opcode()],
1229 NodeClassNames[argument(2)->Opcode()],
1230 NodeClassNames[argument(3)->Opcode()],
1231 NodeClassNames[argument(4)->Opcode()]);
1232 return false; // not enough info for intrinsification
1233 }
1234
1235 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1236 log_if_needed(" ** klass argument not initialized");
1237 return false;
1238 }
1239
1240 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1241 if (!elem_type->is_primitive_type()) {
1242 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1243 return false; // should be primitive type
1244 }
1245
1246 BasicType elem_bt = elem_type->basic_type();
1392 destruct_map_clone(old_map);
1393
1394 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1395 return true;
1396 }
1397
1398 // public static
1399 // <V extends Vector<E>,
1400 // M extends VectorMask<E>,
1401 // E>
1402 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1403 // Class<E> elementType, int length, V v, M m,
1404 // ReductionOperation<V, M> defaultImpl)
1405 bool LibraryCallKit::inline_vector_reduction() {
1406 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1407 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1408 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1409 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1410 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1411
1412 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1413 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1414 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1415 NodeClassNames[argument(0)->Opcode()],
1416 NodeClassNames[argument(1)->Opcode()],
1417 NodeClassNames[argument(3)->Opcode()],
1418 NodeClassNames[argument(4)->Opcode()]);
1419 return false; // not enough info for intrinsification
1420 }
1421 if (!is_klass_initialized(vector_klass)) {
1422 log_if_needed(" ** klass argument not initialized");
1423 return false;
1424 }
1425 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1426 if (!elem_type->is_primitive_type()) {
1427 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1428 return false; // should be primitive type
1429 }
1430
1431 const Type* vmask_type = gvn().type(argument(6));
1432 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1433 if (is_masked_op) {
1530 bits = value; // no conversion needed
1531 break;
1532 }
1533 default: fatal("%s", type2name(elem_bt));
1534 }
1535 set_result(bits);
1536 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1537 return true;
1538 }
1539
1540 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1541 // V v1, V v2,
1542 // BiFunction<V, V, Boolean> defaultImpl)
1543 //
1544 bool LibraryCallKit::inline_vector_test() {
1545 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1546 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1547 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1548 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1549
1550 if (cond == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1551 !cond->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1552 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1553 NodeClassNames[argument(0)->Opcode()],
1554 NodeClassNames[argument(1)->Opcode()],
1555 NodeClassNames[argument(2)->Opcode()],
1556 NodeClassNames[argument(3)->Opcode()]);
1557 return false; // not enough info for intrinsification
1558 }
1559 if (!is_klass_initialized(vector_klass)) {
1560 log_if_needed(" ** klass argument not initialized");
1561 return false;
1562 }
1563 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1564 if (!elem_type->is_primitive_type()) {
1565 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1566 return false; // should be primitive type
1567 }
1568 BasicType elem_bt = elem_type->basic_type();
1569 int num_elem = vlen->get_con();
1570 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1571 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2491 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2492 return true;
2493 }
2494
2495 // public static
2496 // <VM extends VectorPayload,
2497 // E>
2498 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2499 // int length,
2500 // VM vm, int i,
2501 // VecExtractOp<VM> defaultImpl)
2502 bool LibraryCallKit::inline_vector_extract() {
2503 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2504 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2505 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2506 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2507
2508 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2509 return false; // dead code
2510 }
2511 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2512 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2513 NodeClassNames[argument(0)->Opcode()],
2514 NodeClassNames[argument(1)->Opcode()],
2515 NodeClassNames[argument(2)->Opcode()]);
2516 return false; // not enough info for intrinsification
2517 }
2518 if (!is_klass_initialized(vector_klass)) {
2519 log_if_needed(" ** klass argument not initialized");
2520 return false;
2521 }
2522 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2523 if (!elem_type->is_primitive_type()) {
2524 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2525 return false; // should be primitive type
2526 }
2527 BasicType elem_bt = elem_type->basic_type();
2528 int num_elem = vlen->get_con();
2529
2530 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2531 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2794 set_result(vbox);
2795 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2796 return true;
2797 }
2798
2799 // public static
2800 // <V extends Vector<E>,
2801 // M extends VectorMask<E>,
2802 // E>
2803 // V compressExpandOp(int opr,
2804 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2805 // int length, V v, M m,
2806 // CompressExpandOperation<V, M> defaultImpl)
2807 bool LibraryCallKit::inline_vector_compress_expand() {
2808 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2809 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2810 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2811 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2812 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2813
2814 if (vector_klass == nullptr || elem_klass == nullptr || mask_klass == nullptr || vlen == nullptr ||
2815 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
2816 elem_klass->const_oop() == nullptr || !vlen->is_con() || !opr->is_con()) {
2817 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2818 NodeClassNames[argument(0)->Opcode()],
2819 NodeClassNames[argument(1)->Opcode()],
2820 NodeClassNames[argument(2)->Opcode()],
2821 NodeClassNames[argument(3)->Opcode()],
2822 NodeClassNames[argument(4)->Opcode()]);
2823 return false; // not enough info for intrinsification
2824 }
2825
2826 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2827 log_if_needed(" ** klass argument not initialized");
2828 return false;
2829 }
2830
2831 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2832 if (!elem_type->is_primitive_type()) {
2833 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2834 return false; // should be primitive type
2835 }
2836
2875 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2876 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2877 set_result(vbox);
2878 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2879 return true;
2880 }
2881
2882 // public static
2883 // <V extends Vector<E>,
2884 // E,
2885 // S extends VectorSpecies<E>>
2886 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2887 // int length,
2888 // V v, int step, S s,
2889 // IndexOperation<V, S> defaultImpl)
2890 bool LibraryCallKit::inline_index_vector() {
2891 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2892 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2893 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2894
2895 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2896 vector_klass->const_oop() == nullptr || !vlen->is_con() ||
2897 elem_klass->const_oop() == nullptr) {
2898 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2899 NodeClassNames[argument(0)->Opcode()],
2900 NodeClassNames[argument(1)->Opcode()],
2901 NodeClassNames[argument(2)->Opcode()]);
2902 return false; // not enough info for intrinsification
2903 }
2904
2905 if (!is_klass_initialized(vector_klass)) {
2906 log_if_needed(" ** klass argument not initialized");
2907 return false;
2908 }
2909
2910 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2911 if (!elem_type->is_primitive_type()) {
2912 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2913 return false; // should be primitive type
2914 }
2915
2916 int num_elem = vlen->get_con();
2917 BasicType elem_bt = elem_type->basic_type();
3009 if (needs_add) {
3010 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3011 }
3012 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3013 set_result(vbox);
3014 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3015 return true;
3016 }
3017
3018 // public static
3019 // <E,
3020 // M extends VectorMask<E>>
3021 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3022 // long offset, long limit,
3023 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3024 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3025 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
3026 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3027 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3028
3029 if (mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
3030 mask_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
3031 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
3032 NodeClassNames[argument(0)->Opcode()],
3033 NodeClassNames[argument(1)->Opcode()],
3034 NodeClassNames[argument(2)->Opcode()]);
3035 return false; // not enough info for intrinsification
3036 }
3037
3038 if (!is_klass_initialized(mask_klass)) {
3039 log_if_needed(" ** klass argument not initialized");
3040 return false;
3041 }
3042
3043 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3044 if (!elem_type->is_primitive_type()) {
3045 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3046 return false; // should be primitive type
3047 }
3048
3049 int num_elem = vlen->get_con();
3050 BasicType elem_bt = elem_type->basic_type();
|
301 // E>
302 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
303 // int length, V v1, V v2, M m,
304 // BinaryOperation<V, M> defaultImpl)
305 //
306 // public static
307 // <V extends Vector<E>,
308 // M extends VectorMask<E>,
309 // E>
310 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
311 // int length, V v1, V v2, V v3, M m,
312 // TernaryOperation<V, M> defaultImpl)
313 //
314 bool LibraryCallKit::inline_vector_nary_operation(int n) {
315 const TypeInt* opr = gvn().type(argument(0))->isa_int();
316 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
317 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
318 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
319 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
320
321 if (opr == nullptr || !opr->is_con() ||
322 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
323 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
324 vlen == nullptr || !vlen->is_con()) {
325 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
326 NodeClassNames[argument(0)->Opcode()],
327 NodeClassNames[argument(1)->Opcode()],
328 NodeClassNames[argument(3)->Opcode()],
329 NodeClassNames[argument(4)->Opcode()]);
330 return false; // not enough info for intrinsification
331 }
332
333 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
334 if (!elem_type->is_primitive_type()) {
335 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
336 return false; // should be primitive type
337 }
338 if (!is_klass_initialized(vector_klass)) {
339 log_if_needed(" ** klass argument not initialized");
340 return false;
341 }
342
343 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
344 // is used to control this operation.
572
573 vcall = gvn().transform(new ProjNode(gvn().transform(vcall), TypeFunc::Parms));
574
575 // Wrap it up in VectorBox to keep object type information.
576 Node* vbox = box_vector(vcall, vbox_type, elem_bt, num_elem);
577 set_result(vbox);
578 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
579 return true;
580 }
581
582 // <E, M>
583 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
584 // int length, M m, VectorMaskOp<M> defaultImpl)
585 bool LibraryCallKit::inline_vector_mask_operation() {
586 const TypeInt* oper = gvn().type(argument(0))->isa_int();
587 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
588 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
589 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
590 Node* mask = argument(4);
591
592 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
593 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
594 vlen == nullptr || !vlen->is_con() ||
595 oper == nullptr || !oper->is_con() ||
596 mask->is_top()) {
597 return false; // dead code
598 }
599
600 if (!is_klass_initialized(mask_klass)) {
601 log_if_needed(" ** klass argument not initialized");
602 return false;
603 }
604
605 int num_elem = vlen->get_con();
606 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
607 BasicType elem_bt = elem_type->basic_type();
608
609 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
610 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
611 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
612 mopc, num_elem, type2name(elem_bt));
613 return false; // not supported
614 }
615
616 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
636 return true;
637 }
638
639 // public static
640 // <M,
641 // S extends VectorSpecies<E>,
642 // E>
643 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
644 // long bits, int mode, S s,
645 // BroadcastOperation<M, E, S> defaultImpl)
646 bool LibraryCallKit::inline_vector_frombits_coerced() {
647 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
648 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
649 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
650 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
651 // Mode argument determines the mode of operation it can take following values:-
652 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
653 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
654 const TypeInt* mode = gvn().type(argument(5))->isa_int();
655
656 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
657 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
658 vlen == nullptr || !vlen->is_con() ||
659 bits_type == nullptr ||
660 mode == nullptr || !mode->is_con()) {
661 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
662 NodeClassNames[argument(0)->Opcode()],
663 NodeClassNames[argument(1)->Opcode()],
664 NodeClassNames[argument(2)->Opcode()],
665 NodeClassNames[argument(5)->Opcode()]);
666 return false; // not enough info for intrinsification
667 }
668
669 if (!is_klass_initialized(vector_klass)) {
670 log_if_needed(" ** klass argument not initialized");
671 return false;
672 }
673 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
674 if (!elem_type->is_primitive_type()) {
675 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
676 return false; // should be primitive type
677 }
678 BasicType elem_bt = elem_type->basic_type();
679 int num_elem = vlen->get_con();
680 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
766 // int length,
767 // Object base, long offset, // Unsafe addressing
768 // boolean fromSegment,
769 // C container, long index, S s, // Arguments for default implementation
770 // LoadOperation<C, VM, S> defaultImpl) {
771 // public static
772 // <C,
773 // V extends VectorPayload>
774 // void store(Class<?> vClass, Class<?> eClass,
775 // int length,
776 // Object base, long offset, // Unsafe addressing
777 // boolean fromSegment,
778 // V v, C container, long index, // Arguments for default implementation
779 // StoreVectorOperation<C, V> defaultImpl) {
780 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
781 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
782 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
783 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
784 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
785
786 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
787 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
788 vlen == nullptr || !vlen->is_con() ||
789 from_ms == nullptr || !from_ms->is_con()) {
790 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
791 NodeClassNames[argument(0)->Opcode()],
792 NodeClassNames[argument(1)->Opcode()],
793 NodeClassNames[argument(2)->Opcode()],
794 NodeClassNames[argument(6)->Opcode()]);
795 return false; // not enough info for intrinsification
796 }
797 if (!is_klass_initialized(vector_klass)) {
798 log_if_needed(" ** klass argument not initialized");
799 return false;
800 }
801
802 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
803 if (!elem_type->is_primitive_type()) {
804 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
805 return false; // should be primitive type
806 }
807 BasicType elem_bt = elem_type->basic_type();
808 int num_elem = vlen->get_con();
809
976 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
977 // public static
978 // <C,
979 // V extends Vector<E>,
980 // M extends VectorMask<E>,
981 // E>
982 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
983 // int length,
984 // Object base, long offset, // Unsafe addressing
985 // boolean fromSegment,
986 // V v, M m, C container, long index, // Arguments for default implementation
987 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
988
989 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
990 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
991 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
992 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
993 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
994 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
995
996 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
997 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
998 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
999 vlen == nullptr || !vlen->is_con() ||
1000 from_ms == nullptr || !from_ms->is_con()) {
1001 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1002 NodeClassNames[argument(0)->Opcode()],
1003 NodeClassNames[argument(1)->Opcode()],
1004 NodeClassNames[argument(2)->Opcode()],
1005 NodeClassNames[argument(3)->Opcode()],
1006 NodeClassNames[argument(7)->Opcode()]);
1007 return false; // not enough info for intrinsification
1008 }
1009 if (!is_klass_initialized(vector_klass)) {
1010 log_if_needed(" ** klass argument not initialized");
1011 return false;
1012 }
1013
1014 if (!is_klass_initialized(mask_klass)) {
1015 log_if_needed(" ** mask klass argument not initialized");
1016 return false;
1017 }
1018
1019 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1020 if (!elem_type->is_primitive_type()) {
1217 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1218 //
1219 // <C,
1220 // V extends Vector<E>,
1221 // W extends Vector<Integer>,
1222 // M extends VectorMask<E>,
1223 // E>
1224 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1225 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1226 // W index_vector, V v, M m,
1227 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1228 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1229 //
1230 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1231 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1232 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1233 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1234 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1235 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1236
1237 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1238 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1239 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1240 vlen == nullptr || !vlen->is_con() ||
1241 vector_idx_klass == nullptr || vector_idx_klass->const_oop() == nullptr) {
1242 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1243 NodeClassNames[argument(0)->Opcode()],
1244 NodeClassNames[argument(2)->Opcode()],
1245 NodeClassNames[argument(3)->Opcode()],
1246 NodeClassNames[argument(4)->Opcode()]);
1247 return false; // not enough info for intrinsification
1248 }
1249
1250 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1251 log_if_needed(" ** klass argument not initialized");
1252 return false;
1253 }
1254
1255 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1256 if (!elem_type->is_primitive_type()) {
1257 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1258 return false; // should be primitive type
1259 }
1260
1261 BasicType elem_bt = elem_type->basic_type();
1407 destruct_map_clone(old_map);
1408
1409 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1410 return true;
1411 }
1412
1413 // public static
1414 // <V extends Vector<E>,
1415 // M extends VectorMask<E>,
1416 // E>
1417 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1418 // Class<E> elementType, int length, V v, M m,
1419 // ReductionOperation<V, M> defaultImpl)
1420 bool LibraryCallKit::inline_vector_reduction() {
1421 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1422 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1423 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1424 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1425 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1426
1427 if (opr == nullptr || !opr->is_con() ||
1428 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1429 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1430 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1431 vlen == nullptr || !vlen->is_con()) {
1432 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1433 NodeClassNames[argument(0)->Opcode()],
1434 NodeClassNames[argument(1)->Opcode()],
1435 NodeClassNames[argument(3)->Opcode()],
1436 NodeClassNames[argument(4)->Opcode()]);
1437 return false; // not enough info for intrinsification
1438 }
1439 if (!is_klass_initialized(vector_klass)) {
1440 log_if_needed(" ** klass argument not initialized");
1441 return false;
1442 }
1443 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1444 if (!elem_type->is_primitive_type()) {
1445 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1446 return false; // should be primitive type
1447 }
1448
1449 const Type* vmask_type = gvn().type(argument(6));
1450 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1451 if (is_masked_op) {
1548 bits = value; // no conversion needed
1549 break;
1550 }
1551 default: fatal("%s", type2name(elem_bt));
1552 }
1553 set_result(bits);
1554 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1555 return true;
1556 }
1557
1558 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1559 // V v1, V v2,
1560 // BiFunction<V, V, Boolean> defaultImpl)
1561 //
1562 bool LibraryCallKit::inline_vector_test() {
1563 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1564 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1565 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1566 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1567
1568 if (cond == nullptr || !cond->is_con() ||
1569 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1570 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1571 vlen == nullptr || !vlen->is_con()) {
1572 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1573 NodeClassNames[argument(0)->Opcode()],
1574 NodeClassNames[argument(1)->Opcode()],
1575 NodeClassNames[argument(2)->Opcode()],
1576 NodeClassNames[argument(3)->Opcode()]);
1577 return false; // not enough info for intrinsification
1578 }
1579 if (!is_klass_initialized(vector_klass)) {
1580 log_if_needed(" ** klass argument not initialized");
1581 return false;
1582 }
1583 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1584 if (!elem_type->is_primitive_type()) {
1585 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1586 return false; // should be primitive type
1587 }
1588 BasicType elem_bt = elem_type->basic_type();
1589 int num_elem = vlen->get_con();
1590 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1591 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2511 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2512 return true;
2513 }
2514
2515 // public static
2516 // <VM extends VectorPayload,
2517 // E>
2518 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2519 // int length,
2520 // VM vm, int i,
2521 // VecExtractOp<VM> defaultImpl)
2522 bool LibraryCallKit::inline_vector_extract() {
2523 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2524 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2525 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2526 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2527
2528 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2529 return false; // dead code
2530 }
2531 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2532 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2533 NodeClassNames[argument(0)->Opcode()],
2534 NodeClassNames[argument(1)->Opcode()],
2535 NodeClassNames[argument(2)->Opcode()]);
2536 return false; // not enough info for intrinsification
2537 }
2538 if (!is_klass_initialized(vector_klass)) {
2539 log_if_needed(" ** klass argument not initialized");
2540 return false;
2541 }
2542 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2543 if (!elem_type->is_primitive_type()) {
2544 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2545 return false; // should be primitive type
2546 }
2547 BasicType elem_bt = elem_type->basic_type();
2548 int num_elem = vlen->get_con();
2549
2550 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2551 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2814 set_result(vbox);
2815 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2816 return true;
2817 }
2818
2819 // public static
2820 // <V extends Vector<E>,
2821 // M extends VectorMask<E>,
2822 // E>
2823 // V compressExpandOp(int opr,
2824 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2825 // int length, V v, M m,
2826 // CompressExpandOperation<V, M> defaultImpl)
2827 bool LibraryCallKit::inline_vector_compress_expand() {
2828 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2829 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2830 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2831 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2832 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2833
2834 if (opr == nullptr || !opr->is_con() ||
2835 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2836 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
2837 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2838 vlen == nullptr || !vlen->is_con()) {
2839 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2840 NodeClassNames[argument(0)->Opcode()],
2841 NodeClassNames[argument(1)->Opcode()],
2842 NodeClassNames[argument(2)->Opcode()],
2843 NodeClassNames[argument(3)->Opcode()],
2844 NodeClassNames[argument(4)->Opcode()]);
2845 return false; // not enough info for intrinsification
2846 }
2847
2848 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2849 log_if_needed(" ** klass argument not initialized");
2850 return false;
2851 }
2852
2853 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2854 if (!elem_type->is_primitive_type()) {
2855 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2856 return false; // should be primitive type
2857 }
2858
2897 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2898 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2899 set_result(vbox);
2900 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2901 return true;
2902 }
2903
2904 // public static
2905 // <V extends Vector<E>,
2906 // E,
2907 // S extends VectorSpecies<E>>
2908 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2909 // int length,
2910 // V v, int step, S s,
2911 // IndexOperation<V, S> defaultImpl)
2912 bool LibraryCallKit::inline_index_vector() {
2913 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2914 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2915 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2916
2917 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2918 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2919 vlen == nullptr || !vlen->is_con() ) {
2920 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2921 NodeClassNames[argument(0)->Opcode()],
2922 NodeClassNames[argument(1)->Opcode()],
2923 NodeClassNames[argument(2)->Opcode()]);
2924 return false; // not enough info for intrinsification
2925 }
2926
2927 if (!is_klass_initialized(vector_klass)) {
2928 log_if_needed(" ** klass argument not initialized");
2929 return false;
2930 }
2931
2932 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2933 if (!elem_type->is_primitive_type()) {
2934 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2935 return false; // should be primitive type
2936 }
2937
2938 int num_elem = vlen->get_con();
2939 BasicType elem_bt = elem_type->basic_type();
3031 if (needs_add) {
3032 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3033 }
3034 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3035 set_result(vbox);
3036 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3037 return true;
3038 }
3039
3040 // public static
3041 // <E,
3042 // M extends VectorMask<E>>
3043 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3044 // long offset, long limit,
3045 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3046 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3047 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
3048 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3049 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3050
3051 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
3052 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
3053 vlen == nullptr || !vlen->is_con()) {
3054 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
3055 NodeClassNames[argument(0)->Opcode()],
3056 NodeClassNames[argument(1)->Opcode()],
3057 NodeClassNames[argument(2)->Opcode()]);
3058 return false; // not enough info for intrinsification
3059 }
3060
3061 if (!is_klass_initialized(mask_klass)) {
3062 log_if_needed(" ** klass argument not initialized");
3063 return false;
3064 }
3065
3066 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3067 if (!elem_type->is_primitive_type()) {
3068 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3069 return false; // should be primitive type
3070 }
3071
3072 int num_elem = vlen->get_con();
3073 BasicType elem_bt = elem_type->basic_type();
|