306 // E>
307 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
308 // int length, V v1, V v2, M m,
309 // BinaryOperation<V, M> defaultImpl)
310 //
311 // public static
312 // <V extends Vector<E>,
313 // M extends VectorMask<E>,
314 // E>
315 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
316 // int length, V v1, V v2, V v3, M m,
317 // TernaryOperation<V, M> defaultImpl)
318 //
319 bool LibraryCallKit::inline_vector_nary_operation(int n) {
320 const TypeInt* opr = gvn().type(argument(0))->isa_int();
321 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
322 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
323 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
324 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
325
326 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
327 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
328 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
329 NodeClassNames[argument(0)->Opcode()],
330 NodeClassNames[argument(1)->Opcode()],
331 NodeClassNames[argument(3)->Opcode()],
332 NodeClassNames[argument(4)->Opcode()]);
333 return false; // not enough info for intrinsification
334 }
335
336 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
337 if (!elem_type->is_primitive_type()) {
338 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
339 return false; // should be primitive type
340 }
341 if (!is_klass_initialized(vector_klass)) {
342 log_if_needed(" ** klass argument not initialized");
343 return false;
344 }
345
346 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
347 // is used to control this operation.
505 operation = gvn().transform(operation);
506
507 // Wrap it up in VectorBox to keep object type information.
508 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
509 set_result(vbox);
510 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
511 return true;
512 }
513
514 // <Sh extends VectorShuffle<E>, E>
515 // Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
516 // int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
517 bool LibraryCallKit::inline_vector_shuffle_iota() {
518 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
519 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
520 const TypeInt* start_val = gvn().type(argument(4))->isa_int();
521 const TypeInt* step_val = gvn().type(argument(5))->isa_int();
522 const TypeInt* wrap = gvn().type(argument(6))->isa_int();
523
524 if (shuffle_klass == nullptr || shuffle_klass->const_oop() == nullptr ||
525 vlen == nullptr || !vlen->is_con() || start_val == nullptr || step_val == nullptr ||
526 wrap == nullptr || !wrap->is_con()) {
527 return false; // not enough info for intrinsification
528 }
529
530 if (!is_klass_initialized(shuffle_klass)) {
531 log_if_needed(" ** klass argument not initialized");
532 return false;
533 }
534
535 int do_wrap = wrap->get_con();
536 int num_elem = vlen->get_con();
537 BasicType elem_bt = T_BYTE;
538
539 bool effective_indices_in_range = false;
540 if (start_val->is_con() && step_val->is_con()) {
541 int effective_min_index = start_val->get_con();
542 int effective_max_index = start_val->get_con() + step_val->get_con() * (num_elem - 1);
543 effective_indices_in_range = effective_max_index >= effective_min_index && effective_min_index >= -128 && effective_max_index <= 127;
544 }
545
546 if (!do_wrap && !effective_indices_in_range) {
613 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
614 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
615
616 // Wrap it up in VectorBox to keep object type information.
617 res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
618 set_result(res);
619 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
620 return true;
621 }
622
623 // <E, M>
624 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
625 // int length, M m, VectorMaskOp<M> defaultImpl)
626 bool LibraryCallKit::inline_vector_mask_operation() {
627 const TypeInt* oper = gvn().type(argument(0))->isa_int();
628 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
629 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
630 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
631 Node* mask = argument(4);
632
633 if (mask_klass == nullptr || elem_klass == nullptr || mask->is_top() || vlen == nullptr) {
634 return false; // dead code
635 }
636
637 if (!is_klass_initialized(mask_klass)) {
638 log_if_needed(" ** klass argument not initialized");
639 return false;
640 }
641
642 int num_elem = vlen->get_con();
643 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
644 BasicType elem_bt = elem_type->basic_type();
645
646 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
647 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
648 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
649 mopc, num_elem, type2name(elem_bt));
650 return false; // not supported
651 }
652
653 const Type* elem_ty = Type::get_const_basic_type(elem_bt);
674 return true;
675 }
676
677 // public static
678 // <V,
679 // Sh extends VectorShuffle<E>,
680 // E>
681 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
682 // Class<? extends Sh> shuffleClass, Sh s, int length,
683 // ShuffleToVectorOperation<V, Sh, E> defaultImpl)
684 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
685 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
686 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
687 const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
688 Node* shuffle = argument(3);
689 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
690
691 if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) {
692 return false; // dead code
693 }
694 if (!vlen->is_con() || vector_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr) {
695 return false; // not enough info for intrinsification
696 }
697 if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
698 log_if_needed(" ** klass argument not initialized");
699 return false;
700 }
701
702 int num_elem = vlen->get_con();
703 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
704 BasicType elem_bt = elem_type->basic_type();
705
706 if (num_elem < 4) {
707 return false;
708 }
709
710 int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE
711 // Make sure that cast is implemented to particular type/size combination.
712 if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
713 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
714 cast_vopc, num_elem, type2name(elem_bt));
735 return true;
736 }
737
738 // public static
739 // <M,
740 // S extends VectorSpecies<E>,
741 // E>
742 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
743 // long bits, int mode, S s,
744 // BroadcastOperation<M, E, S> defaultImpl)
745 bool LibraryCallKit::inline_vector_frombits_coerced() {
746 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
747 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
748 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
749 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
750 // Mode argument determines the mode of operation it can take following values:-
751 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
752 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
753 const TypeInt* mode = gvn().type(argument(5))->isa_int();
754
755 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || mode == nullptr ||
756 bits_type == nullptr || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr ||
757 !vlen->is_con() || !mode->is_con()) {
758 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
759 NodeClassNames[argument(0)->Opcode()],
760 NodeClassNames[argument(1)->Opcode()],
761 NodeClassNames[argument(2)->Opcode()],
762 NodeClassNames[argument(5)->Opcode()]);
763 return false; // not enough info for intrinsification
764 }
765
766 if (!is_klass_initialized(vector_klass)) {
767 log_if_needed(" ** klass argument not initialized");
768 return false;
769 }
770 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
771 if (!elem_type->is_primitive_type()) {
772 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
773 return false; // should be primitive type
774 }
775 BasicType elem_bt = elem_type->basic_type();
776 int num_elem = vlen->get_con();
777 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
863 // int length,
864 // Object base, long offset, // Unsafe addressing
865 // boolean fromSegment,
866 // C container, long index, S s, // Arguments for default implementation
867 // LoadOperation<C, VM, S> defaultImpl) {
868 // public static
869 // <C,
870 // V extends VectorPayload>
871 // void store(Class<?> vClass, Class<?> eClass,
872 // int length,
873 // Object base, long offset, // Unsafe addressing
874 // boolean fromSegment,
875 // V v, C container, long index, // Arguments for default implementation
876 // StoreVectorOperation<C, V> defaultImpl) {
877 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
878 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
879 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
880 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
881 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
882
883 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || !from_ms->is_con() ||
884 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
885 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
886 NodeClassNames[argument(0)->Opcode()],
887 NodeClassNames[argument(1)->Opcode()],
888 NodeClassNames[argument(2)->Opcode()],
889 NodeClassNames[argument(6)->Opcode()]);
890 return false; // not enough info for intrinsification
891 }
892 if (!is_klass_initialized(vector_klass)) {
893 log_if_needed(" ** klass argument not initialized");
894 return false;
895 }
896
897 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
898 if (!elem_type->is_primitive_type()) {
899 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
900 return false; // should be primitive type
901 }
902 BasicType elem_bt = elem_type->basic_type();
903 int num_elem = vlen->get_con();
904
1071 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1072 // public static
1073 // <C,
1074 // V extends Vector<E>,
1075 // M extends VectorMask<E>,
1076 // E>
1077 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1078 // int length,
1079 // Object base, long offset, // Unsafe addressing
1080 // boolean fromSegment,
1081 // V v, M m, C container, long index, // Arguments for default implementation
1082 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
1083
1084 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1085 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1086 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1087 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1088 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1089 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
1090
1091 if (vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1092 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr || from_ms == nullptr ||
1093 elem_klass->const_oop() == nullptr || !vlen->is_con() || !from_ms->is_con()) {
1094 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1095 NodeClassNames[argument(0)->Opcode()],
1096 NodeClassNames[argument(1)->Opcode()],
1097 NodeClassNames[argument(2)->Opcode()],
1098 NodeClassNames[argument(3)->Opcode()],
1099 NodeClassNames[argument(7)->Opcode()]);
1100 return false; // not enough info for intrinsification
1101 }
1102 if (!is_klass_initialized(vector_klass)) {
1103 log_if_needed(" ** klass argument not initialized");
1104 return false;
1105 }
1106
1107 if (!is_klass_initialized(mask_klass)) {
1108 log_if_needed(" ** mask klass argument not initialized");
1109 return false;
1110 }
1111
1112 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1113 if (!elem_type->is_primitive_type()) {
1310 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1311 //
1312 // <C,
1313 // V extends Vector<E>,
1314 // W extends Vector<Integer>,
1315 // M extends VectorMask<E>,
1316 // E>
1317 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1318 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1319 // W index_vector, V v, M m,
1320 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1321 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1322 //
1323 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1324 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1325 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1326 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1327 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1328 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1329
1330 if (vector_klass == nullptr || elem_klass == nullptr || vector_idx_klass == nullptr || vlen == nullptr ||
1331 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || vector_idx_klass->const_oop() == nullptr || !vlen->is_con()) {
1332 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1333 NodeClassNames[argument(0)->Opcode()],
1334 NodeClassNames[argument(2)->Opcode()],
1335 NodeClassNames[argument(3)->Opcode()],
1336 NodeClassNames[argument(4)->Opcode()]);
1337 return false; // not enough info for intrinsification
1338 }
1339
1340 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1341 log_if_needed(" ** klass argument not initialized");
1342 return false;
1343 }
1344
1345 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1346 if (!elem_type->is_primitive_type()) {
1347 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1348 return false; // should be primitive type
1349 }
1350
1351 BasicType elem_bt = elem_type->basic_type();
1497 destruct_map_clone(old_map);
1498
1499 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1500 return true;
1501 }
1502
1503 // public static
1504 // <V extends Vector<E>,
1505 // M extends VectorMask<E>,
1506 // E>
1507 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1508 // Class<E> elementType, int length, V v, M m,
1509 // ReductionOperation<V, M> defaultImpl)
1510 bool LibraryCallKit::inline_vector_reduction() {
1511 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1512 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1513 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1514 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1515 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1516
1517 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1518 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1519 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1520 NodeClassNames[argument(0)->Opcode()],
1521 NodeClassNames[argument(1)->Opcode()],
1522 NodeClassNames[argument(3)->Opcode()],
1523 NodeClassNames[argument(4)->Opcode()]);
1524 return false; // not enough info for intrinsification
1525 }
1526 if (!is_klass_initialized(vector_klass)) {
1527 log_if_needed(" ** klass argument not initialized");
1528 return false;
1529 }
1530 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1531 if (!elem_type->is_primitive_type()) {
1532 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1533 return false; // should be primitive type
1534 }
1535
1536 const Type* vmask_type = gvn().type(argument(6));
1537 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1538 if (is_masked_op) {
1632 bits = value; // no conversion needed
1633 break;
1634 }
1635 default: fatal("%s", type2name(elem_bt));
1636 }
1637 set_result(bits);
1638 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1639 return true;
1640 }
1641
1642 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1643 // V v1, V v2,
1644 // BiFunction<V, V, Boolean> defaultImpl)
1645 //
1646 bool LibraryCallKit::inline_vector_test() {
1647 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1648 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1649 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1650 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1651
1652 if (cond == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1653 !cond->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1654 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1655 NodeClassNames[argument(0)->Opcode()],
1656 NodeClassNames[argument(1)->Opcode()],
1657 NodeClassNames[argument(2)->Opcode()],
1658 NodeClassNames[argument(3)->Opcode()]);
1659 return false; // not enough info for intrinsification
1660 }
1661 if (!is_klass_initialized(vector_klass)) {
1662 log_if_needed(" ** klass argument not initialized");
1663 return false;
1664 }
1665 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1666 if (!elem_type->is_primitive_type()) {
1667 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1668 return false; // should be primitive type
1669 }
1670 BasicType elem_bt = elem_type->basic_type();
1671 int num_elem = vlen->get_con();
1672 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1673 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2474 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2475 return true;
2476 }
2477
2478 // public static
2479 // <VM extends VectorPayload,
2480 // E>
2481 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2482 // int length,
2483 // VM vm, int i,
2484 // VecExtractOp<VM> defaultImpl)
2485 bool LibraryCallKit::inline_vector_extract() {
2486 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2487 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2488 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2489 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2490
2491 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2492 return false; // dead code
2493 }
2494 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2495 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2496 NodeClassNames[argument(0)->Opcode()],
2497 NodeClassNames[argument(1)->Opcode()],
2498 NodeClassNames[argument(2)->Opcode()]);
2499 return false; // not enough info for intrinsification
2500 }
2501 if (!is_klass_initialized(vector_klass)) {
2502 log_if_needed(" ** klass argument not initialized");
2503 return false;
2504 }
2505 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2506 if (!elem_type->is_primitive_type()) {
2507 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2508 return false; // should be primitive type
2509 }
2510 BasicType elem_bt = elem_type->basic_type();
2511 int num_elem = vlen->get_con();
2512
2513 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2514 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2591 }
2592 set_result(opd);
2593 return true;
2594 }
2595
2596 // public static
2597 // <V extends Vector<E>,
2598 // M extends VectorMask<E>,
2599 // E>
2600 // V compressExpandOp(int opr,
2601 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2602 // int length, V v, M m,
2603 // CompressExpandOperation<V, M> defaultImpl)
2604 bool LibraryCallKit::inline_vector_compress_expand() {
2605 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2606 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2607 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2608 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2609 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2610
2611 if (vector_klass == nullptr || elem_klass == nullptr || mask_klass == nullptr || vlen == nullptr ||
2612 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
2613 elem_klass->const_oop() == nullptr || !vlen->is_con() || !opr->is_con()) {
2614 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2615 NodeClassNames[argument(0)->Opcode()],
2616 NodeClassNames[argument(1)->Opcode()],
2617 NodeClassNames[argument(2)->Opcode()],
2618 NodeClassNames[argument(3)->Opcode()],
2619 NodeClassNames[argument(4)->Opcode()]);
2620 return false; // not enough info for intrinsification
2621 }
2622
2623 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2624 log_if_needed(" ** klass argument not initialized");
2625 return false;
2626 }
2627
2628 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2629 if (!elem_type->is_primitive_type()) {
2630 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2631 return false; // should be primitive type
2632 }
2633
2672 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2673 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2674 set_result(vbox);
2675 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2676 return true;
2677 }
2678
2679 // public static
2680 // <V extends Vector<E>,
2681 // E,
2682 // S extends VectorSpecies<E>>
2683 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2684 // int length,
2685 // V v, int step, S s,
2686 // IndexOperation<V, S> defaultImpl)
2687 bool LibraryCallKit::inline_index_vector() {
2688 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2689 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2690 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2691
2692 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2693 vector_klass->const_oop() == nullptr || !vlen->is_con() ||
2694 elem_klass->const_oop() == nullptr) {
2695 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2696 NodeClassNames[argument(0)->Opcode()],
2697 NodeClassNames[argument(1)->Opcode()],
2698 NodeClassNames[argument(2)->Opcode()]);
2699 return false; // not enough info for intrinsification
2700 }
2701
2702 if (!is_klass_initialized(vector_klass)) {
2703 log_if_needed(" ** klass argument not initialized");
2704 return false;
2705 }
2706
2707 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2708 if (!elem_type->is_primitive_type()) {
2709 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2710 return false; // should be primitive type
2711 }
2712
2713 int num_elem = vlen->get_con();
2714 BasicType elem_bt = elem_type->basic_type();
2806 if (needs_add) {
2807 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
2808 }
2809 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
2810 set_result(vbox);
2811 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2812 return true;
2813 }
2814
2815 // public static
2816 // <E,
2817 // M extends VectorMask<E>>
2818 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
2819 // long offset, long limit,
2820 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
2821 bool LibraryCallKit::inline_index_partially_in_upper_range() {
2822 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
2823 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2824 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2825
2826 if (mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2827 mask_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2828 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
2829 NodeClassNames[argument(0)->Opcode()],
2830 NodeClassNames[argument(1)->Opcode()],
2831 NodeClassNames[argument(2)->Opcode()]);
2832 return false; // not enough info for intrinsification
2833 }
2834
2835 if (!is_klass_initialized(mask_klass)) {
2836 log_if_needed(" ** klass argument not initialized");
2837 return false;
2838 }
2839
2840 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2841 if (!elem_type->is_primitive_type()) {
2842 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2843 return false; // should be primitive type
2844 }
2845
2846 int num_elem = vlen->get_con();
2847 BasicType elem_bt = elem_type->basic_type();
|
306 // E>
307 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
308 // int length, V v1, V v2, M m,
309 // BinaryOperation<V, M> defaultImpl)
310 //
311 // public static
312 // <V extends Vector<E>,
313 // M extends VectorMask<E>,
314 // E>
315 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
316 // int length, V v1, V v2, V v3, M m,
317 // TernaryOperation<V, M> defaultImpl)
318 //
319 bool LibraryCallKit::inline_vector_nary_operation(int n) {
320 const TypeInt* opr = gvn().type(argument(0))->isa_int();
321 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
322 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
323 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
324 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
325
326 if (opr == nullptr || !opr->is_con() ||
327 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
328 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
329 vlen == nullptr || !vlen->is_con()) {
330 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
331 NodeClassNames[argument(0)->Opcode()],
332 NodeClassNames[argument(1)->Opcode()],
333 NodeClassNames[argument(3)->Opcode()],
334 NodeClassNames[argument(4)->Opcode()]);
335 return false; // not enough info for intrinsification
336 }
337
338 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
339 if (!elem_type->is_primitive_type()) {
340 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
341 return false; // should be primitive type
342 }
343 if (!is_klass_initialized(vector_klass)) {
344 log_if_needed(" ** klass argument not initialized");
345 return false;
346 }
347
348 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
349 // is used to control this operation.
507 operation = gvn().transform(operation);
508
509 // Wrap it up in VectorBox to keep object type information.
510 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
511 set_result(vbox);
512 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
513 return true;
514 }
515
516 // <Sh extends VectorShuffle<E>, E>
517 // Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
518 // int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
519 bool LibraryCallKit::inline_vector_shuffle_iota() {
520 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
521 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
522 const TypeInt* start_val = gvn().type(argument(4))->isa_int();
523 const TypeInt* step_val = gvn().type(argument(5))->isa_int();
524 const TypeInt* wrap = gvn().type(argument(6))->isa_int();
525
526 if (shuffle_klass == nullptr || shuffle_klass->const_oop() == nullptr ||
527 vlen == nullptr || !vlen->is_con() ||
528 start_val == nullptr ||
529 step_val == nullptr ||
530 wrap == nullptr || !wrap->is_con()) {
531 return false; // not enough info for intrinsification
532 }
533
534 if (!is_klass_initialized(shuffle_klass)) {
535 log_if_needed(" ** klass argument not initialized");
536 return false;
537 }
538
539 int do_wrap = wrap->get_con();
540 int num_elem = vlen->get_con();
541 BasicType elem_bt = T_BYTE;
542
543 bool effective_indices_in_range = false;
544 if (start_val->is_con() && step_val->is_con()) {
545 int effective_min_index = start_val->get_con();
546 int effective_max_index = start_val->get_con() + step_val->get_con() * (num_elem - 1);
547 effective_indices_in_range = effective_max_index >= effective_min_index && effective_min_index >= -128 && effective_max_index <= 127;
548 }
549
550 if (!do_wrap && !effective_indices_in_range) {
617 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
618 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
619
620 // Wrap it up in VectorBox to keep object type information.
621 res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
622 set_result(res);
623 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
624 return true;
625 }
626
627 // <E, M>
628 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
629 // int length, M m, VectorMaskOp<M> defaultImpl)
630 bool LibraryCallKit::inline_vector_mask_operation() {
631 const TypeInt* oper = gvn().type(argument(0))->isa_int();
632 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
633 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
634 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
635 Node* mask = argument(4);
636
637 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
638 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
639 vlen == nullptr || !vlen->is_con() ||
640 oper == nullptr || !oper->is_con() ||
641 mask->is_top()) {
642 return false; // dead code
643 }
644
645 if (!is_klass_initialized(mask_klass)) {
646 log_if_needed(" ** klass argument not initialized");
647 return false;
648 }
649
650 int num_elem = vlen->get_con();
651 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
652 BasicType elem_bt = elem_type->basic_type();
653
654 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
655 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
656 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
657 mopc, num_elem, type2name(elem_bt));
658 return false; // not supported
659 }
660
661 const Type* elem_ty = Type::get_const_basic_type(elem_bt);
682 return true;
683 }
684
685 // public static
686 // <V,
687 // Sh extends VectorShuffle<E>,
688 // E>
689 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
690 // Class<? extends Sh> shuffleClass, Sh s, int length,
691 // ShuffleToVectorOperation<V, Sh, E> defaultImpl)
692 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
693 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
694 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
695 const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
696 Node* shuffle = argument(3);
697 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
698
699 if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) {
700 return false; // dead code
701 }
702 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr || !vlen->is_con()) {
703 return false; // not enough info for intrinsification
704 }
705 if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
706 log_if_needed(" ** klass argument not initialized");
707 return false;
708 }
709
710 int num_elem = vlen->get_con();
711 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
712 BasicType elem_bt = elem_type->basic_type();
713
714 if (num_elem < 4) {
715 return false;
716 }
717
718 int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE
719 // Make sure that cast is implemented to particular type/size combination.
720 if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
721 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
722 cast_vopc, num_elem, type2name(elem_bt));
743 return true;
744 }
745
746 // public static
747 // <M,
748 // S extends VectorSpecies<E>,
749 // E>
750 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
751 // long bits, int mode, S s,
752 // BroadcastOperation<M, E, S> defaultImpl)
753 bool LibraryCallKit::inline_vector_frombits_coerced() {
754 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
755 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
756 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
757 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
758 // Mode argument determines the mode of operation it can take following values:-
759 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
760 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
761 const TypeInt* mode = gvn().type(argument(5))->isa_int();
762
763 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
764 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
765 vlen == nullptr || !vlen->is_con() ||
766 bits_type == nullptr ||
767 mode == nullptr || !mode->is_con()) {
768 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
769 NodeClassNames[argument(0)->Opcode()],
770 NodeClassNames[argument(1)->Opcode()],
771 NodeClassNames[argument(2)->Opcode()],
772 NodeClassNames[argument(5)->Opcode()]);
773 return false; // not enough info for intrinsification
774 }
775
776 if (!is_klass_initialized(vector_klass)) {
777 log_if_needed(" ** klass argument not initialized");
778 return false;
779 }
780 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
781 if (!elem_type->is_primitive_type()) {
782 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
783 return false; // should be primitive type
784 }
785 BasicType elem_bt = elem_type->basic_type();
786 int num_elem = vlen->get_con();
787 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
873 // int length,
874 // Object base, long offset, // Unsafe addressing
875 // boolean fromSegment,
876 // C container, long index, S s, // Arguments for default implementation
877 // LoadOperation<C, VM, S> defaultImpl) {
878 // public static
879 // <C,
880 // V extends VectorPayload>
881 // void store(Class<?> vClass, Class<?> eClass,
882 // int length,
883 // Object base, long offset, // Unsafe addressing
884 // boolean fromSegment,
885 // V v, C container, long index, // Arguments for default implementation
886 // StoreVectorOperation<C, V> defaultImpl) {
887 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
888 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
889 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
890 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
891 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
892
893 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
894 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
895 vlen == nullptr || !vlen->is_con() ||
896 from_ms == nullptr || !from_ms->is_con()) {
897 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
898 NodeClassNames[argument(0)->Opcode()],
899 NodeClassNames[argument(1)->Opcode()],
900 NodeClassNames[argument(2)->Opcode()],
901 NodeClassNames[argument(6)->Opcode()]);
902 return false; // not enough info for intrinsification
903 }
904 if (!is_klass_initialized(vector_klass)) {
905 log_if_needed(" ** klass argument not initialized");
906 return false;
907 }
908
909 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
910 if (!elem_type->is_primitive_type()) {
911 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
912 return false; // should be primitive type
913 }
914 BasicType elem_bt = elem_type->basic_type();
915 int num_elem = vlen->get_con();
916
1083 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1084 // public static
1085 // <C,
1086 // V extends Vector<E>,
1087 // M extends VectorMask<E>,
1088 // E>
1089 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1090 // int length,
1091 // Object base, long offset, // Unsafe addressing
1092 // boolean fromSegment,
1093 // V v, M m, C container, long index, // Arguments for default implementation
1094 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
1095
1096 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1097 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1098 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1099 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1100 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1101 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
1102
1103 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1104 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1105 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1106 vlen == nullptr || !vlen->is_con() ||
1107 from_ms == nullptr || !from_ms->is_con()) {
1108 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1109 NodeClassNames[argument(0)->Opcode()],
1110 NodeClassNames[argument(1)->Opcode()],
1111 NodeClassNames[argument(2)->Opcode()],
1112 NodeClassNames[argument(3)->Opcode()],
1113 NodeClassNames[argument(7)->Opcode()]);
1114 return false; // not enough info for intrinsification
1115 }
1116 if (!is_klass_initialized(vector_klass)) {
1117 log_if_needed(" ** klass argument not initialized");
1118 return false;
1119 }
1120
1121 if (!is_klass_initialized(mask_klass)) {
1122 log_if_needed(" ** mask klass argument not initialized");
1123 return false;
1124 }
1125
1126 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1127 if (!elem_type->is_primitive_type()) {
1324 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1325 //
1326 // <C,
1327 // V extends Vector<E>,
1328 // W extends Vector<Integer>,
1329 // M extends VectorMask<E>,
1330 // E>
1331 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1332 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1333 // W index_vector, V v, M m,
1334 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1335 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1336 //
1337 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1338 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1339 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1340 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1341 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1342 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1343
1344 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1345 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1346 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1347 vlen == nullptr || !vlen->is_con() ||
1348 vector_idx_klass == nullptr || vector_idx_klass->const_oop() == nullptr) {
1349 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1350 NodeClassNames[argument(0)->Opcode()],
1351 NodeClassNames[argument(2)->Opcode()],
1352 NodeClassNames[argument(3)->Opcode()],
1353 NodeClassNames[argument(4)->Opcode()]);
1354 return false; // not enough info for intrinsification
1355 }
1356
1357 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1358 log_if_needed(" ** klass argument not initialized");
1359 return false;
1360 }
1361
1362 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1363 if (!elem_type->is_primitive_type()) {
1364 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1365 return false; // should be primitive type
1366 }
1367
1368 BasicType elem_bt = elem_type->basic_type();
1514 destruct_map_clone(old_map);
1515
1516 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1517 return true;
1518 }
1519
1520 // public static
1521 // <V extends Vector<E>,
1522 // M extends VectorMask<E>,
1523 // E>
1524 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1525 // Class<E> elementType, int length, V v, M m,
1526 // ReductionOperation<V, M> defaultImpl)
1527 bool LibraryCallKit::inline_vector_reduction() {
1528 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1529 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1530 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1531 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1532 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1533
1534 if (opr == nullptr || !opr->is_con() ||
1535 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1536 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1537 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1538 vlen == nullptr || !vlen->is_con()) {
1539 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1540 NodeClassNames[argument(0)->Opcode()],
1541 NodeClassNames[argument(1)->Opcode()],
1542 NodeClassNames[argument(3)->Opcode()],
1543 NodeClassNames[argument(4)->Opcode()]);
1544 return false; // not enough info for intrinsification
1545 }
1546 if (!is_klass_initialized(vector_klass)) {
1547 log_if_needed(" ** klass argument not initialized");
1548 return false;
1549 }
1550 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1551 if (!elem_type->is_primitive_type()) {
1552 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1553 return false; // should be primitive type
1554 }
1555
1556 const Type* vmask_type = gvn().type(argument(6));
1557 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1558 if (is_masked_op) {
1652 bits = value; // no conversion needed
1653 break;
1654 }
1655 default: fatal("%s", type2name(elem_bt));
1656 }
1657 set_result(bits);
1658 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1659 return true;
1660 }
1661
1662 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1663 // V v1, V v2,
1664 // BiFunction<V, V, Boolean> defaultImpl)
1665 //
1666 bool LibraryCallKit::inline_vector_test() {
1667 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1668 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1669 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1670 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1671
1672 if (cond == nullptr || !cond->is_con() ||
1673 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1674 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1675 vlen == nullptr || !vlen->is_con()) {
1676 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1677 NodeClassNames[argument(0)->Opcode()],
1678 NodeClassNames[argument(1)->Opcode()],
1679 NodeClassNames[argument(2)->Opcode()],
1680 NodeClassNames[argument(3)->Opcode()]);
1681 return false; // not enough info for intrinsification
1682 }
1683 if (!is_klass_initialized(vector_klass)) {
1684 log_if_needed(" ** klass argument not initialized");
1685 return false;
1686 }
1687 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1688 if (!elem_type->is_primitive_type()) {
1689 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1690 return false; // should be primitive type
1691 }
1692 BasicType elem_bt = elem_type->basic_type();
1693 int num_elem = vlen->get_con();
1694 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1695 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2496 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2497 return true;
2498 }
2499
2500 // public static
2501 // <VM extends VectorPayload,
2502 // E>
2503 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2504 // int length,
2505 // VM vm, int i,
2506 // VecExtractOp<VM> defaultImpl)
2507 bool LibraryCallKit::inline_vector_extract() {
2508 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2509 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2510 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2511 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2512
2513 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2514 return false; // dead code
2515 }
2516 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2517 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2518 NodeClassNames[argument(0)->Opcode()],
2519 NodeClassNames[argument(1)->Opcode()],
2520 NodeClassNames[argument(2)->Opcode()]);
2521 return false; // not enough info for intrinsification
2522 }
2523 if (!is_klass_initialized(vector_klass)) {
2524 log_if_needed(" ** klass argument not initialized");
2525 return false;
2526 }
2527 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2528 if (!elem_type->is_primitive_type()) {
2529 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2530 return false; // should be primitive type
2531 }
2532 BasicType elem_bt = elem_type->basic_type();
2533 int num_elem = vlen->get_con();
2534
2535 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2536 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2613 }
2614 set_result(opd);
2615 return true;
2616 }
2617
2618 // public static
2619 // <V extends Vector<E>,
2620 // M extends VectorMask<E>,
2621 // E>
2622 // V compressExpandOp(int opr,
2623 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2624 // int length, V v, M m,
2625 // CompressExpandOperation<V, M> defaultImpl)
2626 bool LibraryCallKit::inline_vector_compress_expand() {
2627 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2628 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2629 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2630 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2631 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2632
2633 if (opr == nullptr || !opr->is_con() ||
2634 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2635 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
2636 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2637 vlen == nullptr || !vlen->is_con()) {
2638 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2639 NodeClassNames[argument(0)->Opcode()],
2640 NodeClassNames[argument(1)->Opcode()],
2641 NodeClassNames[argument(2)->Opcode()],
2642 NodeClassNames[argument(3)->Opcode()],
2643 NodeClassNames[argument(4)->Opcode()]);
2644 return false; // not enough info for intrinsification
2645 }
2646
2647 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2648 log_if_needed(" ** klass argument not initialized");
2649 return false;
2650 }
2651
2652 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2653 if (!elem_type->is_primitive_type()) {
2654 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2655 return false; // should be primitive type
2656 }
2657
2696 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2697 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2698 set_result(vbox);
2699 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2700 return true;
2701 }
2702
2703 // public static
2704 // <V extends Vector<E>,
2705 // E,
2706 // S extends VectorSpecies<E>>
2707 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2708 // int length,
2709 // V v, int step, S s,
2710 // IndexOperation<V, S> defaultImpl)
2711 bool LibraryCallKit::inline_index_vector() {
2712 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2713 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2714 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2715
2716 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2717 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2718 vlen == nullptr || !vlen->is_con() ) {
2719 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2720 NodeClassNames[argument(0)->Opcode()],
2721 NodeClassNames[argument(1)->Opcode()],
2722 NodeClassNames[argument(2)->Opcode()]);
2723 return false; // not enough info for intrinsification
2724 }
2725
2726 if (!is_klass_initialized(vector_klass)) {
2727 log_if_needed(" ** klass argument not initialized");
2728 return false;
2729 }
2730
2731 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2732 if (!elem_type->is_primitive_type()) {
2733 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2734 return false; // should be primitive type
2735 }
2736
2737 int num_elem = vlen->get_con();
2738 BasicType elem_bt = elem_type->basic_type();
2830 if (needs_add) {
2831 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
2832 }
2833 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
2834 set_result(vbox);
2835 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2836 return true;
2837 }
2838
2839 // public static
2840 // <E,
2841 // M extends VectorMask<E>>
2842 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
2843 // long offset, long limit,
2844 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
2845 bool LibraryCallKit::inline_index_partially_in_upper_range() {
2846 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
2847 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2848 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2849
2850 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
2851 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2852 vlen == nullptr || !vlen->is_con()) {
2853 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
2854 NodeClassNames[argument(0)->Opcode()],
2855 NodeClassNames[argument(1)->Opcode()],
2856 NodeClassNames[argument(2)->Opcode()]);
2857 return false; // not enough info for intrinsification
2858 }
2859
2860 if (!is_klass_initialized(mask_klass)) {
2861 log_if_needed(" ** klass argument not initialized");
2862 return false;
2863 }
2864
2865 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2866 if (!elem_type->is_primitive_type()) {
2867 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2868 return false; // should be primitive type
2869 }
2870
2871 int num_elem = vlen->get_con();
2872 BasicType elem_bt = elem_type->basic_type();
|