302 // E>
303 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
304 // int length, V v1, V v2, M m,
305 // BinaryOperation<V, M> defaultImpl)
306 //
307 // public static
308 // <V extends Vector<E>,
309 // M extends VectorMask<E>,
310 // E>
311 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
312 // int length, V v1, V v2, V v3, M m,
313 // TernaryOperation<V, M> defaultImpl)
314 //
315 bool LibraryCallKit::inline_vector_nary_operation(int n) {
316 const TypeInt* opr = gvn().type(argument(0))->isa_int();
317 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
318 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
319 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
320 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
321
322 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
323 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
324 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
325 NodeClassNames[argument(0)->Opcode()],
326 NodeClassNames[argument(1)->Opcode()],
327 NodeClassNames[argument(3)->Opcode()],
328 NodeClassNames[argument(4)->Opcode()]);
329 return false; // not enough info for intrinsification
330 }
331
332 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
333 if (!elem_type->is_primitive_type()) {
334 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
335 return false; // should be primitive type
336 }
337 if (!is_klass_initialized(vector_klass)) {
338 log_if_needed(" ** klass argument not initialized");
339 return false;
340 }
341
342 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
343 // is used to control this operation.
504 operation = gvn().transform(operation);
505
506 // Wrap it up in VectorBox to keep object type information.
507 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
508 set_result(vbox);
509 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
510 return true;
511 }
512
513 // public static
514 // <E, M>
515 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
516 // int length, M m, VectorMaskOp<M> defaultImpl)
517 bool LibraryCallKit::inline_vector_mask_operation() {
518 const TypeInt* oper = gvn().type(argument(0))->isa_int();
519 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
520 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
521 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
522 Node* mask = argument(4);
523
524 if (mask_klass == nullptr || elem_klass == nullptr || mask->is_top() || vlen == nullptr) {
525 return false; // dead code
526 }
527
528 if (!is_klass_initialized(mask_klass)) {
529 log_if_needed(" ** klass argument not initialized");
530 return false;
531 }
532
533 int num_elem = vlen->get_con();
534 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
535 BasicType elem_bt = elem_type->basic_type();
536
537 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
538 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
539 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
540 mopc, num_elem, type2name(elem_bt));
541 return false; // not supported
542 }
543
544 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
564 return true;
565 }
566
567 // public static
568 // <M,
569 // S extends VectorSpecies<E>,
570 // E>
571 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
572 // long bits, int mode, S s,
573 // BroadcastOperation<M, E, S> defaultImpl)
574 bool LibraryCallKit::inline_vector_frombits_coerced() {
575 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
576 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
577 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
578 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
579 // Mode argument determines the mode of operation it can take following values:-
580 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
581 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
582 const TypeInt* mode = gvn().type(argument(5))->isa_int();
583
584 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || mode == nullptr ||
585 bits_type == nullptr || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr ||
586 !vlen->is_con() || !mode->is_con()) {
587 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
588 NodeClassNames[argument(0)->Opcode()],
589 NodeClassNames[argument(1)->Opcode()],
590 NodeClassNames[argument(2)->Opcode()],
591 NodeClassNames[argument(5)->Opcode()]);
592 return false; // not enough info for intrinsification
593 }
594
595 if (!is_klass_initialized(vector_klass)) {
596 log_if_needed(" ** klass argument not initialized");
597 return false;
598 }
599 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
600 if (!elem_type->is_primitive_type()) {
601 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
602 return false; // should be primitive type
603 }
604 BasicType elem_bt = elem_type->basic_type();
605 int num_elem = vlen->get_con();
606 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
692 // int length,
693 // Object base, long offset, // Unsafe addressing
694 // boolean fromSegment,
695 // C container, long index, S s, // Arguments for default implementation
696 // LoadOperation<C, VM, S> defaultImpl) {
697 // public static
698 // <C,
699 // V extends VectorPayload>
700 // void store(Class<?> vClass, Class<?> eClass,
701 // int length,
702 // Object base, long offset, // Unsafe addressing
703 // boolean fromSegment,
704 // V v, C container, long index, // Arguments for default implementation
705 // StoreVectorOperation<C, V> defaultImpl) {
706 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
707 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
708 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
709 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
710 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
711
712 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || !from_ms->is_con() ||
713 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
714 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
715 NodeClassNames[argument(0)->Opcode()],
716 NodeClassNames[argument(1)->Opcode()],
717 NodeClassNames[argument(2)->Opcode()],
718 NodeClassNames[argument(6)->Opcode()]);
719 return false; // not enough info for intrinsification
720 }
721 if (!is_klass_initialized(vector_klass)) {
722 log_if_needed(" ** klass argument not initialized");
723 return false;
724 }
725
726 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
727 if (!elem_type->is_primitive_type()) {
728 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
729 return false; // should be primitive type
730 }
731 BasicType elem_bt = elem_type->basic_type();
732 int num_elem = vlen->get_con();
733
900 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
901 // public static
902 // <C,
903 // V extends Vector<E>,
904 // M extends VectorMask<E>,
905 // E>
906 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
907 // int length,
908 // Object base, long offset, // Unsafe addressing
909 // boolean fromSegment,
910 // V v, M m, C container, long index, // Arguments for default implementation
911 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
912
913 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
914 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
915 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
916 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
917 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
918 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
919
920 if (vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
921 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr || from_ms == nullptr ||
922 elem_klass->const_oop() == nullptr || !vlen->is_con() || !from_ms->is_con()) {
923 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
924 NodeClassNames[argument(0)->Opcode()],
925 NodeClassNames[argument(1)->Opcode()],
926 NodeClassNames[argument(2)->Opcode()],
927 NodeClassNames[argument(3)->Opcode()],
928 NodeClassNames[argument(7)->Opcode()]);
929 return false; // not enough info for intrinsification
930 }
931 if (!is_klass_initialized(vector_klass)) {
932 log_if_needed(" ** klass argument not initialized");
933 return false;
934 }
935
936 if (!is_klass_initialized(mask_klass)) {
937 log_if_needed(" ** mask klass argument not initialized");
938 return false;
939 }
940
941 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
942 if (!elem_type->is_primitive_type()) {
1139 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1140 //
1141 // <C,
1142 // V extends Vector<E>,
1143 // W extends Vector<Integer>,
1144 // M extends VectorMask<E>,
1145 // E>
1146 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1147 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1148 // W index_vector, V v, M m,
1149 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1150 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1151 //
1152 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1153 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1154 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1155 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1156 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1157 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1158
1159 if (vector_klass == nullptr || elem_klass == nullptr || vector_idx_klass == nullptr || vlen == nullptr ||
1160 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || vector_idx_klass->const_oop() == nullptr || !vlen->is_con()) {
1161 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1162 NodeClassNames[argument(0)->Opcode()],
1163 NodeClassNames[argument(2)->Opcode()],
1164 NodeClassNames[argument(3)->Opcode()],
1165 NodeClassNames[argument(4)->Opcode()]);
1166 return false; // not enough info for intrinsification
1167 }
1168
1169 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1170 log_if_needed(" ** klass argument not initialized");
1171 return false;
1172 }
1173
1174 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1175 if (!elem_type->is_primitive_type()) {
1176 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1177 return false; // should be primitive type
1178 }
1179
1180 BasicType elem_bt = elem_type->basic_type();
1326 destruct_map_clone(old_map);
1327
1328 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1329 return true;
1330 }
1331
1332 // public static
1333 // <V extends Vector<E>,
1334 // M extends VectorMask<E>,
1335 // E>
1336 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1337 // Class<E> elementType, int length, V v, M m,
1338 // ReductionOperation<V, M> defaultImpl)
1339 bool LibraryCallKit::inline_vector_reduction() {
1340 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1341 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1342 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1343 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1344 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1345
1346 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1347 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1348 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1349 NodeClassNames[argument(0)->Opcode()],
1350 NodeClassNames[argument(1)->Opcode()],
1351 NodeClassNames[argument(3)->Opcode()],
1352 NodeClassNames[argument(4)->Opcode()]);
1353 return false; // not enough info for intrinsification
1354 }
1355 if (!is_klass_initialized(vector_klass)) {
1356 log_if_needed(" ** klass argument not initialized");
1357 return false;
1358 }
1359 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1360 if (!elem_type->is_primitive_type()) {
1361 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1362 return false; // should be primitive type
1363 }
1364
1365 const Type* vmask_type = gvn().type(argument(6));
1366 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1367 if (is_masked_op) {
1464 bits = value; // no conversion needed
1465 break;
1466 }
1467 default: fatal("%s", type2name(elem_bt));
1468 }
1469 set_result(bits);
1470 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1471 return true;
1472 }
1473
1474 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1475 // V v1, V v2,
1476 // BiFunction<V, V, Boolean> defaultImpl)
1477 //
1478 bool LibraryCallKit::inline_vector_test() {
1479 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1480 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1481 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1482 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1483
1484 if (cond == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1485 !cond->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1486 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1487 NodeClassNames[argument(0)->Opcode()],
1488 NodeClassNames[argument(1)->Opcode()],
1489 NodeClassNames[argument(2)->Opcode()],
1490 NodeClassNames[argument(3)->Opcode()]);
1491 return false; // not enough info for intrinsification
1492 }
1493 if (!is_klass_initialized(vector_klass)) {
1494 log_if_needed(" ** klass argument not initialized");
1495 return false;
1496 }
1497 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1498 if (!elem_type->is_primitive_type()) {
1499 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1500 return false; // should be primitive type
1501 }
1502 BasicType elem_bt = elem_type->basic_type();
1503 int num_elem = vlen->get_con();
1504 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1505 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2495 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2496 return true;
2497 }
2498
2499 // public static
2500 // <VM extends VectorPayload,
2501 // E>
2502 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2503 // int length,
2504 // VM vm, int i,
2505 // VecExtractOp<VM> defaultImpl)
2506 bool LibraryCallKit::inline_vector_extract() {
2507 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2508 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2509 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2510 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2511
2512 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2513 return false; // dead code
2514 }
2515 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2516 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2517 NodeClassNames[argument(0)->Opcode()],
2518 NodeClassNames[argument(1)->Opcode()],
2519 NodeClassNames[argument(2)->Opcode()]);
2520 return false; // not enough info for intrinsification
2521 }
2522 if (!is_klass_initialized(vector_klass)) {
2523 log_if_needed(" ** klass argument not initialized");
2524 return false;
2525 }
2526 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2527 if (!elem_type->is_primitive_type()) {
2528 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2529 return false; // should be primitive type
2530 }
2531 BasicType elem_bt = elem_type->basic_type();
2532 int num_elem = vlen->get_con();
2533
2534 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2535 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2798 set_result(vbox);
2799 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2800 return true;
2801 }
2802
2803 // public static
2804 // <V extends Vector<E>,
2805 // M extends VectorMask<E>,
2806 // E>
2807 // V compressExpandOp(int opr,
2808 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2809 // int length, V v, M m,
2810 // CompressExpandOperation<V, M> defaultImpl)
2811 bool LibraryCallKit::inline_vector_compress_expand() {
2812 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2813 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2814 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2815 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2816 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2817
2818 if (vector_klass == nullptr || elem_klass == nullptr || mask_klass == nullptr || vlen == nullptr ||
2819 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
2820 elem_klass->const_oop() == nullptr || !vlen->is_con() || !opr->is_con()) {
2821 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2822 NodeClassNames[argument(0)->Opcode()],
2823 NodeClassNames[argument(1)->Opcode()],
2824 NodeClassNames[argument(2)->Opcode()],
2825 NodeClassNames[argument(3)->Opcode()],
2826 NodeClassNames[argument(4)->Opcode()]);
2827 return false; // not enough info for intrinsification
2828 }
2829
2830 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2831 log_if_needed(" ** klass argument not initialized");
2832 return false;
2833 }
2834
2835 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2836 if (!elem_type->is_primitive_type()) {
2837 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2838 return false; // should be primitive type
2839 }
2840
2879 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2880 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2881 set_result(vbox);
2882 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2883 return true;
2884 }
2885
2886 // public static
2887 // <V extends Vector<E>,
2888 // E,
2889 // S extends VectorSpecies<E>>
2890 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2891 // int length,
2892 // V v, int step, S s,
2893 // IndexOperation<V, S> defaultImpl)
2894 bool LibraryCallKit::inline_index_vector() {
2895 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2896 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2897 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2898
2899 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2900 vector_klass->const_oop() == nullptr || !vlen->is_con() ||
2901 elem_klass->const_oop() == nullptr) {
2902 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2903 NodeClassNames[argument(0)->Opcode()],
2904 NodeClassNames[argument(1)->Opcode()],
2905 NodeClassNames[argument(2)->Opcode()]);
2906 return false; // not enough info for intrinsification
2907 }
2908
2909 if (!is_klass_initialized(vector_klass)) {
2910 log_if_needed(" ** klass argument not initialized");
2911 return false;
2912 }
2913
2914 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2915 if (!elem_type->is_primitive_type()) {
2916 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2917 return false; // should be primitive type
2918 }
2919
2920 int num_elem = vlen->get_con();
2921 BasicType elem_bt = elem_type->basic_type();
3013 if (needs_add) {
3014 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3015 }
3016 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3017 set_result(vbox);
3018 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3019 return true;
3020 }
3021
3022 // public static
3023 // <E,
3024 // M extends VectorMask<E>>
3025 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3026 // long offset, long limit,
3027 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3028 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3029 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
3030 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3031 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3032
3033 if (mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
3034 mask_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
3035 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
3036 NodeClassNames[argument(0)->Opcode()],
3037 NodeClassNames[argument(1)->Opcode()],
3038 NodeClassNames[argument(2)->Opcode()]);
3039 return false; // not enough info for intrinsification
3040 }
3041
3042 if (!is_klass_initialized(mask_klass)) {
3043 log_if_needed(" ** klass argument not initialized");
3044 return false;
3045 }
3046
3047 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3048 if (!elem_type->is_primitive_type()) {
3049 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3050 return false; // should be primitive type
3051 }
3052
3053 int num_elem = vlen->get_con();
3054 BasicType elem_bt = elem_type->basic_type();
|
302 // E>
303 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
304 // int length, V v1, V v2, M m,
305 // BinaryOperation<V, M> defaultImpl)
306 //
307 // public static
308 // <V extends Vector<E>,
309 // M extends VectorMask<E>,
310 // E>
311 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
312 // int length, V v1, V v2, V v3, M m,
313 // TernaryOperation<V, M> defaultImpl)
314 //
315 bool LibraryCallKit::inline_vector_nary_operation(int n) {
316 const TypeInt* opr = gvn().type(argument(0))->isa_int();
317 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
318 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
319 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
320 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
321
322 if (opr == nullptr || !opr->is_con() ||
323 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
324 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
325 vlen == nullptr || !vlen->is_con()) {
326 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
327 NodeClassNames[argument(0)->Opcode()],
328 NodeClassNames[argument(1)->Opcode()],
329 NodeClassNames[argument(3)->Opcode()],
330 NodeClassNames[argument(4)->Opcode()]);
331 return false; // not enough info for intrinsification
332 }
333
334 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
335 if (!elem_type->is_primitive_type()) {
336 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
337 return false; // should be primitive type
338 }
339 if (!is_klass_initialized(vector_klass)) {
340 log_if_needed(" ** klass argument not initialized");
341 return false;
342 }
343
344 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
345 // is used to control this operation.
506 operation = gvn().transform(operation);
507
508 // Wrap it up in VectorBox to keep object type information.
509 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
510 set_result(vbox);
511 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
512 return true;
513 }
514
515 // public static
516 // <E, M>
517 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
518 // int length, M m, VectorMaskOp<M> defaultImpl)
519 bool LibraryCallKit::inline_vector_mask_operation() {
520 const TypeInt* oper = gvn().type(argument(0))->isa_int();
521 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
522 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
523 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
524 Node* mask = argument(4);
525
526 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
527 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
528 vlen == nullptr || !vlen->is_con() ||
529 oper == nullptr || !oper->is_con() ||
530 mask->is_top()) {
531 return false; // dead code
532 }
533
534 if (!is_klass_initialized(mask_klass)) {
535 log_if_needed(" ** klass argument not initialized");
536 return false;
537 }
538
539 int num_elem = vlen->get_con();
540 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
541 BasicType elem_bt = elem_type->basic_type();
542
543 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
544 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
545 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
546 mopc, num_elem, type2name(elem_bt));
547 return false; // not supported
548 }
549
550 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
570 return true;
571 }
572
573 // public static
574 // <M,
575 // S extends VectorSpecies<E>,
576 // E>
577 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
578 // long bits, int mode, S s,
579 // BroadcastOperation<M, E, S> defaultImpl)
580 bool LibraryCallKit::inline_vector_frombits_coerced() {
581 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
582 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
583 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
584 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
585 // Mode argument determines the mode of operation it can take following values:-
586 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
587 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
588 const TypeInt* mode = gvn().type(argument(5))->isa_int();
589
590 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
591 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
592 vlen == nullptr || !vlen->is_con() ||
593 bits_type == nullptr ||
594 mode == nullptr || !mode->is_con()) {
595 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
596 NodeClassNames[argument(0)->Opcode()],
597 NodeClassNames[argument(1)->Opcode()],
598 NodeClassNames[argument(2)->Opcode()],
599 NodeClassNames[argument(5)->Opcode()]);
600 return false; // not enough info for intrinsification
601 }
602
603 if (!is_klass_initialized(vector_klass)) {
604 log_if_needed(" ** klass argument not initialized");
605 return false;
606 }
607 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
608 if (!elem_type->is_primitive_type()) {
609 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
610 return false; // should be primitive type
611 }
612 BasicType elem_bt = elem_type->basic_type();
613 int num_elem = vlen->get_con();
614 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
700 // int length,
701 // Object base, long offset, // Unsafe addressing
702 // boolean fromSegment,
703 // C container, long index, S s, // Arguments for default implementation
704 // LoadOperation<C, VM, S> defaultImpl) {
705 // public static
706 // <C,
707 // V extends VectorPayload>
708 // void store(Class<?> vClass, Class<?> eClass,
709 // int length,
710 // Object base, long offset, // Unsafe addressing
711 // boolean fromSegment,
712 // V v, C container, long index, // Arguments for default implementation
713 // StoreVectorOperation<C, V> defaultImpl) {
714 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
715 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
716 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
717 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
718 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
719
720 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
721 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
722 vlen == nullptr || !vlen->is_con() ||
723 from_ms == nullptr || !from_ms->is_con()) {
724 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
725 NodeClassNames[argument(0)->Opcode()],
726 NodeClassNames[argument(1)->Opcode()],
727 NodeClassNames[argument(2)->Opcode()],
728 NodeClassNames[argument(6)->Opcode()]);
729 return false; // not enough info for intrinsification
730 }
731 if (!is_klass_initialized(vector_klass)) {
732 log_if_needed(" ** klass argument not initialized");
733 return false;
734 }
735
736 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
737 if (!elem_type->is_primitive_type()) {
738 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
739 return false; // should be primitive type
740 }
741 BasicType elem_bt = elem_type->basic_type();
742 int num_elem = vlen->get_con();
743
910 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
911 // public static
912 // <C,
913 // V extends Vector<E>,
914 // M extends VectorMask<E>,
915 // E>
916 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
917 // int length,
918 // Object base, long offset, // Unsafe addressing
919 // boolean fromSegment,
920 // V v, M m, C container, long index, // Arguments for default implementation
921 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
922
923 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
924 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
925 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
926 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
927 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
928 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
929
930 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
931 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
932 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
933 vlen == nullptr || !vlen->is_con() ||
934 from_ms == nullptr || !from_ms->is_con()) {
935 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
936 NodeClassNames[argument(0)->Opcode()],
937 NodeClassNames[argument(1)->Opcode()],
938 NodeClassNames[argument(2)->Opcode()],
939 NodeClassNames[argument(3)->Opcode()],
940 NodeClassNames[argument(7)->Opcode()]);
941 return false; // not enough info for intrinsification
942 }
943 if (!is_klass_initialized(vector_klass)) {
944 log_if_needed(" ** klass argument not initialized");
945 return false;
946 }
947
948 if (!is_klass_initialized(mask_klass)) {
949 log_if_needed(" ** mask klass argument not initialized");
950 return false;
951 }
952
953 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
954 if (!elem_type->is_primitive_type()) {
1151 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1152 //
1153 // <C,
1154 // V extends Vector<E>,
1155 // W extends Vector<Integer>,
1156 // M extends VectorMask<E>,
1157 // E>
1158 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1159 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1160 // W index_vector, V v, M m,
1161 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1162 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1163 //
1164 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1165 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1166 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1167 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1168 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1169 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1170
1171 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1172 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1173 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1174 vlen == nullptr || !vlen->is_con() ||
1175 vector_idx_klass == nullptr || vector_idx_klass->const_oop() == nullptr) {
1176 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1177 NodeClassNames[argument(0)->Opcode()],
1178 NodeClassNames[argument(2)->Opcode()],
1179 NodeClassNames[argument(3)->Opcode()],
1180 NodeClassNames[argument(4)->Opcode()]);
1181 return false; // not enough info for intrinsification
1182 }
1183
1184 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1185 log_if_needed(" ** klass argument not initialized");
1186 return false;
1187 }
1188
1189 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1190 if (!elem_type->is_primitive_type()) {
1191 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1192 return false; // should be primitive type
1193 }
1194
1195 BasicType elem_bt = elem_type->basic_type();
1341 destruct_map_clone(old_map);
1342
1343 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1344 return true;
1345 }
1346
1347 // public static
1348 // <V extends Vector<E>,
1349 // M extends VectorMask<E>,
1350 // E>
1351 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1352 // Class<E> elementType, int length, V v, M m,
1353 // ReductionOperation<V, M> defaultImpl)
1354 bool LibraryCallKit::inline_vector_reduction() {
1355 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1356 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1357 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1358 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1359 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1360
1361 if (opr == nullptr || !opr->is_con() ||
1362 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1363 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1364 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1365 vlen == nullptr || !vlen->is_con()) {
1366 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1367 NodeClassNames[argument(0)->Opcode()],
1368 NodeClassNames[argument(1)->Opcode()],
1369 NodeClassNames[argument(3)->Opcode()],
1370 NodeClassNames[argument(4)->Opcode()]);
1371 return false; // not enough info for intrinsification
1372 }
1373 if (!is_klass_initialized(vector_klass)) {
1374 log_if_needed(" ** klass argument not initialized");
1375 return false;
1376 }
1377 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1378 if (!elem_type->is_primitive_type()) {
1379 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1380 return false; // should be primitive type
1381 }
1382
1383 const Type* vmask_type = gvn().type(argument(6));
1384 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1385 if (is_masked_op) {
1482 bits = value; // no conversion needed
1483 break;
1484 }
1485 default: fatal("%s", type2name(elem_bt));
1486 }
1487 set_result(bits);
1488 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1489 return true;
1490 }
1491
1492 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1493 // V v1, V v2,
1494 // BiFunction<V, V, Boolean> defaultImpl)
1495 //
1496 bool LibraryCallKit::inline_vector_test() {
1497 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1498 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1499 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1500 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1501
1502 if (cond == nullptr || !cond->is_con() ||
1503 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1504 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1505 vlen == nullptr || !vlen->is_con()) {
1506 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1507 NodeClassNames[argument(0)->Opcode()],
1508 NodeClassNames[argument(1)->Opcode()],
1509 NodeClassNames[argument(2)->Opcode()],
1510 NodeClassNames[argument(3)->Opcode()]);
1511 return false; // not enough info for intrinsification
1512 }
1513 if (!is_klass_initialized(vector_klass)) {
1514 log_if_needed(" ** klass argument not initialized");
1515 return false;
1516 }
1517 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1518 if (!elem_type->is_primitive_type()) {
1519 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1520 return false; // should be primitive type
1521 }
1522 BasicType elem_bt = elem_type->basic_type();
1523 int num_elem = vlen->get_con();
1524 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1525 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2515 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2516 return true;
2517 }
2518
2519 // public static
2520 // <VM extends VectorPayload,
2521 // E>
2522 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2523 // int length,
2524 // VM vm, int i,
2525 // VecExtractOp<VM> defaultImpl)
2526 bool LibraryCallKit::inline_vector_extract() {
2527 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2528 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2529 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2530 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2531
2532 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2533 return false; // dead code
2534 }
2535 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2536 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2537 NodeClassNames[argument(0)->Opcode()],
2538 NodeClassNames[argument(1)->Opcode()],
2539 NodeClassNames[argument(2)->Opcode()]);
2540 return false; // not enough info for intrinsification
2541 }
2542 if (!is_klass_initialized(vector_klass)) {
2543 log_if_needed(" ** klass argument not initialized");
2544 return false;
2545 }
2546 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2547 if (!elem_type->is_primitive_type()) {
2548 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2549 return false; // should be primitive type
2550 }
2551 BasicType elem_bt = elem_type->basic_type();
2552 int num_elem = vlen->get_con();
2553
2554 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2555 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2818 set_result(vbox);
2819 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2820 return true;
2821 }
2822
2823 // public static
2824 // <V extends Vector<E>,
2825 // M extends VectorMask<E>,
2826 // E>
2827 // V compressExpandOp(int opr,
2828 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2829 // int length, V v, M m,
2830 // CompressExpandOperation<V, M> defaultImpl)
2831 bool LibraryCallKit::inline_vector_compress_expand() {
2832 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2833 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2834 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2835 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2836 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2837
2838 if (opr == nullptr || !opr->is_con() ||
2839 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2840 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
2841 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2842 vlen == nullptr || !vlen->is_con()) {
2843 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2844 NodeClassNames[argument(0)->Opcode()],
2845 NodeClassNames[argument(1)->Opcode()],
2846 NodeClassNames[argument(2)->Opcode()],
2847 NodeClassNames[argument(3)->Opcode()],
2848 NodeClassNames[argument(4)->Opcode()]);
2849 return false; // not enough info for intrinsification
2850 }
2851
2852 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2853 log_if_needed(" ** klass argument not initialized");
2854 return false;
2855 }
2856
2857 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2858 if (!elem_type->is_primitive_type()) {
2859 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2860 return false; // should be primitive type
2861 }
2862
2901 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2902 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2903 set_result(vbox);
2904 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2905 return true;
2906 }
2907
2908 // public static
2909 // <V extends Vector<E>,
2910 // E,
2911 // S extends VectorSpecies<E>>
2912 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2913 // int length,
2914 // V v, int step, S s,
2915 // IndexOperation<V, S> defaultImpl)
2916 bool LibraryCallKit::inline_index_vector() {
2917 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2918 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2919 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2920
2921 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2922 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2923 vlen == nullptr || !vlen->is_con() ) {
2924 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2925 NodeClassNames[argument(0)->Opcode()],
2926 NodeClassNames[argument(1)->Opcode()],
2927 NodeClassNames[argument(2)->Opcode()]);
2928 return false; // not enough info for intrinsification
2929 }
2930
2931 if (!is_klass_initialized(vector_klass)) {
2932 log_if_needed(" ** klass argument not initialized");
2933 return false;
2934 }
2935
2936 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2937 if (!elem_type->is_primitive_type()) {
2938 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2939 return false; // should be primitive type
2940 }
2941
2942 int num_elem = vlen->get_con();
2943 BasicType elem_bt = elem_type->basic_type();
3035 if (needs_add) {
3036 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3037 }
3038 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3039 set_result(vbox);
3040 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3041 return true;
3042 }
3043
3044 // public static
3045 // <E,
3046 // M extends VectorMask<E>>
3047 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3048 // long offset, long limit,
3049 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3050 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3051 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
3052 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3053 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3054
3055 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
3056 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
3057 vlen == nullptr || !vlen->is_con()) {
3058 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
3059 NodeClassNames[argument(0)->Opcode()],
3060 NodeClassNames[argument(1)->Opcode()],
3061 NodeClassNames[argument(2)->Opcode()]);
3062 return false; // not enough info for intrinsification
3063 }
3064
3065 if (!is_klass_initialized(mask_klass)) {
3066 log_if_needed(" ** klass argument not initialized");
3067 return false;
3068 }
3069
3070 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3071 if (!elem_type->is_primitive_type()) {
3072 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3073 return false; // should be primitive type
3074 }
3075
3076 int num_elem = vlen->get_con();
3077 BasicType elem_bt = elem_type->basic_type();
|