301 // E>
302 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
303 // int length, V v1, V v2, M m,
304 // BinaryOperation<V, M> defaultImpl)
305 //
306 // public static
307 // <V extends Vector<E>,
308 // M extends VectorMask<E>,
309 // E>
310 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
311 // int length, V v1, V v2, V v3, M m,
312 // TernaryOperation<V, M> defaultImpl)
313 //
314 bool LibraryCallKit::inline_vector_nary_operation(int n) {
315 const TypeInt* opr = gvn().type(argument(0))->isa_int();
316 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
317 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
318 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
319 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
320
321 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
322 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
323 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
324 NodeClassNames[argument(0)->Opcode()],
325 NodeClassNames[argument(1)->Opcode()],
326 NodeClassNames[argument(3)->Opcode()],
327 NodeClassNames[argument(4)->Opcode()]);
328 return false; // not enough info for intrinsification
329 }
330
331 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
332 if (!elem_type->is_primitive_type()) {
333 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
334 return false; // should be primitive type
335 }
336 if (!is_klass_initialized(vector_klass)) {
337 log_if_needed(" ** klass argument not initialized");
338 return false;
339 }
340
341 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
342 // is used to control this operation.
503 operation = gvn().transform(operation);
504
505 // Wrap it up in VectorBox to keep object type information.
506 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
507 set_result(vbox);
508 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
509 return true;
510 }
511
512 // public static
513 // <E, M>
514 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
515 // int length, M m, VectorMaskOp<M> defaultImpl)
516 bool LibraryCallKit::inline_vector_mask_operation() {
517 const TypeInt* oper = gvn().type(argument(0))->isa_int();
518 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
519 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
520 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
521 Node* mask = argument(4);
522
523 if (mask_klass == nullptr || elem_klass == nullptr || mask->is_top() || vlen == nullptr) {
524 return false; // dead code
525 }
526
527 if (!is_klass_initialized(mask_klass)) {
528 log_if_needed(" ** klass argument not initialized");
529 return false;
530 }
531
532 int num_elem = vlen->get_con();
533 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
534 BasicType elem_bt = elem_type->basic_type();
535
536 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
537 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
538 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
539 mopc, num_elem, type2name(elem_bt));
540 return false; // not supported
541 }
542
543 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
563 return true;
564 }
565
566 // public static
567 // <M,
568 // S extends VectorSpecies<E>,
569 // E>
570 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
571 // long bits, int mode, S s,
572 // BroadcastOperation<M, E, S> defaultImpl)
573 bool LibraryCallKit::inline_vector_frombits_coerced() {
574 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
575 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
576 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
577 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
578 // Mode argument determines the mode of operation it can take following values:-
579 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
580 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
581 const TypeInt* mode = gvn().type(argument(5))->isa_int();
582
583 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || mode == nullptr ||
584 bits_type == nullptr || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr ||
585 !vlen->is_con() || !mode->is_con()) {
586 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
587 NodeClassNames[argument(0)->Opcode()],
588 NodeClassNames[argument(1)->Opcode()],
589 NodeClassNames[argument(2)->Opcode()],
590 NodeClassNames[argument(5)->Opcode()]);
591 return false; // not enough info for intrinsification
592 }
593
594 if (!is_klass_initialized(vector_klass)) {
595 log_if_needed(" ** klass argument not initialized");
596 return false;
597 }
598 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
599 if (!elem_type->is_primitive_type()) {
600 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
601 return false; // should be primitive type
602 }
603 BasicType elem_bt = elem_type->basic_type();
604 int num_elem = vlen->get_con();
605 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
691 // int length,
692 // Object base, long offset, // Unsafe addressing
693 // boolean fromSegment,
694 // C container, long index, S s, // Arguments for default implementation
695 // LoadOperation<C, VM, S> defaultImpl) {
696 // public static
697 // <C,
698 // V extends VectorPayload>
699 // void store(Class<?> vClass, Class<?> eClass,
700 // int length,
701 // Object base, long offset, // Unsafe addressing
702 // boolean fromSegment,
703 // V v, C container, long index, // Arguments for default implementation
704 // StoreVectorOperation<C, V> defaultImpl) {
705 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
706 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
707 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
708 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
709 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
710
711 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || !from_ms->is_con() ||
712 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
713 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
714 NodeClassNames[argument(0)->Opcode()],
715 NodeClassNames[argument(1)->Opcode()],
716 NodeClassNames[argument(2)->Opcode()],
717 NodeClassNames[argument(6)->Opcode()]);
718 return false; // not enough info for intrinsification
719 }
720 if (!is_klass_initialized(vector_klass)) {
721 log_if_needed(" ** klass argument not initialized");
722 return false;
723 }
724
725 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
726 if (!elem_type->is_primitive_type()) {
727 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
728 return false; // should be primitive type
729 }
730 BasicType elem_bt = elem_type->basic_type();
731 int num_elem = vlen->get_con();
732
899 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
900 // public static
901 // <C,
902 // V extends Vector<E>,
903 // M extends VectorMask<E>,
904 // E>
905 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
906 // int length,
907 // Object base, long offset, // Unsafe addressing
908 // boolean fromSegment,
909 // V v, M m, C container, long index, // Arguments for default implementation
910 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
911
912 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
913 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
914 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
915 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
916 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
917 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
918
919 if (vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
920 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr || from_ms == nullptr ||
921 elem_klass->const_oop() == nullptr || !vlen->is_con() || !from_ms->is_con()) {
922 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
923 NodeClassNames[argument(0)->Opcode()],
924 NodeClassNames[argument(1)->Opcode()],
925 NodeClassNames[argument(2)->Opcode()],
926 NodeClassNames[argument(3)->Opcode()],
927 NodeClassNames[argument(7)->Opcode()]);
928 return false; // not enough info for intrinsification
929 }
930 if (!is_klass_initialized(vector_klass)) {
931 log_if_needed(" ** klass argument not initialized");
932 return false;
933 }
934
935 if (!is_klass_initialized(mask_klass)) {
936 log_if_needed(" ** mask klass argument not initialized");
937 return false;
938 }
939
940 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
941 if (!elem_type->is_primitive_type()) {
1138 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1139 //
1140 // <C,
1141 // V extends Vector<E>,
1142 // W extends Vector<Integer>,
1143 // M extends VectorMask<E>,
1144 // E>
1145 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1146 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1147 // W index_vector, V v, M m,
1148 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1149 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1150 //
1151 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1152 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1153 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1154 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1155 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1156 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1157
1158 if (vector_klass == nullptr || elem_klass == nullptr || vector_idx_klass == nullptr || vlen == nullptr ||
1159 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || vector_idx_klass->const_oop() == nullptr || !vlen->is_con()) {
1160 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1161 NodeClassNames[argument(0)->Opcode()],
1162 NodeClassNames[argument(2)->Opcode()],
1163 NodeClassNames[argument(3)->Opcode()],
1164 NodeClassNames[argument(4)->Opcode()]);
1165 return false; // not enough info for intrinsification
1166 }
1167
1168 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1169 log_if_needed(" ** klass argument not initialized");
1170 return false;
1171 }
1172
1173 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1174 if (!elem_type->is_primitive_type()) {
1175 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1176 return false; // should be primitive type
1177 }
1178
1179 BasicType elem_bt = elem_type->basic_type();
1325 destruct_map_clone(old_map);
1326
1327 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1328 return true;
1329 }
1330
1331 // public static
1332 // <V extends Vector<E>,
1333 // M extends VectorMask<E>,
1334 // E>
1335 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1336 // Class<E> elementType, int length, V v, M m,
1337 // ReductionOperation<V, M> defaultImpl)
1338 bool LibraryCallKit::inline_vector_reduction() {
1339 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1340 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1341 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1342 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1343 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1344
1345 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1346 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1347 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1348 NodeClassNames[argument(0)->Opcode()],
1349 NodeClassNames[argument(1)->Opcode()],
1350 NodeClassNames[argument(3)->Opcode()],
1351 NodeClassNames[argument(4)->Opcode()]);
1352 return false; // not enough info for intrinsification
1353 }
1354 if (!is_klass_initialized(vector_klass)) {
1355 log_if_needed(" ** klass argument not initialized");
1356 return false;
1357 }
1358 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1359 if (!elem_type->is_primitive_type()) {
1360 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1361 return false; // should be primitive type
1362 }
1363
1364 const Type* vmask_type = gvn().type(argument(6));
1365 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1366 if (is_masked_op) {
1463 bits = value; // no conversion needed
1464 break;
1465 }
1466 default: fatal("%s", type2name(elem_bt));
1467 }
1468 set_result(bits);
1469 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1470 return true;
1471 }
1472
1473 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1474 // V v1, V v2,
1475 // BiFunction<V, V, Boolean> defaultImpl)
1476 //
1477 bool LibraryCallKit::inline_vector_test() {
1478 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1479 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1480 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1481 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1482
1483 if (cond == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1484 !cond->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1485 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1486 NodeClassNames[argument(0)->Opcode()],
1487 NodeClassNames[argument(1)->Opcode()],
1488 NodeClassNames[argument(2)->Opcode()],
1489 NodeClassNames[argument(3)->Opcode()]);
1490 return false; // not enough info for intrinsification
1491 }
1492 if (!is_klass_initialized(vector_klass)) {
1493 log_if_needed(" ** klass argument not initialized");
1494 return false;
1495 }
1496 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1497 if (!elem_type->is_primitive_type()) {
1498 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1499 return false; // should be primitive type
1500 }
1501 BasicType elem_bt = elem_type->basic_type();
1502 int num_elem = vlen->get_con();
1503 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1504 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2494 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2495 return true;
2496 }
2497
2498 // public static
2499 // <VM extends VectorPayload,
2500 // E>
2501 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2502 // int length,
2503 // VM vm, int i,
2504 // VecExtractOp<VM> defaultImpl)
2505 bool LibraryCallKit::inline_vector_extract() {
2506 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2507 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2508 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2509 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2510
2511 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2512 return false; // dead code
2513 }
2514 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2515 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2516 NodeClassNames[argument(0)->Opcode()],
2517 NodeClassNames[argument(1)->Opcode()],
2518 NodeClassNames[argument(2)->Opcode()]);
2519 return false; // not enough info for intrinsification
2520 }
2521 if (!is_klass_initialized(vector_klass)) {
2522 log_if_needed(" ** klass argument not initialized");
2523 return false;
2524 }
2525 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2526 if (!elem_type->is_primitive_type()) {
2527 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2528 return false; // should be primitive type
2529 }
2530 BasicType elem_bt = elem_type->basic_type();
2531 int num_elem = vlen->get_con();
2532
2533 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2534 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2797 set_result(vbox);
2798 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2799 return true;
2800 }
2801
2802 // public static
2803 // <V extends Vector<E>,
2804 // M extends VectorMask<E>,
2805 // E>
2806 // V compressExpandOp(int opr,
2807 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2808 // int length, V v, M m,
2809 // CompressExpandOperation<V, M> defaultImpl)
2810 bool LibraryCallKit::inline_vector_compress_expand() {
2811 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2812 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2813 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2814 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2815 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2816
2817 if (vector_klass == nullptr || elem_klass == nullptr || mask_klass == nullptr || vlen == nullptr ||
2818 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
2819 elem_klass->const_oop() == nullptr || !vlen->is_con() || !opr->is_con()) {
2820 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2821 NodeClassNames[argument(0)->Opcode()],
2822 NodeClassNames[argument(1)->Opcode()],
2823 NodeClassNames[argument(2)->Opcode()],
2824 NodeClassNames[argument(3)->Opcode()],
2825 NodeClassNames[argument(4)->Opcode()]);
2826 return false; // not enough info for intrinsification
2827 }
2828
2829 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2830 log_if_needed(" ** klass argument not initialized");
2831 return false;
2832 }
2833
2834 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2835 if (!elem_type->is_primitive_type()) {
2836 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2837 return false; // should be primitive type
2838 }
2839
2878 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2879 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2880 set_result(vbox);
2881 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2882 return true;
2883 }
2884
2885 // public static
2886 // <V extends Vector<E>,
2887 // E,
2888 // S extends VectorSpecies<E>>
2889 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2890 // int length,
2891 // V v, int step, S s,
2892 // IndexOperation<V, S> defaultImpl)
2893 bool LibraryCallKit::inline_index_vector() {
2894 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2895 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2896 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2897
2898 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2899 vector_klass->const_oop() == nullptr || !vlen->is_con() ||
2900 elem_klass->const_oop() == nullptr) {
2901 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2902 NodeClassNames[argument(0)->Opcode()],
2903 NodeClassNames[argument(1)->Opcode()],
2904 NodeClassNames[argument(2)->Opcode()]);
2905 return false; // not enough info for intrinsification
2906 }
2907
2908 if (!is_klass_initialized(vector_klass)) {
2909 log_if_needed(" ** klass argument not initialized");
2910 return false;
2911 }
2912
2913 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2914 if (!elem_type->is_primitive_type()) {
2915 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2916 return false; // should be primitive type
2917 }
2918
2919 int num_elem = vlen->get_con();
2920 BasicType elem_bt = elem_type->basic_type();
3012 if (needs_add) {
3013 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3014 }
3015 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3016 set_result(vbox);
3017 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3018 return true;
3019 }
3020
3021 // public static
3022 // <E,
3023 // M extends VectorMask<E>>
3024 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3025 // long offset, long limit,
3026 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3027 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3028 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
3029 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3030 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3031
3032 if (mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
3033 mask_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
3034 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
3035 NodeClassNames[argument(0)->Opcode()],
3036 NodeClassNames[argument(1)->Opcode()],
3037 NodeClassNames[argument(2)->Opcode()]);
3038 return false; // not enough info for intrinsification
3039 }
3040
3041 if (!is_klass_initialized(mask_klass)) {
3042 log_if_needed(" ** klass argument not initialized");
3043 return false;
3044 }
3045
3046 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3047 if (!elem_type->is_primitive_type()) {
3048 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3049 return false; // should be primitive type
3050 }
3051
3052 int num_elem = vlen->get_con();
3053 BasicType elem_bt = elem_type->basic_type();
|
301 // E>
302 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
303 // int length, V v1, V v2, M m,
304 // BinaryOperation<V, M> defaultImpl)
305 //
306 // public static
307 // <V extends Vector<E>,
308 // M extends VectorMask<E>,
309 // E>
310 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
311 // int length, V v1, V v2, V v3, M m,
312 // TernaryOperation<V, M> defaultImpl)
313 //
314 bool LibraryCallKit::inline_vector_nary_operation(int n) {
315 const TypeInt* opr = gvn().type(argument(0))->isa_int();
316 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
317 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
318 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
319 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
320
321 if (opr == nullptr || !opr->is_con() ||
322 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
323 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
324 vlen == nullptr || !vlen->is_con()) {
325 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
326 NodeClassNames[argument(0)->Opcode()],
327 NodeClassNames[argument(1)->Opcode()],
328 NodeClassNames[argument(3)->Opcode()],
329 NodeClassNames[argument(4)->Opcode()]);
330 return false; // not enough info for intrinsification
331 }
332
333 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
334 if (!elem_type->is_primitive_type()) {
335 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
336 return false; // should be primitive type
337 }
338 if (!is_klass_initialized(vector_klass)) {
339 log_if_needed(" ** klass argument not initialized");
340 return false;
341 }
342
343 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
344 // is used to control this operation.
505 operation = gvn().transform(operation);
506
507 // Wrap it up in VectorBox to keep object type information.
508 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
509 set_result(vbox);
510 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
511 return true;
512 }
513
514 // public static
515 // <E, M>
516 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
517 // int length, M m, VectorMaskOp<M> defaultImpl)
518 bool LibraryCallKit::inline_vector_mask_operation() {
519 const TypeInt* oper = gvn().type(argument(0))->isa_int();
520 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
521 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
522 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
523 Node* mask = argument(4);
524
525 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
526 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
527 vlen == nullptr || !vlen->is_con() ||
528 oper == nullptr || !oper->is_con() ||
529 mask->is_top()) {
530 return false; // dead code
531 }
532
533 if (!is_klass_initialized(mask_klass)) {
534 log_if_needed(" ** klass argument not initialized");
535 return false;
536 }
537
538 int num_elem = vlen->get_con();
539 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
540 BasicType elem_bt = elem_type->basic_type();
541
542 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
543 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
544 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
545 mopc, num_elem, type2name(elem_bt));
546 return false; // not supported
547 }
548
549 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
569 return true;
570 }
571
572 // public static
573 // <M,
574 // S extends VectorSpecies<E>,
575 // E>
576 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
577 // long bits, int mode, S s,
578 // BroadcastOperation<M, E, S> defaultImpl)
579 bool LibraryCallKit::inline_vector_frombits_coerced() {
580 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
581 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
582 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
583 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
584 // Mode argument determines the mode of operation it can take following values:-
585 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
586 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
587 const TypeInt* mode = gvn().type(argument(5))->isa_int();
588
589 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
590 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
591 vlen == nullptr || !vlen->is_con() ||
592 bits_type == nullptr ||
593 mode == nullptr || !mode->is_con()) {
594 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
595 NodeClassNames[argument(0)->Opcode()],
596 NodeClassNames[argument(1)->Opcode()],
597 NodeClassNames[argument(2)->Opcode()],
598 NodeClassNames[argument(5)->Opcode()]);
599 return false; // not enough info for intrinsification
600 }
601
602 if (!is_klass_initialized(vector_klass)) {
603 log_if_needed(" ** klass argument not initialized");
604 return false;
605 }
606 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
607 if (!elem_type->is_primitive_type()) {
608 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
609 return false; // should be primitive type
610 }
611 BasicType elem_bt = elem_type->basic_type();
612 int num_elem = vlen->get_con();
613 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
699 // int length,
700 // Object base, long offset, // Unsafe addressing
701 // boolean fromSegment,
702 // C container, long index, S s, // Arguments for default implementation
703 // LoadOperation<C, VM, S> defaultImpl) {
704 // public static
705 // <C,
706 // V extends VectorPayload>
707 // void store(Class<?> vClass, Class<?> eClass,
708 // int length,
709 // Object base, long offset, // Unsafe addressing
710 // boolean fromSegment,
711 // V v, C container, long index, // Arguments for default implementation
712 // StoreVectorOperation<C, V> defaultImpl) {
713 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
714 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
715 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
716 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
717 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
718
719 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
720 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
721 vlen == nullptr || !vlen->is_con() ||
722 from_ms == nullptr || !from_ms->is_con()) {
723 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
724 NodeClassNames[argument(0)->Opcode()],
725 NodeClassNames[argument(1)->Opcode()],
726 NodeClassNames[argument(2)->Opcode()],
727 NodeClassNames[argument(6)->Opcode()]);
728 return false; // not enough info for intrinsification
729 }
730 if (!is_klass_initialized(vector_klass)) {
731 log_if_needed(" ** klass argument not initialized");
732 return false;
733 }
734
735 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
736 if (!elem_type->is_primitive_type()) {
737 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
738 return false; // should be primitive type
739 }
740 BasicType elem_bt = elem_type->basic_type();
741 int num_elem = vlen->get_con();
742
909 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
910 // public static
911 // <C,
912 // V extends Vector<E>,
913 // M extends VectorMask<E>,
914 // E>
915 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
916 // int length,
917 // Object base, long offset, // Unsafe addressing
918 // boolean fromSegment,
919 // V v, M m, C container, long index, // Arguments for default implementation
920 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
921
922 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
923 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
924 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
925 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
926 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
927 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
928
929 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
930 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
931 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
932 vlen == nullptr || !vlen->is_con() ||
933 from_ms == nullptr || !from_ms->is_con()) {
934 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
935 NodeClassNames[argument(0)->Opcode()],
936 NodeClassNames[argument(1)->Opcode()],
937 NodeClassNames[argument(2)->Opcode()],
938 NodeClassNames[argument(3)->Opcode()],
939 NodeClassNames[argument(7)->Opcode()]);
940 return false; // not enough info for intrinsification
941 }
942 if (!is_klass_initialized(vector_klass)) {
943 log_if_needed(" ** klass argument not initialized");
944 return false;
945 }
946
947 if (!is_klass_initialized(mask_klass)) {
948 log_if_needed(" ** mask klass argument not initialized");
949 return false;
950 }
951
952 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
953 if (!elem_type->is_primitive_type()) {
1150 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1151 //
1152 // <C,
1153 // V extends Vector<E>,
1154 // W extends Vector<Integer>,
1155 // M extends VectorMask<E>,
1156 // E>
1157 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1158 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1159 // W index_vector, V v, M m,
1160 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1161 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1162 //
1163 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1164 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1165 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1166 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1167 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1168 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1169
1170 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1171 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1172 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1173 vlen == nullptr || !vlen->is_con() ||
1174 vector_idx_klass == nullptr || vector_idx_klass->const_oop() == nullptr) {
1175 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1176 NodeClassNames[argument(0)->Opcode()],
1177 NodeClassNames[argument(2)->Opcode()],
1178 NodeClassNames[argument(3)->Opcode()],
1179 NodeClassNames[argument(4)->Opcode()]);
1180 return false; // not enough info for intrinsification
1181 }
1182
1183 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1184 log_if_needed(" ** klass argument not initialized");
1185 return false;
1186 }
1187
1188 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1189 if (!elem_type->is_primitive_type()) {
1190 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1191 return false; // should be primitive type
1192 }
1193
1194 BasicType elem_bt = elem_type->basic_type();
1340 destruct_map_clone(old_map);
1341
1342 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1343 return true;
1344 }
1345
1346 // public static
1347 // <V extends Vector<E>,
1348 // M extends VectorMask<E>,
1349 // E>
1350 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1351 // Class<E> elementType, int length, V v, M m,
1352 // ReductionOperation<V, M> defaultImpl)
1353 bool LibraryCallKit::inline_vector_reduction() {
1354 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1355 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1356 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1357 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1358 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1359
1360 if (opr == nullptr || !opr->is_con() ||
1361 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1362 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1363 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1364 vlen == nullptr || !vlen->is_con()) {
1365 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1366 NodeClassNames[argument(0)->Opcode()],
1367 NodeClassNames[argument(1)->Opcode()],
1368 NodeClassNames[argument(3)->Opcode()],
1369 NodeClassNames[argument(4)->Opcode()]);
1370 return false; // not enough info for intrinsification
1371 }
1372 if (!is_klass_initialized(vector_klass)) {
1373 log_if_needed(" ** klass argument not initialized");
1374 return false;
1375 }
1376 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1377 if (!elem_type->is_primitive_type()) {
1378 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1379 return false; // should be primitive type
1380 }
1381
1382 const Type* vmask_type = gvn().type(argument(6));
1383 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1384 if (is_masked_op) {
1481 bits = value; // no conversion needed
1482 break;
1483 }
1484 default: fatal("%s", type2name(elem_bt));
1485 }
1486 set_result(bits);
1487 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1488 return true;
1489 }
1490
1491 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1492 // V v1, V v2,
1493 // BiFunction<V, V, Boolean> defaultImpl)
1494 //
1495 bool LibraryCallKit::inline_vector_test() {
1496 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1497 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1498 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1499 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1500
1501 if (cond == nullptr || !cond->is_con() ||
1502 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1503 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1504 vlen == nullptr || !vlen->is_con()) {
1505 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1506 NodeClassNames[argument(0)->Opcode()],
1507 NodeClassNames[argument(1)->Opcode()],
1508 NodeClassNames[argument(2)->Opcode()],
1509 NodeClassNames[argument(3)->Opcode()]);
1510 return false; // not enough info for intrinsification
1511 }
1512 if (!is_klass_initialized(vector_klass)) {
1513 log_if_needed(" ** klass argument not initialized");
1514 return false;
1515 }
1516 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1517 if (!elem_type->is_primitive_type()) {
1518 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1519 return false; // should be primitive type
1520 }
1521 BasicType elem_bt = elem_type->basic_type();
1522 int num_elem = vlen->get_con();
1523 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1524 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2514 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2515 return true;
2516 }
2517
2518 // public static
2519 // <VM extends VectorPayload,
2520 // E>
2521 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2522 // int length,
2523 // VM vm, int i,
2524 // VecExtractOp<VM> defaultImpl)
2525 bool LibraryCallKit::inline_vector_extract() {
2526 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2527 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2528 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2529 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2530
2531 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2532 return false; // dead code
2533 }
2534 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2535 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2536 NodeClassNames[argument(0)->Opcode()],
2537 NodeClassNames[argument(1)->Opcode()],
2538 NodeClassNames[argument(2)->Opcode()]);
2539 return false; // not enough info for intrinsification
2540 }
2541 if (!is_klass_initialized(vector_klass)) {
2542 log_if_needed(" ** klass argument not initialized");
2543 return false;
2544 }
2545 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2546 if (!elem_type->is_primitive_type()) {
2547 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2548 return false; // should be primitive type
2549 }
2550 BasicType elem_bt = elem_type->basic_type();
2551 int num_elem = vlen->get_con();
2552
2553 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2554 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2817 set_result(vbox);
2818 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2819 return true;
2820 }
2821
2822 // public static
2823 // <V extends Vector<E>,
2824 // M extends VectorMask<E>,
2825 // E>
2826 // V compressExpandOp(int opr,
2827 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2828 // int length, V v, M m,
2829 // CompressExpandOperation<V, M> defaultImpl)
2830 bool LibraryCallKit::inline_vector_compress_expand() {
2831 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2832 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2833 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2834 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2835 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2836
2837 if (opr == nullptr || !opr->is_con() ||
2838 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2839 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
2840 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2841 vlen == nullptr || !vlen->is_con()) {
2842 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2843 NodeClassNames[argument(0)->Opcode()],
2844 NodeClassNames[argument(1)->Opcode()],
2845 NodeClassNames[argument(2)->Opcode()],
2846 NodeClassNames[argument(3)->Opcode()],
2847 NodeClassNames[argument(4)->Opcode()]);
2848 return false; // not enough info for intrinsification
2849 }
2850
2851 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2852 log_if_needed(" ** klass argument not initialized");
2853 return false;
2854 }
2855
2856 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2857 if (!elem_type->is_primitive_type()) {
2858 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2859 return false; // should be primitive type
2860 }
2861
2900 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2901 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2902 set_result(vbox);
2903 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2904 return true;
2905 }
2906
2907 // public static
2908 // <V extends Vector<E>,
2909 // E,
2910 // S extends VectorSpecies<E>>
2911 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2912 // int length,
2913 // V v, int step, S s,
2914 // IndexOperation<V, S> defaultImpl)
2915 bool LibraryCallKit::inline_index_vector() {
2916 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2917 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2918 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2919
2920 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2921 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2922 vlen == nullptr || !vlen->is_con() ) {
2923 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2924 NodeClassNames[argument(0)->Opcode()],
2925 NodeClassNames[argument(1)->Opcode()],
2926 NodeClassNames[argument(2)->Opcode()]);
2927 return false; // not enough info for intrinsification
2928 }
2929
2930 if (!is_klass_initialized(vector_klass)) {
2931 log_if_needed(" ** klass argument not initialized");
2932 return false;
2933 }
2934
2935 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2936 if (!elem_type->is_primitive_type()) {
2937 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2938 return false; // should be primitive type
2939 }
2940
2941 int num_elem = vlen->get_con();
2942 BasicType elem_bt = elem_type->basic_type();
3034 if (needs_add) {
3035 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3036 }
3037 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3038 set_result(vbox);
3039 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3040 return true;
3041 }
3042
3043 // public static
3044 // <E,
3045 // M extends VectorMask<E>>
3046 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3047 // long offset, long limit,
3048 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3049 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3050 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
3051 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3052 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3053
3054 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
3055 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
3056 vlen == nullptr || !vlen->is_con()) {
3057 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
3058 NodeClassNames[argument(0)->Opcode()],
3059 NodeClassNames[argument(1)->Opcode()],
3060 NodeClassNames[argument(2)->Opcode()]);
3061 return false; // not enough info for intrinsification
3062 }
3063
3064 if (!is_klass_initialized(mask_klass)) {
3065 log_if_needed(" ** klass argument not initialized");
3066 return false;
3067 }
3068
3069 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3070 if (!elem_type->is_primitive_type()) {
3071 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3072 return false; // should be primitive type
3073 }
3074
3075 int num_elem = vlen->get_con();
3076 BasicType elem_bt = elem_type->basic_type();
|