< prev index next >

src/hotspot/share/opto/vectorIntrinsics.cpp

Print this page

  65 
  66 static bool is_vector_shuffle(ciKlass* klass) {
  67   return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
  68 }
  69 
  70 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
  71                                                  VectorMaskUseType mask_use_type, bool has_scalar_args) {
  72   bool is_supported = true;
  73 
  74   // has_scalar_args flag is true only for non-constant scalar shift count,
  75   // since in this case shift needs to be broadcasted.
  76   if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
  77        (has_scalar_args &&
  78          !arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed))) {
  79     is_supported = false;
  80   }
  81 
  82   if (is_supported) {
  83     // Check whether mask unboxing is supported.
  84     if ((mask_use_type & VecMaskUseLoad) != 0) {
  85       if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt)) {

  86       #ifndef PRODUCT
  87         if (C->print_intrinsics()) {
  88           tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
  89                         NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
  90         }
  91       #endif
  92         return false;
  93       }
  94     }
  95 
  96     if ((mask_use_type & VecMaskUsePred) != 0) {
  97       if (!Matcher::has_predicated_vectors() ||
  98           !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
  99       #ifndef PRODUCT
 100         if (C->print_intrinsics()) {
 101           tty->print_cr("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
 102                         NodeClassNames[opc], type2name(elem_bt), num_elem);
 103         }
 104       #endif
 105         return false;

 245           tty->print_cr("  ** Not a svml call or load/store vector op (%s,%s,%d)",
 246                         NodeClassNames[sopc], type2name(type), num_elem);
 247         }
 248 #endif
 249         return false;
 250       }
 251     }
 252   }
 253 
 254   if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
 255       Matcher::supports_vector_variable_shifts() == false) {
 256     if (C->print_intrinsics()) {
 257       tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
 258                     NodeClassNames[sopc], type2name(type), num_elem);
 259     }
 260     return false;
 261   }
 262 
 263   // Check whether mask unboxing is supported.
 264   if ((mask_use_type & VecMaskUseLoad) != 0) {
 265     if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type)) {

 266     #ifndef PRODUCT
 267       if (C->print_intrinsics()) {
 268         tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
 269                       NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
 270       }
 271     #endif
 272       return false;
 273     }
 274   }
 275 
 276   // Check whether mask boxing is supported.
 277   if ((mask_use_type & VecMaskUseStore) != 0) {
 278     if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type)) {

 279     #ifndef PRODUCT
 280       if (C->print_intrinsics()) {
 281         tty->print_cr("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
 282                       NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
 283       }
 284     #endif
 285       return false;
 286     }
 287   }
 288 
 289   if ((mask_use_type & VecMaskUsePred) != 0) {
 290     bool is_supported = false;
 291     if (Matcher::has_predicated_vectors()) {
 292       if (VectorNode::is_vector_integral_negate(sopc)) {
 293         is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
 294       } else {
 295         is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
 296       }
 297     }
 298 

 544     switch (n) {
 545       case 1:
 546       case 2: {
 547         operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc));
 548         break;
 549       }
 550       case 3: {
 551         operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
 552         break;
 553       }
 554       default: fatal("unsupported arity: %d", n);
 555     }
 556   }
 557 
 558   if (is_masked_op && mask != NULL) {
 559     if (use_predicate) {
 560       operation->add_req(mask);
 561       operation->add_flag(Node::Flag_is_predicated_vector);
 562     } else {
 563       operation = gvn().transform(operation);

 564       operation = new VectorBlendNode(opd1, operation, mask);
 565     }
 566   }
 567   operation = gvn().transform(operation);
 568 
 569   // Wrap it up in VectorBox to keep object type information.
 570   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
 571   set_result(vbox);
 572   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 573   return true;
 574 }
 575 
 576 // <Sh extends VectorShuffle<E>,  E>
 577 //  Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
 578 //                  int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
 579 bool LibraryCallKit::inline_vector_shuffle_iota() {
 580   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
 581   const TypeInt*     vlen          = gvn().type(argument(3))->isa_int();
 582   const TypeInt*     start_val     = gvn().type(argument(4))->isa_int();
 583   const TypeInt*     step_val      = gvn().type(argument(5))->isa_int();

 678   const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
 679   const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
 680   const TypeInt*     vlen       = gvn().type(argument(3))->isa_int();
 681   Node*              mask       = argument(4);
 682 
 683   if (mask_klass == NULL || elem_klass == NULL || mask->is_top() || vlen == NULL) {
 684     return false; // dead code
 685   }
 686 
 687   if (!is_klass_initialized(mask_klass)) {
 688     if (C->print_intrinsics()) {
 689       tty->print_cr("  ** klass argument not initialized");
 690     }
 691     return false;
 692   }
 693 
 694   int num_elem = vlen->get_con();
 695   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 696   BasicType elem_bt = elem_type->basic_type();
 697 
 698   if (!arch_supports_vector(Op_LoadVector, num_elem, T_BOOLEAN, VecMaskNotUsed)) {
 699     if (C->print_intrinsics()) {
 700       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 701                     Op_LoadVector, num_elem, type2name(T_BOOLEAN));
 702     }
 703     return false; // not supported
 704   }
 705 
 706   int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
 707   if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskNotUsed)) {
 708     if (C->print_intrinsics()) {
 709       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 710                     mopc, num_elem, type2name(elem_bt));
 711     }
 712     return false; // not supported
 713   }
 714 
 715   const Type* elem_ty = Type::get_const_basic_type(elem_bt);
 716   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
 717   const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
 718   Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
 719   if (mask_vec == NULL) {
 720     if (C->print_intrinsics()) {
 721         tty->print_cr("  ** unbox failed mask=%s",
 722                       NodeClassNames[argument(4)->Opcode()]);
 723     }
 724     return false;
 725   }
 726 
 727   if (mask_vec->bottom_type()->isa_vectmask() == NULL) {

 920   if (elem_bt == arr_elem_bt) {
 921     return true;
 922   } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
 923     // Load/store of short vector from/to char[] is supported
 924     return true;
 925   } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
 926     // Load/store of byte vector from/to boolean[] is supported
 927     return true;
 928   } else {
 929     return false;
 930   }
 931 }
 932 
 933 // public static
 934 // <C,
 935 //  VM,
 936 //  E,
 937 //  S extends VectorSpecies<E>>
 938 // VM load(Class<? extends VM> vmClass, Class<E> elementType, int length,
 939 //         Object base, long offset,    // Unsafe addressing
 940 //         C container, int index, S s,     // Arguments for default implementation
 941 //         LoadOperation<C, VM, E, S> defaultImpl)
 942 //
 943 // public static
 944 // <C,
 945 //  V extends Vector<?>>
 946 // void store(Class<?> vectorClass, Class<?> elementType, int length,
 947 //            Object base, long offset,    // Unsafe addressing
 948 //            V v,
 949 //            C container, int index,      // Arguments for default implementation
 950 //            StoreVectorOperation<C, V> defaultImpl)
 951 
 952 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
 953   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 954   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 955   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 956 
 957   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
 958       vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 959     if (C->print_intrinsics()) {
 960       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
 961                     NodeClassNames[argument(0)->Opcode()],
 962                     NodeClassNames[argument(1)->Opcode()],
 963                     NodeClassNames[argument(2)->Opcode()]);
 964     }
 965     return false; // not enough info for intrinsification
 966   }
 967   if (!is_klass_initialized(vector_klass)) {
 968     if (C->print_intrinsics()) {
 969       tty->print_cr("  ** klass argument not initialized");

1032     set_map(old_map);
1033     set_sp(old_sp);
1034     return false;
1035   }
1036   // Since we are using byte array, we need to double check that the byte operations are supported by backend.
1037   if (using_byte_array) {
1038     int byte_num_elem = num_elem * type2aelembytes(elem_bt);
1039     if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, byte_num_elem, T_BYTE, VecMaskNotUsed)
1040         || !arch_supports_vector(Op_VectorReinterpret, byte_num_elem, T_BYTE, VecMaskNotUsed)) {
1041       if (C->print_intrinsics()) {
1042         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
1043                       is_store, is_store ? "store" : "load",
1044                       byte_num_elem, type2name(elem_bt));
1045       }
1046       set_map(old_map);
1047       set_sp(old_sp);
1048       return false; // not supported
1049     }
1050   }
1051   if (is_mask) {
1052     if (!arch_supports_vector(Op_LoadVector, num_elem, T_BOOLEAN, VecMaskNotUsed)) {
1053       if (C->print_intrinsics()) {
1054         tty->print_cr("  ** not supported: arity=%d op=%s/mask vlen=%d etype=bit ismask=no",
1055                       is_store, is_store ? "store" : "load",
1056                       num_elem);
1057       }
1058       set_map(old_map);
1059       set_sp(old_sp);
1060       return false; // not supported
1061     }
1062     if (!is_store) {
1063       if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1064         set_map(old_map);
1065         set_sp(old_sp);
1066         return false; // not supported
1067       }
1068     } else {
1069       if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
1070         set_map(old_map);
1071         set_sp(old_sp);
1072         return false; // not supported
1073       }
1074     }
1075   }
1076 
1077   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1078 
1079   if (needs_cpu_membar) {
1080     insert_mem_bar(Op_MemBarCPUOrder);
1081   }
1082 
1083   if (is_store) {
1084     Node* val = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1085     if (val == NULL) {
1086       set_map(old_map);
1087       set_sp(old_sp);
1088       return false; // operand unboxing failed
1089     }
1090     set_all_memory(reset_memory());
1091 
1092     // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
1093     int store_num_elem = num_elem;
1094     if (using_byte_array) {
1095       store_num_elem = num_elem * type2aelembytes(elem_bt);
1096       const TypeVect* to_vect_type = TypeVect::make(T_BYTE, store_num_elem);
1097       val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1098     }
1099 


1100     Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
1101     set_memory(vstore, addr_type);
1102   } else {
1103     // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
1104     Node* vload = NULL;
1105     if (using_byte_array) {
1106       int load_num_elem = num_elem * type2aelembytes(elem_bt);
1107       vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, load_num_elem, T_BYTE));
1108       const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1109       vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1110     } else {
1111       // Special handle for masks
1112       if (is_mask) {
1113         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
1114         vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
1115       } else {
1116         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
1117       }
1118     }
1119     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);

1121   }
1122 
1123   old_map->destruct(&_gvn);
1124 
1125   if (needs_cpu_membar) {
1126     insert_mem_bar(Op_MemBarCPUOrder);
1127   }
1128 
1129   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1130   return true;
1131 }
1132 
1133 // public static
1134 // <C,
1135 //  V extends Vector<?>,
1136 //  E,
1137 //  S extends VectorSpecies<E>,
1138 //  M extends VectorMask<E>>
1139 // V loadMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1140 //              int length, Object base, long offset, M m,
1141 //              C container, int index, S s,  // Arguments for default implementation
1142 //              LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1143 //
1144 // public static
1145 // <C,
1146 //  V extends Vector<E>,
1147 //  M extends VectorMask<E>,
1148 //  E>
1149 // void storeMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1150 //                  int length, Object base, long offset,
1151 //                  V v, M m,
1152 //                  C container, int index,  // Arguments for default implementation
1153 //                  StoreVectorMaskedOperation<C, V, M, E> defaultImpl) {
1154 //
1155 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1156   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1157   const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1158   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1159   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1160 
1161   if (vector_klass == NULL || mask_klass == NULL || elem_klass == NULL || vlen == NULL ||
1162       vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
1163       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1164     if (C->print_intrinsics()) {
1165       tty->print_cr("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1166                     NodeClassNames[argument(0)->Opcode()],
1167                     NodeClassNames[argument(1)->Opcode()],
1168                     NodeClassNames[argument(2)->Opcode()],
1169                     NodeClassNames[argument(3)->Opcode()]);
1170     }
1171     return false; // not enough info for intrinsification
1172   }

2719     case T_FLOAT: {
2720       bits = gvn().transform(new MoveF2INode(operation));
2721       bits = gvn().transform(new ConvI2LNode(bits));
2722       break;
2723     }
2724     case T_DOUBLE: {
2725       bits = gvn().transform(new MoveD2LNode(operation));
2726       break;
2727     }
2728     case T_LONG: {
2729       bits = operation; // no conversion needed
2730       break;
2731     }
2732     default: fatal("%s", type2name(elem_bt));
2733   }
2734 
2735   set_result(bits);
2736   return true;
2737 }
2738 































































































  65 
  66 static bool is_vector_shuffle(ciKlass* klass) {
  67   return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
  68 }
  69 
  70 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
  71                                                  VectorMaskUseType mask_use_type, bool has_scalar_args) {
  72   bool is_supported = true;
  73 
  74   // has_scalar_args flag is true only for non-constant scalar shift count,
  75   // since in this case shift needs to be broadcasted.
  76   if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
  77        (has_scalar_args &&
  78          !arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed))) {
  79     is_supported = false;
  80   }
  81 
  82   if (is_supported) {
  83     // Check whether mask unboxing is supported.
  84     if ((mask_use_type & VecMaskUseLoad) != 0) {
  85       if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt) ||
  86           !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
  87       #ifndef PRODUCT
  88         if (C->print_intrinsics()) {
  89           tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
  90                         NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
  91         }
  92       #endif
  93         return false;
  94       }
  95     }
  96 
  97     if ((mask_use_type & VecMaskUsePred) != 0) {
  98       if (!Matcher::has_predicated_vectors() ||
  99           !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
 100       #ifndef PRODUCT
 101         if (C->print_intrinsics()) {
 102           tty->print_cr("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
 103                         NodeClassNames[opc], type2name(elem_bt), num_elem);
 104         }
 105       #endif
 106         return false;

 246           tty->print_cr("  ** Not a svml call or load/store vector op (%s,%s,%d)",
 247                         NodeClassNames[sopc], type2name(type), num_elem);
 248         }
 249 #endif
 250         return false;
 251       }
 252     }
 253   }
 254 
 255   if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
 256       Matcher::supports_vector_variable_shifts() == false) {
 257     if (C->print_intrinsics()) {
 258       tty->print_cr("  ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
 259                     NodeClassNames[sopc], type2name(type), num_elem);
 260     }
 261     return false;
 262   }
 263 
 264   // Check whether mask unboxing is supported.
 265   if ((mask_use_type & VecMaskUseLoad) != 0) {
 266     if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type) ||
 267         !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
 268     #ifndef PRODUCT
 269       if (C->print_intrinsics()) {
 270         tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
 271                       NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
 272       }
 273     #endif
 274       return false;
 275     }
 276   }
 277 
 278   // Check whether mask boxing is supported.
 279   if ((mask_use_type & VecMaskUseStore) != 0) {
 280     if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type) ||
 281         !Matcher::match_rule_supported_vector(Op_StoreVector, num_elem, T_BOOLEAN)) {
 282     #ifndef PRODUCT
 283       if (C->print_intrinsics()) {
 284         tty->print_cr("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
 285                       NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
 286       }
 287     #endif
 288       return false;
 289     }
 290   }
 291 
 292   if ((mask_use_type & VecMaskUsePred) != 0) {
 293     bool is_supported = false;
 294     if (Matcher::has_predicated_vectors()) {
 295       if (VectorNode::is_vector_integral_negate(sopc)) {
 296         is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
 297       } else {
 298         is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
 299       }
 300     }
 301 

 547     switch (n) {
 548       case 1:
 549       case 2: {
 550         operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc));
 551         break;
 552       }
 553       case 3: {
 554         operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
 555         break;
 556       }
 557       default: fatal("unsupported arity: %d", n);
 558     }
 559   }
 560 
 561   if (is_masked_op && mask != NULL) {
 562     if (use_predicate) {
 563       operation->add_req(mask);
 564       operation->add_flag(Node::Flag_is_predicated_vector);
 565     } else {
 566       operation = gvn().transform(operation);
 567       operation->add_flag(Node::Flag_is_predicated_using_blend);
 568       operation = new VectorBlendNode(opd1, operation, mask);
 569     }
 570   }
 571   operation = gvn().transform(operation);
 572 
 573   // Wrap it up in VectorBox to keep object type information.
 574   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
 575   set_result(vbox);
 576   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
 577   return true;
 578 }
 579 
 580 // <Sh extends VectorShuffle<E>,  E>
 581 //  Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
 582 //                  int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
 583 bool LibraryCallKit::inline_vector_shuffle_iota() {
 584   const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
 585   const TypeInt*     vlen          = gvn().type(argument(3))->isa_int();
 586   const TypeInt*     start_val     = gvn().type(argument(4))->isa_int();
 587   const TypeInt*     step_val      = gvn().type(argument(5))->isa_int();

 682   const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
 683   const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
 684   const TypeInt*     vlen       = gvn().type(argument(3))->isa_int();
 685   Node*              mask       = argument(4);
 686 
 687   if (mask_klass == NULL || elem_klass == NULL || mask->is_top() || vlen == NULL) {
 688     return false; // dead code
 689   }
 690 
 691   if (!is_klass_initialized(mask_klass)) {
 692     if (C->print_intrinsics()) {
 693       tty->print_cr("  ** klass argument not initialized");
 694     }
 695     return false;
 696   }
 697 
 698   int num_elem = vlen->get_con();
 699   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
 700   BasicType elem_bt = elem_type->basic_type();
 701 








 702   int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
 703   if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
 704     if (C->print_intrinsics()) {
 705       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
 706                     mopc, num_elem, type2name(elem_bt));
 707     }
 708     return false; // not supported
 709   }
 710 
 711   const Type* elem_ty = Type::get_const_basic_type(elem_bt);
 712   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
 713   const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
 714   Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
 715   if (mask_vec == NULL) {
 716     if (C->print_intrinsics()) {
 717         tty->print_cr("  ** unbox failed mask=%s",
 718                       NodeClassNames[argument(4)->Opcode()]);
 719     }
 720     return false;
 721   }
 722 
 723   if (mask_vec->bottom_type()->isa_vectmask() == NULL) {

 916   if (elem_bt == arr_elem_bt) {
 917     return true;
 918   } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
 919     // Load/store of short vector from/to char[] is supported
 920     return true;
 921   } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
 922     // Load/store of byte vector from/to boolean[] is supported
 923     return true;
 924   } else {
 925     return false;
 926   }
 927 }
 928 
 929 // public static
 930 // <C,
 931 //  VM,
 932 //  E,
 933 //  S extends VectorSpecies<E>>
 934 // VM load(Class<? extends VM> vmClass, Class<E> elementType, int length,
 935 //         Object base, long offset,    // Unsafe addressing
 936 //         C container, long index, S s,     // Arguments for default implementation
 937 //         LoadOperation<C, VM, E, S> defaultImpl)
 938 //
 939 // public static
 940 // <C,
 941 //  V extends Vector<?>>
 942 // void store(Class<?> vectorClass, Class<?> elementType, int length,
 943 //            Object base, long offset,    // Unsafe addressing
 944 //            V v,
 945 //            C container, long index,      // Arguments for default implementation
 946 //            StoreVectorOperation<C, V> defaultImpl)
 947 
 948 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
 949   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
 950   const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
 951   const TypeInt*     vlen         = gvn().type(argument(2))->isa_int();
 952 
 953   if (vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
 954       vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
 955     if (C->print_intrinsics()) {
 956       tty->print_cr("  ** missing constant: vclass=%s etype=%s vlen=%s",
 957                     NodeClassNames[argument(0)->Opcode()],
 958                     NodeClassNames[argument(1)->Opcode()],
 959                     NodeClassNames[argument(2)->Opcode()]);
 960     }
 961     return false; // not enough info for intrinsification
 962   }
 963   if (!is_klass_initialized(vector_klass)) {
 964     if (C->print_intrinsics()) {
 965       tty->print_cr("  ** klass argument not initialized");

1028     set_map(old_map);
1029     set_sp(old_sp);
1030     return false;
1031   }
1032   // Since we are using byte array, we need to double check that the byte operations are supported by backend.
1033   if (using_byte_array) {
1034     int byte_num_elem = num_elem * type2aelembytes(elem_bt);
1035     if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, byte_num_elem, T_BYTE, VecMaskNotUsed)
1036         || !arch_supports_vector(Op_VectorReinterpret, byte_num_elem, T_BYTE, VecMaskNotUsed)) {
1037       if (C->print_intrinsics()) {
1038         tty->print_cr("  ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
1039                       is_store, is_store ? "store" : "load",
1040                       byte_num_elem, type2name(elem_bt));
1041       }
1042       set_map(old_map);
1043       set_sp(old_sp);
1044       return false; // not supported
1045     }
1046   }
1047   if (is_mask) {










1048     if (!is_store) {
1049       if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1050         set_map(old_map);
1051         set_sp(old_sp);
1052         return false; // not supported
1053       }
1054     } else {
1055       if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
1056         set_map(old_map);
1057         set_sp(old_sp);
1058         return false; // not supported
1059       }
1060     }
1061   }
1062 
1063   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1064 
1065   if (needs_cpu_membar) {
1066     insert_mem_bar(Op_MemBarCPUOrder);
1067   }
1068 
1069   if (is_store) {
1070     Node* val = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1071     if (val == NULL) {
1072       set_map(old_map);
1073       set_sp(old_sp);
1074       return false; // operand unboxing failed
1075     }
1076     set_all_memory(reset_memory());
1077 
1078     // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
1079     int store_num_elem = num_elem;
1080     if (using_byte_array) {
1081       store_num_elem = num_elem * type2aelembytes(elem_bt);
1082       const TypeVect* to_vect_type = TypeVect::make(T_BYTE, store_num_elem);
1083       val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1084     }
1085     if (is_mask) {
1086       val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
1087     }
1088     Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
1089     set_memory(vstore, addr_type);
1090   } else {
1091     // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
1092     Node* vload = NULL;
1093     if (using_byte_array) {
1094       int load_num_elem = num_elem * type2aelembytes(elem_bt);
1095       vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, load_num_elem, T_BYTE));
1096       const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1097       vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1098     } else {
1099       // Special handle for masks
1100       if (is_mask) {
1101         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
1102         vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
1103       } else {
1104         vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
1105       }
1106     }
1107     Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);

1109   }
1110 
1111   old_map->destruct(&_gvn);
1112 
1113   if (needs_cpu_membar) {
1114     insert_mem_bar(Op_MemBarCPUOrder);
1115   }
1116 
1117   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1118   return true;
1119 }
1120 
1121 // public static
1122 // <C,
1123 //  V extends Vector<?>,
1124 //  E,
1125 //  S extends VectorSpecies<E>,
1126 //  M extends VectorMask<E>>
1127 // V loadMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1128 //              int length, Object base, long offset, M m,
1129 //              C container, long index, S s,  // Arguments for default implementation
1130 //              LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1131 //
1132 // public static
1133 // <C,
1134 //  V extends Vector<E>,
1135 //  M extends VectorMask<E>,
1136 //  E>
1137 // void storeMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1138 //                  int length, Object base, long offset,
1139 //                  V v, M m,
1140 //                  C container, long index,  // Arguments for default implementation
1141 //                  StoreVectorMaskedOperation<C, V, M, E> defaultImpl) {
1142 //
1143 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1144   const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1145   const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
1146   const TypeInstPtr* elem_klass   = gvn().type(argument(2))->isa_instptr();
1147   const TypeInt*     vlen         = gvn().type(argument(3))->isa_int();
1148 
1149   if (vector_klass == NULL || mask_klass == NULL || elem_klass == NULL || vlen == NULL ||
1150       vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
1151       elem_klass->const_oop() == NULL || !vlen->is_con()) {
1152     if (C->print_intrinsics()) {
1153       tty->print_cr("  ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1154                     NodeClassNames[argument(0)->Opcode()],
1155                     NodeClassNames[argument(1)->Opcode()],
1156                     NodeClassNames[argument(2)->Opcode()],
1157                     NodeClassNames[argument(3)->Opcode()]);
1158     }
1159     return false; // not enough info for intrinsification
1160   }

2707     case T_FLOAT: {
2708       bits = gvn().transform(new MoveF2INode(operation));
2709       bits = gvn().transform(new ConvI2LNode(bits));
2710       break;
2711     }
2712     case T_DOUBLE: {
2713       bits = gvn().transform(new MoveD2LNode(operation));
2714       break;
2715     }
2716     case T_LONG: {
2717       bits = operation; // no conversion needed
2718       break;
2719     }
2720     default: fatal("%s", type2name(elem_bt));
2721   }
2722 
2723   set_result(bits);
2724   return true;
2725 }
2726 
2727 // public static
2728 // <V extends Vector<E>,
2729 //  M extends VectorMask<E>,
2730 //  E>
2731 //  V comExpOp(int opr,
2732 //             Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2733 //             int length, V v, M m,
2734 //             CmpExpOperation<V, M> defaultImpl)
2735 bool LibraryCallKit::inline_vector_compress_expand() {
2736   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
2737   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2738   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
2739   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
2740   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
2741 
2742   if (vector_klass == NULL || elem_klass == NULL || mask_klass == NULL || vlen == NULL ||
2743       vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
2744       elem_klass->const_oop() == NULL || !vlen->is_con()) {
2745     if (C->print_intrinsics()) {
2746       tty->print_cr("  ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2747                     NodeClassNames[argument(0)->Opcode()],
2748                     NodeClassNames[argument(1)->Opcode()],
2749                     NodeClassNames[argument(2)->Opcode()],
2750                     NodeClassNames[argument(3)->Opcode()],
2751                     NodeClassNames[argument(4)->Opcode()]);
2752     }
2753     return false; // not enough info for intrinsification
2754   }
2755 
2756   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2757     if (C->print_intrinsics()) {
2758       tty->print_cr("  ** klass argument not initialized");
2759     }
2760     return false;
2761   }
2762 
2763   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2764   if (!elem_type->is_primitive_type()) {
2765     if (C->print_intrinsics()) {
2766       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
2767     }
2768     return false; // should be primitive type
2769   }
2770 
2771   int num_elem = vlen->get_con();
2772   BasicType elem_bt = elem_type->basic_type();
2773   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2774 
2775   if (!arch_supports_vector(opc, num_elem, elem_bt, VecMaskUseLoad)) {
2776     if (C->print_intrinsics()) {
2777       tty->print_cr("  ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2778                     opc, num_elem, type2name(elem_bt));
2779     }
2780     return false; // not supported
2781   }
2782 
2783   Node* opd1 = NULL;
2784   const TypeInstPtr* vbox_type = NULL;
2785   if (opc != Op_CompressM) {
2786     ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2787     vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2788     opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2789     if (opd1 == NULL) {
2790       if (C->print_intrinsics()) {
2791         tty->print_cr("  ** unbox failed vector=%s",
2792                       NodeClassNames[argument(5)->Opcode()]);
2793       }
2794       return false;
2795     }
2796   }
2797 
2798   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2799   assert(is_vector_mask(mbox_klass), "argument(6) should be a mask class");
2800   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2801 
2802   Node* mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2803   if (mask == NULL) {
2804     if (C->print_intrinsics()) {
2805       tty->print_cr("  ** unbox failed mask=%s",
2806                     NodeClassNames[argument(6)->Opcode()]);
2807     }
2808     return false;
2809   }
2810 
2811   const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
2812   Node* operation = gvn().transform(VectorNode::make(opc, opd1, mask, vt));
2813 
2814   // Wrap it up in VectorBox to keep object type information.
2815   const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2816   Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2817   set_result(vbox);
2818   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2819   return true;
2820 }
< prev index next >