< prev index next >

src/hotspot/share/opto/vectorIntrinsics.cpp

Print this page
*** 80,11 ***
    }
  
    if (is_supported) {
      // Check whether mask unboxing is supported.
      if ((mask_use_type & VecMaskUseLoad) != 0) {
!       if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt)) {
        #ifndef PRODUCT
          if (C->print_intrinsics()) {
            tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
                          NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
          }
--- 80,12 ---
    }
  
    if (is_supported) {
      // Check whether mask unboxing is supported.
      if ((mask_use_type & VecMaskUseLoad) != 0) {
!       if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt) ||
+           !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
        #ifndef PRODUCT
          if (C->print_intrinsics()) {
            tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
                          NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
          }

*** 260,11 ***
      return false;
    }
  
    // Check whether mask unboxing is supported.
    if ((mask_use_type & VecMaskUseLoad) != 0) {
!     if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type)) {
      #ifndef PRODUCT
        if (C->print_intrinsics()) {
          tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
                        NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
        }
--- 261,12 ---
      return false;
    }
  
    // Check whether mask unboxing is supported.
    if ((mask_use_type & VecMaskUseLoad) != 0) {
!     if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type) ||
+         !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
      #ifndef PRODUCT
        if (C->print_intrinsics()) {
          tty->print_cr("  ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
                        NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
        }

*** 273,11 ***
      }
    }
  
    // Check whether mask boxing is supported.
    if ((mask_use_type & VecMaskUseStore) != 0) {
!     if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type)) {
      #ifndef PRODUCT
        if (C->print_intrinsics()) {
          tty->print_cr("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
                        NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
        }
--- 275,12 ---
      }
    }
  
    // Check whether mask boxing is supported.
    if ((mask_use_type & VecMaskUseStore) != 0) {
!     if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type) ||
+         !Matcher::match_rule_supported_vector(Op_StoreVector, num_elem, T_BOOLEAN)) {
      #ifndef PRODUCT
        if (C->print_intrinsics()) {
          tty->print_cr("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
                        NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
        }

*** 559,10 ***
--- 562,11 ---
      if (use_predicate) {
        operation->add_req(mask);
        operation->add_flag(Node::Flag_is_predicated_vector);
      } else {
        operation = gvn().transform(operation);
+       operation->add_flag(Node::Flag_is_predicated_using_blend);
        operation = new VectorBlendNode(opd1, operation, mask);
      }
    }
    operation = gvn().transform(operation);
  

*** 693,20 ***
  
    int num_elem = vlen->get_con();
    ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
    BasicType elem_bt = elem_type->basic_type();
  
-   if (!arch_supports_vector(Op_LoadVector, num_elem, T_BOOLEAN, VecMaskNotUsed)) {
-     if (C->print_intrinsics()) {
-       tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
-                     Op_LoadVector, num_elem, type2name(T_BOOLEAN));
-     }
-     return false; // not supported
-   }
- 
    int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
!   if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskNotUsed)) {
      if (C->print_intrinsics()) {
        tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
                      mopc, num_elem, type2name(elem_bt));
      }
      return false; // not supported
--- 697,12 ---
  
    int num_elem = vlen->get_con();
    ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
    BasicType elem_bt = elem_type->basic_type();
  
    int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
!   if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
      if (C->print_intrinsics()) {
        tty->print_cr("  ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
                      mopc, num_elem, type2name(elem_bt));
      }
      return false; // not supported

*** 935,20 ***
  //  VM,
  //  E,
  //  S extends VectorSpecies<E>>
  // VM load(Class<? extends VM> vmClass, Class<E> elementType, int length,
  //         Object base, long offset,    // Unsafe addressing
! //         C container, int index, S s,     // Arguments for default implementation
  //         LoadOperation<C, VM, E, S> defaultImpl)
  //
  // public static
  // <C,
  //  V extends Vector<?>>
  // void store(Class<?> vectorClass, Class<?> elementType, int length,
  //            Object base, long offset,    // Unsafe addressing
  //            V v,
! //            C container, int index,      // Arguments for default implementation
  //            StoreVectorOperation<C, V> defaultImpl)
  
  bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
    const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
    const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();
--- 931,20 ---
  //  VM,
  //  E,
  //  S extends VectorSpecies<E>>
  // VM load(Class<? extends VM> vmClass, Class<E> elementType, int length,
  //         Object base, long offset,    // Unsafe addressing
! //         C container, long index, S s,     // Arguments for default implementation
  //         LoadOperation<C, VM, E, S> defaultImpl)
  //
  // public static
  // <C,
  //  V extends Vector<?>>
  // void store(Class<?> vectorClass, Class<?> elementType, int length,
  //            Object base, long offset,    // Unsafe addressing
  //            V v,
! //            C container, long index,      // Arguments for default implementation
  //            StoreVectorOperation<C, V> defaultImpl)
  
  bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
    const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
    const TypeInstPtr* elem_klass   = gvn().type(argument(1))->isa_instptr();

*** 1047,20 ***
        set_sp(old_sp);
        return false; // not supported
      }
    }
    if (is_mask) {
-     if (!arch_supports_vector(Op_LoadVector, num_elem, T_BOOLEAN, VecMaskNotUsed)) {
-       if (C->print_intrinsics()) {
-         tty->print_cr("  ** not supported: arity=%d op=%s/mask vlen=%d etype=bit ismask=no",
-                       is_store, is_store ? "store" : "load",
-                       num_elem);
-       }
-       set_map(old_map);
-       set_sp(old_sp);
-       return false; // not supported
-     }
      if (!is_store) {
        if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
          set_map(old_map);
          set_sp(old_sp);
          return false; // not supported
--- 1043,10 ---

*** 1094,11 ***
      if (using_byte_array) {
        store_num_elem = num_elem * type2aelembytes(elem_bt);
        const TypeVect* to_vect_type = TypeVect::make(T_BYTE, store_num_elem);
        val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
      }
! 
      Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
      set_memory(vstore, addr_type);
    } else {
      // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
      Node* vload = NULL;
--- 1080,13 ---
      if (using_byte_array) {
        store_num_elem = num_elem * type2aelembytes(elem_bt);
        const TypeVect* to_vect_type = TypeVect::make(T_BYTE, store_num_elem);
        val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
      }
!     if (is_mask) {
+       val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
+     }
      Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
      set_memory(vstore, addr_type);
    } else {
      // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
      Node* vload = NULL;

*** 1136,22 ***
  //  E,
  //  S extends VectorSpecies<E>,
  //  M extends VectorMask<E>>
  // V loadMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
  //              int length, Object base, long offset, M m,
! //              C container, int index, S s,  // Arguments for default implementation
  //              LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
  //
  // public static
  // <C,
  //  V extends Vector<E>,
  //  M extends VectorMask<E>,
  //  E>
  // void storeMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
  //                  int length, Object base, long offset,
  //                  V v, M m,
! //                  C container, int index,  // Arguments for default implementation
  //                  StoreVectorMaskedOperation<C, V, M, E> defaultImpl) {
  //
  bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
    const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
    const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();
--- 1124,22 ---
  //  E,
  //  S extends VectorSpecies<E>,
  //  M extends VectorMask<E>>
  // V loadMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
  //              int length, Object base, long offset, M m,
! //              C container, long index, S s,  // Arguments for default implementation
  //              LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
  //
  // public static
  // <C,
  //  V extends Vector<E>,
  //  M extends VectorMask<E>,
  //  E>
  // void storeMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
  //                  int length, Object base, long offset,
  //                  V v, M m,
! //                  C container, long index,  // Arguments for default implementation
  //                  StoreVectorMaskedOperation<C, V, M, E> defaultImpl) {
  //
  bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
    const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
    const TypeInstPtr* mask_klass   = gvn().type(argument(1))->isa_instptr();

*** 2734,5 ***
--- 2722,99 ---
  
    set_result(bits);
    return true;
  }
  
+ // public static
+ // <V extends Vector<E>,
+ //  M extends VectorMask<E>,
+ //  E>
+ //  V comExpOp(int opr,
+ //             Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
+ //             int length, V v, M m,
+ //             CmpExpOperation<V, M> defaultImpl)
+ bool LibraryCallKit::inline_vector_compress_expand() {
+   const TypeInt*     opr          = gvn().type(argument(0))->isa_int();
+   const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
+   const TypeInstPtr* mask_klass   = gvn().type(argument(2))->isa_instptr();
+   const TypeInstPtr* elem_klass   = gvn().type(argument(3))->isa_instptr();
+   const TypeInt*     vlen         = gvn().type(argument(4))->isa_int();
+ 
+   if (vector_klass == NULL || elem_klass == NULL || mask_klass == NULL || vlen == NULL ||
+       vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
+       elem_klass->const_oop() == NULL || !vlen->is_con()) {
+     if (C->print_intrinsics()) {
+       tty->print_cr("  ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
+                     NodeClassNames[argument(0)->Opcode()],
+                     NodeClassNames[argument(1)->Opcode()],
+                     NodeClassNames[argument(2)->Opcode()],
+                     NodeClassNames[argument(3)->Opcode()],
+                     NodeClassNames[argument(4)->Opcode()]);
+     }
+     return false; // not enough info for intrinsification
+   }
+ 
+   if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
+     if (C->print_intrinsics()) {
+       tty->print_cr("  ** klass argument not initialized");
+     }
+     return false;
+   }
+ 
+   ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
+   if (!elem_type->is_primitive_type()) {
+     if (C->print_intrinsics()) {
+       tty->print_cr("  ** not a primitive bt=%d", elem_type->basic_type());
+     }
+     return false; // should be primitive type
+   }
+ 
+   int num_elem = vlen->get_con();
+   BasicType elem_bt = elem_type->basic_type();
+   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
+ 
+   if (!arch_supports_vector(opc, num_elem, elem_bt, VecMaskUseLoad)) {
+     if (C->print_intrinsics()) {
+       tty->print_cr("  ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
+                     opc, num_elem, type2name(elem_bt));
+     }
+     return false; // not supported
+   }
+ 
+   Node* opd1 = NULL;
+   const TypeInstPtr* vbox_type = NULL;
+   if (opc != Op_CompressM) {
+     ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
+     vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
+     opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
+     if (opd1 == NULL) {
+       if (C->print_intrinsics()) {
+         tty->print_cr("  ** unbox failed vector=%s",
+                       NodeClassNames[argument(5)->Opcode()]);
+       }
+       return false;
+     }
+   }
+ 
+   ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
+   assert(is_vector_mask(mbox_klass), "argument(6) should be a mask class");
+   const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
+ 
+   Node* mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
+   if (mask == NULL) {
+     if (C->print_intrinsics()) {
+       tty->print_cr("  ** unbox failed mask=%s",
+                     NodeClassNames[argument(6)->Opcode()]);
+     }
+     return false;
+   }
+ 
+   const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
+   Node* operation = gvn().transform(VectorNode::make(opc, opd1, mask, vt));
+ 
+   // Wrap it up in VectorBox to keep object type information.
+   const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
+   Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
+   set_result(vbox);
+   C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
+   return true;
+ }
< prev index next >