< prev index next >

src/hotspot/share/opto/memnode.cpp

Print this page
*** 21,11 ***
--- 21,13 ---
   * questions.
   *
   */
  
  #include "precompiled.hpp"
+ #include "ci/ciFlatArrayKlass.hpp"
  #include "classfile/javaClasses.hpp"
+ #include "classfile/systemDictionary.hpp"
  #include "compiler/compileLog.hpp"
  #include "gc/shared/barrierSet.hpp"
  #include "gc/shared/c2/barrierSetC2.hpp"
  #include "gc/shared/tlab_globals.hpp"
  #include "memory/allocation.inline.hpp"

*** 36,10 ***
--- 38,11 ---
  #include "opto/cfgnode.hpp"
  #include "opto/regalloc.hpp"
  #include "opto/compile.hpp"
  #include "opto/connode.hpp"
  #include "opto/convertnode.hpp"
+ #include "opto/inlinetypenode.hpp"
  #include "opto/loopnode.hpp"
  #include "opto/machnode.hpp"
  #include "opto/matcher.hpp"
  #include "opto/memnode.hpp"
  #include "opto/mulnode.hpp"

*** 238,11 ***
      assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
      bool consistent =  adr_check == NULL || adr_check->empty() ||
                         phase->C->must_alias(adr_check, alias_idx );
      // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
      if( !consistent && adr_check != NULL && !adr_check->empty() &&
!                tp->isa_aryptr() &&        tp->offset() == Type::OffsetBot &&
          adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
          ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
            adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
            adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
        // don't assert if it is dead code.
--- 241,11 ---
      assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
      bool consistent =  adr_check == NULL || adr_check->empty() ||
                         phase->C->must_alias(adr_check, alias_idx );
      // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
      if( !consistent && adr_check != NULL && !adr_check->empty() &&
!         tp->isa_aryptr() &&        tp->offset() == Type::OffsetBot &&
          adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
          ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
            adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
            adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
        // don't assert if it is dead code.

*** 886,10 ***
--- 889,11 ---
    case T_SHORT:   load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
    case T_LONG:    load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency); break;
    case T_FLOAT:   load = new LoadFNode (ctl, mem, adr, adr_type, rt,            mo, control_dependency); break;
    case T_DOUBLE:  load = new LoadDNode (ctl, mem, adr, adr_type, rt,            mo, control_dependency); break;
    case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(),  mo, control_dependency); break;
+   case T_INLINE_TYPE:
    case T_OBJECT:
  #ifdef _LP64
      if (adr->bottom_type()->is_ptr_to_narrowoop()) {
        load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
      } else

*** 1013,13 ***
        assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
        addp->set_req(AddPNode::Base, src);
        addp->set_req(AddPNode::Address, src);
  
        const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
!       BasicType ary_elem  = ary_t->klass()->as_array_klass()->element_type()->basic_type();
        uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
        uint shift  = exact_log2(type2aelembytes(ary_elem));
  
        Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
  #ifdef _LP64
        diff = phase->transform(new ConvI2LNode(diff));
  #endif
--- 1017,17 ---
        assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
        addp->set_req(AddPNode::Base, src);
        addp->set_req(AddPNode::Address, src);
  
        const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
!       BasicType ary_elem = ary_t->klass()->as_array_klass()->element_type()->basic_type();
        uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
        uint shift  = exact_log2(type2aelembytes(ary_elem));
+       if (ary_t->klass()->is_flat_array_klass()) {
+         ciFlatArrayKlass* vak = ary_t->klass()->as_flat_array_klass();
+         shift = vak->log2_element_size();
+       }
  
        Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
  #ifdef _LP64
        diff = phase->transform(new ConvI2LNode(diff));
  #endif

*** 1135,11 ***
          return NULL;
        }
        // LoadVector/StoreVector needs additional check to ensure the types match.
        if (store_Opcode() == Op_StoreVector) {
          const TypeVect*  in_vt = st->as_StoreVector()->vect_type();
!         const TypeVect* out_vt = as_LoadVector()->vect_type();
          if (in_vt != out_vt) {
            return NULL;
          }
        }
        return st->in(MemNode::ValueIn);
--- 1143,11 ---
          return NULL;
        }
        // LoadVector/StoreVector needs additional check to ensure the types match.
        if (store_Opcode() == Op_StoreVector) {
          const TypeVect*  in_vt = st->as_StoreVector()->vect_type();
!         const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
          if (in_vt != out_vt) {
            return NULL;
          }
        }
        return st->in(MemNode::ValueIn);

*** 1153,10 ***
--- 1161,16 ---
          (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
        // return a zero value for the load's basic type
        // (This is one of the few places where a generic PhaseTransform
        // can create new nodes.  Think of it as lazily manifesting
        // virtually pre-existing constants.)
+       assert(memory_type() != T_INLINE_TYPE, "should not be used for inline types");
+       Node* default_value = ld_alloc->in(AllocateNode::DefaultValue);
+       if (default_value != NULL) {
+         return default_value;
+       }
+       assert(ld_alloc->in(AllocateNode::RawDefaultValue) == NULL, "default value may not be null");
        if (memory_type() != T_VOID) {
          if (ReduceBulkZeroing || find_array_copy_clone(phase, ld_alloc, in(MemNode::Memory)) == NULL) {
            // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
            // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
            // by the ArrayCopyNode.

*** 1220,10 ***
--- 1234,38 ---
  }
  
  //------------------------------Identity---------------------------------------
  // Loads are identity if previous store is to same address
  Node* LoadNode::Identity(PhaseGVN* phase) {
+   // Loading from an InlineTypePtr? The InlineTypePtr has the values of
+   // all fields as input. Look for the field with matching offset.
+   Node* addr = in(Address);
+   intptr_t offset;
+   Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
+   InlineTypePtrNode* vt = (base != NULL) ? base->uncast()->isa_InlineTypePtr() : NULL;
+   if (vt != NULL && offset > oopDesc::klass_offset_in_bytes()) {
+     Node* value = vt->field_value_by_offset((int)offset, true);
+     if (value->is_InlineType()) {
+       // Non-flattened inline type field
+       InlineTypeNode* vt = value->as_InlineType();
+       if (vt->is_allocated(phase)) {
+         value = vt->get_oop();
+       } else {
+         // Not yet allocated, bail out
+         value = NULL;
+       }
+     }
+     if (value != NULL) {
+       if (Opcode() == Op_LoadN) {
+         // Encode oop value if we are loading a narrow oop
+         assert(!phase->type(value)->isa_narrowoop(), "should already be decoded");
+         value = phase->transform(new EncodePNode(value, bottom_type()));
+       }
+       return value;
+     }
+   }
+ 
    // If the previous store-maker is the right kind of Store, and the store is
    // to the same address, then we are equal to the value stored.
    Node* mem = in(Memory);
    Node* value = can_see_stored_value(mem, phase);
    if( value ) {

*** 1929,10 ***
--- 1971,11 ---
      // In fact, that could have been the original type of p1, and p1 could have
      // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
      // expression (LShiftL quux 3) independently optimized to the constant 8.
      if ((t->isa_int() == NULL) && (t->isa_long() == NULL)
          && (_type->isa_vect() == NULL)
+         && t->isa_inlinetype() == NULL
          && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
        // t might actually be lower than _type, if _type is a unique
        // concrete subclass of abstract class t.
        if (off_beyond_header || off == Type::OffsetBot) {  // is the offset beyond the header?
          const Type* jt = t->join_speculative(_type);

*** 1963,56 ***
      }
    } else if (tp->base() == Type::InstPtr) {
      assert( off != Type::OffsetBot ||
              // arrays can be cast to Objects
              tp->is_oopptr()->klass()->is_java_lang_Object() ||
              // unsafe field access may not have a constant offset
              C->has_unsafe_access(),
              "Field accesses must be precise" );
      // For oop loads, we expect the _type to be precise.
  
-     // Optimize loads from constant fields.
      const TypeInstPtr* tinst = tp->is_instptr();
      ciObject* const_oop = tinst->const_oop();
      if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != NULL && const_oop->is_instance()) {
!       const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), memory_type());
        if (con_type != NULL) {
          return con_type;
        }
      }
    } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
      assert( off != Type::OffsetBot ||
              // arrays can be cast to Objects
              tp->is_klassptr()->klass()->is_java_lang_Object() ||
              // also allow array-loading from the primary supertype
              // array during subtype checks
              Opcode() == Op_LoadKlass,
              "Field accesses must be precise" );
      // For klass/static loads, we expect the _type to be precise
!   } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
!     /* With mirrors being an indirect in the Klass*
!      * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
!      * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
!      *
!      * So check the type and klass of the node before the LoadP.
!      */
!     Node* adr2 = adr->in(MemNode::Address);
!     const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
!     if (tkls != NULL && !StressReflectiveCode) {
!       ciKlass* klass = tkls->klass();
!       if (klass->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
!         assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
!         assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
!         return TypeInstPtr::make(klass->java_mirror());
        }
      }
    }
  
    const TypeKlassPtr *tkls = tp->isa_klassptr();
    if (tkls != NULL && !StressReflectiveCode) {
      ciKlass* klass = tkls->klass();
!     if (klass->is_loaded() && tkls->klass_is_exact()) {
        // We are loading a field from a Klass metaobject whose identity
        // is known at compile time (the type is "exact" or "precise").
        // Check for fields we know are maintained as constants by the VM.
        if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
          // The field is Klass::_super_check_offset.  Return its (constant) value.
--- 2006,94 ---
      }
    } else if (tp->base() == Type::InstPtr) {
      assert( off != Type::OffsetBot ||
              // arrays can be cast to Objects
              tp->is_oopptr()->klass()->is_java_lang_Object() ||
+             tp->is_oopptr()->klass() == ciEnv::current()->Class_klass() ||
              // unsafe field access may not have a constant offset
              C->has_unsafe_access(),
              "Field accesses must be precise" );
      // For oop loads, we expect the _type to be precise.
  
      const TypeInstPtr* tinst = tp->is_instptr();
+     BasicType bt = memory_type();
+ 
+     // Optimize loads from constant fields.
      ciObject* const_oop = tinst->const_oop();
      if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != NULL && const_oop->is_instance()) {
!       ciType* mirror_type = const_oop->as_instance()->java_mirror_type();
+       if (mirror_type != NULL) {
+         const Type* const_oop = NULL;
+         ciInlineKlass* vk = mirror_type->is_inlinetype() ? mirror_type->as_inline_klass() : NULL;
+         // Fold default value loads
+         if (vk != NULL && off == vk->default_value_offset()) {
+           const_oop = TypeInstPtr::make(vk->default_instance());
+         }
+         // Fold class mirror loads
+         if (off == java_lang_Class::primary_mirror_offset()) {
+           const_oop = (vk == NULL) ? TypePtr::NULL_PTR : TypeInstPtr::make(vk->ref_instance());
+         } else if (off == java_lang_Class::secondary_mirror_offset()) {
+           const_oop = (vk == NULL) ? TypePtr::NULL_PTR : TypeInstPtr::make(vk->val_instance());
+         }
+         if (const_oop != NULL) {
+           return (bt == T_NARROWOOP) ? const_oop->make_narrowoop() : const_oop;
+         }
+       }
+       const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
        if (con_type != NULL) {
          return con_type;
        }
      }
    } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
      assert( off != Type::OffsetBot ||
              // arrays can be cast to Objects
+             tp->is_klassptr()->klass() == NULL ||
              tp->is_klassptr()->klass()->is_java_lang_Object() ||
              // also allow array-loading from the primary supertype
              // array during subtype checks
              Opcode() == Op_LoadKlass,
              "Field accesses must be precise" );
      // For klass/static loads, we expect the _type to be precise
!   } else if (tp->base() == Type::RawPtr && !StressReflectiveCode) {
!     if (adr->is_Load() && off == 0) {
!       /* With mirrors being an indirect in the Klass*
!        * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
!        * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
!        *
!        * So check the type and klass of the node before the LoadP.
!        */
!       Node* adr2 = adr->in(MemNode::Address);
!       const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
!       if (tkls != NULL) {
!         ciKlass* klass = tkls->klass();
!         if (klass != NULL && klass->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
!           assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
!           assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
+           return TypeInstPtr::make(klass->java_mirror());
+         }
+       }
+     } else {
+       // Check for a load of the default value offset from the InlineKlassFixedBlock:
+       // LoadI(LoadP(inline_klass, adr_inlineklass_fixed_block_offset), default_value_offset_offset)
+       intptr_t offset = 0;
+       Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
+       if (base != NULL && base->is_Load() && offset == in_bytes(InlineKlass::default_value_offset_offset())) {
+         const TypeKlassPtr* tkls = phase->type(base->in(MemNode::Address))->isa_klassptr();
+         if (tkls != NULL && tkls->is_loaded() && tkls->klass_is_exact() && tkls->isa_inlinetype() &&
+             tkls->offset() == in_bytes(InstanceKlass::adr_inlineklass_fixed_block_offset())) {
+           assert(base->Opcode() == Op_LoadP, "must load an oop from klass");
+           assert(Opcode() == Op_LoadI, "must load an int from fixed block");
+           return TypeInt::make(tkls->klass()->as_inline_klass()->default_value_offset());
+         }
        }
      }
    }
  
    const TypeKlassPtr *tkls = tp->isa_klassptr();
    if (tkls != NULL && !StressReflectiveCode) {
      ciKlass* klass = tkls->klass();
!     if (tkls->is_loaded() && tkls->klass_is_exact()) {
        // We are loading a field from a Klass metaobject whose identity
        // is known at compile time (the type is "exact" or "precise").
        // Check for fields we know are maintained as constants by the VM.
        if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
          // The field is Klass::_super_check_offset.  Return its (constant) value.

*** 2035,11 ***
      }
  
      // We can still check if we are loading from the primary_supers array at a
      // shallow enough depth.  Even though the klass is not exact, entries less
      // than or equal to its super depth are correct.
!     if (klass->is_loaded() ) {
        ciType *inner = klass;
        while( inner->is_obj_array_klass() )
          inner = inner->as_obj_array_klass()->base_element_type();
        if( inner->is_instance_klass() &&
            !inner->as_instance_klass()->flags().is_interface() ) {
--- 2116,11 ---
      }
  
      // We can still check if we are loading from the primary_supers array at a
      // shallow enough depth.  Even though the klass is not exact, entries less
      // than or equal to its super depth are correct.
!     if (tkls->is_loaded()) {
        ciType *inner = klass;
        while( inner->is_obj_array_klass() )
          inner = inner->as_obj_array_klass()->base_element_type();
        if( inner->is_instance_klass() &&
            !inner->as_instance_klass()->flags().is_interface() ) {

*** 2100,14 ***
      if (mem->is_Parm() && mem->in(0)->is_Start()) {
        assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
        return Type::get_zero_type(_type->basic_type());
      }
    }
- 
    Node* alloc = is_new_object_mark_load(phase);
    if (alloc != NULL) {
!     return TypeX::make(markWord::prototype().value());
    }
  
    return _type;
  }
  
--- 2181,23 ---
      if (mem->is_Parm() && mem->in(0)->is_Start()) {
        assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
        return Type::get_zero_type(_type->basic_type());
      }
    }
    Node* alloc = is_new_object_mark_load(phase);
    if (alloc != NULL) {
!     if (EnableValhalla) {
+       // The mark word may contain property bits (inline, flat, null-free)
+       Node* klass_node = alloc->in(AllocateNode::KlassNode);
+       const TypeKlassPtr* tkls = phase->type(klass_node)->is_klassptr();
+       ciKlass* klass = tkls->klass();
+       if (klass != NULL && klass->is_loaded() && tkls->klass_is_exact()) {
+         return TypeX::make(klass->prototype_header().value());
+       }
+     } else {
+       return TypeX::make(markWord::prototype().value());
+     }
    }
  
    return _type;
  }
  

*** 2254,11 ***
  }
  
  //=============================================================================
  //----------------------------LoadKlassNode::make------------------------------
  // Polymorphic factory method:
! Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
    // sanity check the alias category against the created node type
    const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
    assert(adr_type != NULL, "expecting TypeKlassPtr");
  #ifdef _LP64
    if (adr_type->is_ptr_to_narrowklass()) {
--- 2344,12 ---
  }
  
  //=============================================================================
  //----------------------------LoadKlassNode::make------------------------------
  // Polymorphic factory method:
! Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at,
+                           const TypeKlassPtr* tk) {
    // sanity check the alias category against the created node type
    const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
    assert(adr_type != NULL, "expecting TypeKlassPtr");
  #ifdef _LP64
    if (adr_type->is_ptr_to_narrowklass()) {

*** 2301,21 ***
      if (ik == phase->C->env()->Class_klass()
          && (offset == java_lang_Class::klass_offset() ||
              offset == java_lang_Class::array_klass_offset())) {
        // We are loading a special hidden field from a Class mirror object,
        // the field which points to the VM's Klass metaobject.
!       ciType* t = tinst->java_mirror_type();
        // java_mirror_type returns non-null for compile-time Class constants.
        if (t != NULL) {
          // constant oop => constant klass
          if (offset == java_lang_Class::array_klass_offset()) {
            if (t->is_void()) {
              // We cannot create a void array.  Since void is a primitive type return null
              // klass.  Users of this result need to do a null check on the returned klass.
              return TypePtr::NULL_PTR;
            }
!           return TypeKlassPtr::make(ciArrayKlass::make(t));
          }
          if (!t->is_klass()) {
            // a primitive Class (e.g., int.class) has NULL for a klass field
            return TypePtr::NULL_PTR;
          }
--- 2392,22 ---
      if (ik == phase->C->env()->Class_klass()
          && (offset == java_lang_Class::klass_offset() ||
              offset == java_lang_Class::array_klass_offset())) {
        // We are loading a special hidden field from a Class mirror object,
        // the field which points to the VM's Klass metaobject.
!       bool null_free = false;
+       ciType* t = tinst->java_mirror_type(&null_free);
        // java_mirror_type returns non-null for compile-time Class constants.
        if (t != NULL) {
          // constant oop => constant klass
          if (offset == java_lang_Class::array_klass_offset()) {
            if (t->is_void()) {
              // We cannot create a void array.  Since void is a primitive type return null
              // klass.  Users of this result need to do a null check on the returned klass.
              return TypePtr::NULL_PTR;
            }
!           return TypeKlassPtr::make(ciArrayKlass::make(t, null_free));
          }
          if (!t->is_klass()) {
            // a primitive Class (e.g., int.class) has NULL for a klass field
            return TypePtr::NULL_PTR;
          }

*** 2341,65 ***
          // Return precise klass
          return TypeKlassPtr::make(ik);
        }
  
        // Return root of possible klass
!       return TypeKlassPtr::make(TypePtr::NotNull, ik, 0/*offset*/);
      }
    }
  
    // Check for loading klass from an array
    const TypeAryPtr *tary = tp->isa_aryptr();
!   if( tary != NULL ) {
      ciKlass *tary_klass = tary->klass();
      if (tary_klass != NULL   // can be NULL when at BOTTOM or TOP
          && tary->offset() == oopDesc::klass_offset_in_bytes()) {
        if (tary->klass_is_exact()) {
          return TypeKlassPtr::make(tary_klass);
        }
!       ciArrayKlass *ak = tary->klass()->as_array_klass();
        // If the klass is an object array, we defer the question to the
        // array component klass.
!       if( ak->is_obj_array_klass() ) {
!         assert( ak->is_loaded(), "" );
          ciKlass *base_k = ak->as_obj_array_klass()->base_element_klass();
!         if( base_k->is_loaded() && base_k->is_instance_klass() ) {
!           ciInstanceKlass* ik = base_k->as_instance_klass();
            // See if we can become precise: no subklasses and no interface
!           if (!ik->is_interface() && !ik->has_subklass()) {
              // Add a dependence; if any subclass added we need to recompile
              if (!ik->is_final()) {
                phase->C->dependencies()->assert_leaf_type(ik);
              }
              // Return precise array klass
              return TypeKlassPtr::make(ak);
            }
          }
!         return TypeKlassPtr::make(TypePtr::NotNull, ak, 0/*offset*/);
!       } else {                  // Found a type-array?
-         assert( ak->is_type_array_klass(), "" );
          return TypeKlassPtr::make(ak); // These are always precise
        }
      }
    }
  
    // Check for loading klass from an array klass
    const TypeKlassPtr *tkls = tp->isa_klassptr();
    if (tkls != NULL && !StressReflectiveCode) {
!     ciKlass* klass = tkls->klass();
-     if( !klass->is_loaded() )
        return _type;             // Bail out if not loaded
      if( klass->is_obj_array_klass() &&
          tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
        ciKlass* elem = klass->as_obj_array_klass()->element_klass();
        // // Always returning precise element type is incorrect,
        // // e.g., element type could be object and array may contain strings
        // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
  
        // The array's TypeKlassPtr was declared 'precise' or 'not precise'
        // according to the element type's subclassing.
!       return TypeKlassPtr::make(tkls->ptr(), elem, 0/*offset*/);
      }
      if( klass->is_instance_klass() && tkls->klass_is_exact() &&
          tkls->offset() == in_bytes(Klass::super_offset())) {
        ciKlass* sup = klass->as_instance_klass()->super();
        // The field is Klass::_super.  Return its (constant) value.
--- 2433,71 ---
          // Return precise klass
          return TypeKlassPtr::make(ik);
        }
  
        // Return root of possible klass
!       return TypeInstKlassPtr::make(TypePtr::NotNull, ik, Type::Offset(0), tinst->flatten_array());
      }
    }
  
    // Check for loading klass from an array
    const TypeAryPtr *tary = tp->isa_aryptr();
!   if (tary != NULL) {
      ciKlass *tary_klass = tary->klass();
      if (tary_klass != NULL   // can be NULL when at BOTTOM or TOP
          && tary->offset() == oopDesc::klass_offset_in_bytes()) {
        if (tary->klass_is_exact()) {
          return TypeKlassPtr::make(tary_klass);
        }
!       ciArrayKlass* ak = tary_klass->as_array_klass();
        // If the klass is an object array, we defer the question to the
        // array component klass.
!       if (ak->is_obj_array_klass()) {
!         assert(ak->is_loaded(), "");
          ciKlass *base_k = ak->as_obj_array_klass()->base_element_klass();
!         if (base_k->is_loaded() && base_k->is_instance_klass()) {
!           ciInstanceKlass *ik = base_k->as_instance_klass();
            // See if we can become precise: no subklasses and no interface
!           // Do not fold klass loads from [LMyValue. The runtime type might be [QMyValue due to [QMyValue <: [LMyValue
+           // and the klass for [QMyValue is not equal to the klass for [LMyValue.
+           if (!ik->is_interface() && !ik->has_subklass() && (!ik->is_inlinetype() || ak->is_elem_null_free())) {
              // Add a dependence; if any subclass added we need to recompile
              if (!ik->is_final()) {
                phase->C->dependencies()->assert_leaf_type(ik);
              }
              // Return precise array klass
              return TypeKlassPtr::make(ak);
            }
          }
!         return TypeAryKlassPtr::make(TypePtr::NotNull, ak, Type::Offset(0), tary->is_not_flat(), tary->is_not_null_free(), tary->is_null_free());
!       } else if (ak->is_type_array_klass()) {
          return TypeKlassPtr::make(ak); // These are always precise
        }
      }
    }
  
    // Check for loading klass from an array klass
    const TypeKlassPtr *tkls = tp->isa_klassptr();
    if (tkls != NULL && !StressReflectiveCode) {
!     if (!tkls->is_loaded()) {
        return _type;             // Bail out if not loaded
+     }
+     ciKlass* klass = tkls->klass();
      if( klass->is_obj_array_klass() &&
          tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
        ciKlass* elem = klass->as_obj_array_klass()->element_klass();
        // // Always returning precise element type is incorrect,
        // // e.g., element type could be object and array may contain strings
        // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
  
        // The array's TypeKlassPtr was declared 'precise' or 'not precise'
        // according to the element type's subclassing.
!       return TypeKlassPtr::make(tkls->ptr(), elem, Type::Offset(0));
+     } else if (klass->is_flat_array_klass() &&
+                tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
+       ciKlass* elem = klass->as_flat_array_klass()->element_klass();
+       return TypeInstKlassPtr::make(tkls->ptr(), elem, Type::Offset(0), /* flatten_array= */ true);
      }
      if( klass->is_instance_klass() && tkls->klass_is_exact() &&
          tkls->offset() == in_bytes(Klass::super_offset())) {
        ciKlass* sup = klass->as_instance_klass()->super();
        // The field is Klass::_super.  Return its (constant) value.

*** 2603,10 ***
--- 2701,11 ---
    case T_LONG:    return new StoreLNode(ctl, mem, adr, adr_type, val, mo);
    case T_FLOAT:   return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
    case T_DOUBLE:  return new StoreDNode(ctl, mem, adr, adr_type, val, mo);
    case T_METADATA:
    case T_ADDRESS:
+   case T_INLINE_TYPE:
    case T_OBJECT:
  #ifdef _LP64
      if (adr->bottom_type()->is_ptr_to_narrowoop()) {
        val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
        return new StoreNNode(ctl, mem, adr, adr_type, val, mo);

*** 2664,11 ***
    Node* value   = in(MemNode::ValueIn);
    // Back-to-back stores to same address?  Fold em up.  Generally
    // unsafe if I have intervening uses...  Also disallowed for StoreCM
    // since they must follow each StoreP operation.  Redundant StoreCMs
    // are eliminated just before matching in final_graph_reshape.
!   {
      Node* st = mem;
      // If Store 'st' has more than one use, we cannot fold 'st' away.
      // For example, 'st' might be the final state at a conditional
      // return.  Or, 'st' might be used by some node which is live at
      // the same time 'st' is live, which might be unschedulable.  So,
--- 2763,11 ---
    Node* value   = in(MemNode::ValueIn);
    // Back-to-back stores to same address?  Fold em up.  Generally
    // unsafe if I have intervening uses...  Also disallowed for StoreCM
    // since they must follow each StoreP operation.  Redundant StoreCMs
    // are eliminated just before matching in final_graph_reshape.
!   if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
      Node* st = mem;
      // If Store 'st' has more than one use, we cannot fold 'st' away.
      // For example, 'st' might be the final state at a conditional
      // return.  Or, 'st' might be used by some node which is live at
      // the same time 'st' is live, which might be unschedulable.  So,

*** 2684,10 ***
--- 2783,11 ---
               st->Opcode() == Op_StoreVectorScatter ||
               Opcode() == Op_StoreVectorScatter ||
               phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
               (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
               (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
+              (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
               (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
               "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
  
        if (st->in(MemNode::Address)->eqv_uncast(address) &&
            st->as_Store()->memory_size() <= this->memory_size()) {

*** 2780,18 ***
    }
  
    // Store of zero anywhere into a freshly-allocated object?
    // Then the store is useless.
    // (It must already have been captured by the InitializeNode.)
!   if (result == this &&
-       ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
      // a newly allocated object is already all-zeroes everywhere
!     if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
        result = mem;
      }
  
!     if (result == this) {
        // the store may also apply to zero-bits in an earlier object
        Node* prev_mem = find_previous_store(phase);
        // Steps (a), (b):  Walk past independent stores to find an exact match.
        if (prev_mem != NULL) {
          Node* prev_val = can_see_stored_value(prev_mem, phase);
--- 2880,19 ---
    }
  
    // Store of zero anywhere into a freshly-allocated object?
    // Then the store is useless.
    // (It must already have been captured by the InitializeNode.)
!   if (result == this && ReduceFieldZeroing) {
      // a newly allocated object is already all-zeroes everywhere
!     if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
+         (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::DefaultValue) == val)) {
+       assert(!phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::DefaultValue) == NULL, "storing null to inline type array is forbidden");
        result = mem;
      }
  
!     if (result == this && phase->type(val)->is_zero_type()) {
        // the store may also apply to zero-bits in an earlier object
        Node* prev_mem = find_previous_store(phase);
        // Steps (a), (b):  Walk past independent stores to find an exact match.
        if (prev_mem != NULL) {
          Node* prev_val = can_see_stored_value(prev_mem, phase);

*** 2972,13 ***
    Node* progress = StoreNode::Ideal(phase, can_reshape);
    if (progress != NULL) return progress;
  
    Node* my_store = in(MemNode::OopStore);
    if (my_store->is_MergeMem()) {
!     Node* mem = my_store->as_MergeMem()->memory_at(oop_alias_idx());
!     set_req_X(MemNode::OopStore, mem, phase);
!     return this;
    }
  
    return NULL;
  }
  
--- 3073,17 ---
    Node* progress = StoreNode::Ideal(phase, can_reshape);
    if (progress != NULL) return progress;
  
    Node* my_store = in(MemNode::OopStore);
    if (my_store->is_MergeMem()) {
!     if (oop_alias_idx() != phase->C->get_alias_index(TypeAryPtr::INLINES) ||
!         phase->C->flattened_accesses_share_alias()) {
!       // The alias that was recorded is no longer accurate enough.
+       Node* mem = my_store->as_MergeMem()->memory_at(oop_alias_idx());
+       set_req_X(MemNode::OopStore, mem, phase);
+       return this;
+     }
    }
  
    return NULL;
  }
  

*** 3133,11 ***
    if (size <= 0 || size % unit != 0)  return NULL;
    intptr_t count = size / unit;
    // Length too long; communicate this to matchers and assemblers.
    // Assemblers are responsible to produce fast hardware clears for it.
    if (size > InitArrayShortSize) {
!     return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
    } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
      return NULL;
    }
    Node *mem = in(1);
    if( phase->type(mem)==Type::TOP ) return NULL;
--- 3238,11 ---
    if (size <= 0 || size % unit != 0)  return NULL;
    intptr_t count = size / unit;
    // Length too long; communicate this to matchers and assemblers.
    // Assemblers are responsible to produce fast hardware clears for it.
    if (size > InitArrayShortSize) {
!     return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
    } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
      return NULL;
    }
    Node *mem = in(1);
    if( phase->type(mem)==Type::TOP ) return NULL;

*** 3150,18 ***
    else              atp = atp->add_offset(Type::OffsetBot);
    // Get base for derived pointer purposes
    if( adr->Opcode() != Op_AddP ) Unimplemented();
    Node *base = adr->in(1);
  
!   Node *zero = phase->makecon(TypeLong::ZERO);
    Node *off  = phase->MakeConX(BytesPerLong);
!   mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
    count--;
    while( count-- ) {
      mem = phase->transform(mem);
      adr = phase->transform(new AddPNode(base,adr,off));
!     mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
    }
    return mem;
  }
  
  //----------------------------step_through----------------------------------
--- 3255,18 ---
    else              atp = atp->add_offset(Type::OffsetBot);
    // Get base for derived pointer purposes
    if( adr->Opcode() != Op_AddP ) Unimplemented();
    Node *base = adr->in(1);
  
!   Node *val = in(4);
    Node *off  = phase->MakeConX(BytesPerLong);
!   mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
    count--;
    while( count-- ) {
      mem = phase->transform(mem);
      adr = phase->transform(new AddPNode(base,adr,off));
!     mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
    }
    return mem;
  }
  
  //----------------------------step_through----------------------------------

*** 3191,31 ***
  }
  
  //----------------------------clear_memory-------------------------------------
  // Generate code to initialize object storage to zero.
  Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
                                     intptr_t start_offset,
                                     Node* end_offset,
                                     PhaseGVN* phase) {
    intptr_t offset = start_offset;
  
    int unit = BytesPerLong;
    if ((offset % unit) != 0) {
      Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
      adr = phase->transform(adr);
      const TypePtr* atp = TypeRawPtr::BOTTOM;
!     mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
      mem = phase->transform(mem);
      offset += BytesPerInt;
    }
    assert((offset % unit) == 0, "");
  
    // Initialize the remaining stuff, if any, with a ClearArray.
!   return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, phase);
  }
  
  Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
                                     Node* start_offset,
                                     Node* end_offset,
                                     PhaseGVN* phase) {
    if (start_offset == end_offset) {
      // nothing to do
--- 3296,40 ---
  }
  
  //----------------------------clear_memory-------------------------------------
  // Generate code to initialize object storage to zero.
  Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
+                                    Node* val,
+                                    Node* raw_val,
                                     intptr_t start_offset,
                                     Node* end_offset,
                                     PhaseGVN* phase) {
    intptr_t offset = start_offset;
  
    int unit = BytesPerLong;
    if ((offset % unit) != 0) {
      Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
      adr = phase->transform(adr);
      const TypePtr* atp = TypeRawPtr::BOTTOM;
!     if (val != NULL) {
+       assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
+       mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
+     } else {
+       assert(raw_val == NULL, "val may not be null");
+       mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
+     }
      mem = phase->transform(mem);
      offset += BytesPerInt;
    }
    assert((offset % unit) == 0, "");
  
    // Initialize the remaining stuff, if any, with a ClearArray.
!   return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, phase);
  }
  
  Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
+                                    Node* raw_val,
                                     Node* start_offset,
                                     Node* end_offset,
                                     PhaseGVN* phase) {
    if (start_offset == end_offset) {
      // nothing to do

*** 3234,15 ***
    }
  
    // Bulk clear double-words
    Node* zsize = phase->transform(new SubXNode(zend, zbase) );
    Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
!   mem = new ClearArrayNode(ctl, mem, zsize, adr, false);
    return phase->transform(mem);
  }
  
  Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
                                     intptr_t start_offset,
                                     intptr_t end_offset,
                                     PhaseGVN* phase) {
    if (start_offset == end_offset) {
      // nothing to do
--- 3348,20 ---
    }
  
    // Bulk clear double-words
    Node* zsize = phase->transform(new SubXNode(zend, zbase) );
    Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
!   if (raw_val == NULL) {
+     raw_val = phase->MakeConX(0);
+   }
+   mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
    return phase->transform(mem);
  }
  
  Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
+                                    Node* val,
+                                    Node* raw_val,
                                     intptr_t start_offset,
                                     intptr_t end_offset,
                                     PhaseGVN* phase) {
    if (start_offset == end_offset) {
      // nothing to do

*** 3253,18 ***
    intptr_t done_offset = end_offset;
    if ((done_offset % BytesPerLong) != 0) {
      done_offset -= BytesPerInt;
    }
    if (done_offset > start_offset) {
!     mem = clear_memory(ctl, mem, dest,
                         start_offset, phase->MakeConX(done_offset), phase);
    }
    if (done_offset < end_offset) { // emit the final 32-bit store
      Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
      adr = phase->transform(adr);
      const TypePtr* atp = TypeRawPtr::BOTTOM;
!     mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
      mem = phase->transform(mem);
      done_offset += BytesPerInt;
    }
    assert(done_offset == end_offset, "");
    return mem;
--- 3372,24 ---
    intptr_t done_offset = end_offset;
    if ((done_offset % BytesPerLong) != 0) {
      done_offset -= BytesPerInt;
    }
    if (done_offset > start_offset) {
!     mem = clear_memory(ctl, mem, dest, val, raw_val,
                         start_offset, phase->MakeConX(done_offset), phase);
    }
    if (done_offset < end_offset) { // emit the final 32-bit store
      Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
      adr = phase->transform(adr);
      const TypePtr* atp = TypeRawPtr::BOTTOM;
!     if (val != NULL) {
+       assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
+       mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
+     } else {
+       assert(raw_val == NULL, "val may not be null");
+       mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
+     }
      mem = phase->transform(mem);
      done_offset += BytesPerInt;
    }
    assert(done_offset == end_offset, "");
    return mem;

*** 3405,11 ***
    return TypeTuple::MEMBAR;
  }
  
  //------------------------------match------------------------------------------
  // Construct projections for memory.
! Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
    switch (proj->_con) {
    case TypeFunc::Control:
    case TypeFunc::Memory:
      return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
    }
--- 3530,11 ---
    return TypeTuple::MEMBAR;
  }
  
  //------------------------------match------------------------------------------
  // Construct projections for memory.
! Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
    switch (proj->_con) {
    case TypeFunc::Control:
    case TypeFunc::Memory:
      return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
    }

*** 3712,11 ***
  
  // convenience function
  // return false if the init contains any stores already
  bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
    InitializeNode* init = initialization();
!   if (init == NULL || init->is_complete())  return false;
    init->remove_extra_zeroes();
    // for now, if this allocation has already collected any inits, bail:
    if (init->is_non_zero())  return false;
    init->set_complete(phase);
    return true;
--- 3837,13 ---
  
  // convenience function
  // return false if the init contains any stores already
  bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
    InitializeNode* init = initialization();
!   if (init == NULL || init->is_complete()) {
+     return false;
+   }
    init->remove_extra_zeroes();
    // for now, if this allocation has already collected any inits, bail:
    if (init->is_non_zero())  return false;
    init->set_complete(phase);
    return true;

*** 3890,10 ***
--- 4017,16 ---
                  // after the InitializeNode. We check the control of the
                  // object/array that is loaded from. If it's the same as
                  // the store control then we cannot capture the store.
                  assert(!n->is_Store(), "2 stores to same slice on same control?");
                  Node* base = other_adr;
+                 if (base->is_Phi()) {
+                   // In rare case, base may be a PhiNode and it may read
+                   // the same memory slice between InitializeNode and store.
+                   failed = true;
+                   break;
+                 }
                  assert(base->is_AddP(), "should be addp but is %s", base->Name());
                  base = base->in(AddPNode::Base);
                  if (base != NULL) {
                    base = base->uncast();
                    if (base->is_Proj() && base->in(0) == alloc) {

*** 4474,10 ***
--- 4607,12 ---
        if (zeroes_needed > zeroes_done) {
          intptr_t zsize = zeroes_needed - zeroes_done;
          // Do some incremental zeroing on rawmem, in parallel with inits.
          zeroes_done = align_down(zeroes_done, BytesPerInt);
          rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
+                                               allocation()->in(AllocateNode::DefaultValue),
+                                               allocation()->in(AllocateNode::RawDefaultValue),
                                                zeroes_done, zeroes_needed,
                                                phase);
          zeroes_done = zeroes_needed;
          if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
            do_zeroing = false;   // leave the hole, next time

*** 4533,10 ***
--- 4668,12 ---
            zeroes_done = size_limit;
        }
      }
      if (zeroes_done < size_limit) {
        rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
+                                             allocation()->in(AllocateNode::DefaultValue),
+                                             allocation()->in(AllocateNode::RawDefaultValue),
                                              zeroes_done, size_in_bytes, phase);
      }
    }
  
    set_complete(phase);
< prev index next >