< prev index next >

src/hotspot/share/opto/memnode.cpp

Print this page




  28 #include "gc/shared/barrierSet.hpp"
  29 #include "gc/shared/c2/barrierSetC2.hpp"
  30 #include "memory/allocation.inline.hpp"
  31 #include "memory/resourceArea.hpp"
  32 #include "oops/objArrayKlass.hpp"
  33 #include "opto/addnode.hpp"
  34 #include "opto/arraycopynode.hpp"
  35 #include "opto/cfgnode.hpp"
  36 #include "opto/compile.hpp"
  37 #include "opto/connode.hpp"
  38 #include "opto/convertnode.hpp"
  39 #include "opto/loopnode.hpp"
  40 #include "opto/machnode.hpp"
  41 #include "opto/matcher.hpp"
  42 #include "opto/memnode.hpp"
  43 #include "opto/mulnode.hpp"
  44 #include "opto/narrowptrnode.hpp"
  45 #include "opto/phaseX.hpp"
  46 #include "opto/regmask.hpp"
  47 #include "opto/rootnode.hpp"

  48 #include "utilities/align.hpp"
  49 #include "utilities/copy.hpp"
  50 #include "utilities/macros.hpp"
  51 #include "utilities/vmError.hpp"
  52 #if INCLUDE_ZGC
  53 #include "gc/z/c2/zBarrierSetC2.hpp"
  54 #endif
  55 
  56 // Portions of code courtesy of Clifford Click
  57 
  58 // Optimization - Graph Style
  59 
  60 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem,  const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
  61 
  62 //=============================================================================
  63 uint MemNode::size_of() const { return sizeof(*this); }
  64 
  65 const TypePtr *MemNode::adr_type() const {
  66   Node* adr = in(Address);
  67   if (adr == NULL)  return NULL; // node is dead


 222       // clone the Phi with our address type
 223       result = mphi->split_out_instance(t_adr, igvn);
 224     } else {
 225       assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
 226     }
 227   }
 228   return result;
 229 }
 230 
 231 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem,  const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
 232   uint alias_idx = phase->C->get_alias_index(tp);
 233   Node *mem = mmem;
 234 #ifdef ASSERT
 235   {
 236     // Check that current type is consistent with the alias index used during graph construction
 237     assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
 238     bool consistent =  adr_check == NULL || adr_check->empty() ||
 239                        phase->C->must_alias(adr_check, alias_idx );
 240     // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
 241     if( !consistent && adr_check != NULL && !adr_check->empty() &&
 242                tp->isa_aryptr() &&        tp->offset() == Type::OffsetBot &&
 243         adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
 244         ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
 245           adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
 246           adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
 247       // don't assert if it is dead code.
 248       consistent = true;
 249     }
 250     if( !consistent ) {
 251       st->print("alias_idx==%d, adr_check==", alias_idx);
 252       if( adr_check == NULL ) {
 253         st->print("NULL");
 254       } else {
 255         adr_check->dump();
 256       }
 257       st->cr();
 258       print_alias_types();
 259       assert(consistent, "adr_check must match alias idx");
 260     }
 261   }
 262 #endif


 820          "use LoadKlassNode instead");
 821   assert(!(adr_type->isa_aryptr() &&
 822            adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
 823          "use LoadRangeNode instead");
 824   // Check control edge of raw loads
 825   assert( ctl != NULL || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
 826           // oop will be recorded in oop map if load crosses safepoint
 827           rt->isa_oopptr() || is_immutable_value(adr),
 828           "raw memory operations should have control edge");
 829   LoadNode* load = NULL;
 830   switch (bt) {
 831   case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 832   case T_BYTE:    load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 833   case T_INT:     load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 834   case T_CHAR:    load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 835   case T_SHORT:   load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 836   case T_LONG:    load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency); break;
 837   case T_FLOAT:   load = new LoadFNode (ctl, mem, adr, adr_type, rt,            mo, control_dependency); break;
 838   case T_DOUBLE:  load = new LoadDNode (ctl, mem, adr, adr_type, rt,            mo, control_dependency); break;
 839   case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(),  mo, control_dependency); break;

 840   case T_OBJECT:
 841 #ifdef _LP64
 842     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
 843       load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
 844     } else
 845 #endif
 846     {
 847       assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
 848       load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
 849     }
 850     break;
 851   default:
 852     ShouldNotReachHere();
 853     break;
 854   }
 855   assert(load != NULL, "LoadNode should have been created");
 856   if (unaligned) {
 857     load->set_unaligned_access();
 858   }
 859   if (mismatched) {


1075         // the same pointer-and-offset that we stored to.
1076         // Casted version may carry a dependency and it is respected.
1077         // Thus, we are able to replace L by V.
1078       }
1079       // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1080       if (store_Opcode() != st->Opcode())
1081         return NULL;
1082       return st->in(MemNode::ValueIn);
1083     }
1084 
1085     // A load from a freshly-created object always returns zero.
1086     // (This can happen after LoadNode::Ideal resets the load's memory input
1087     // to find_captured_store, which returned InitializeNode::zero_memory.)
1088     if (st->is_Proj() && st->in(0)->is_Allocate() &&
1089         (st->in(0) == ld_alloc) &&
1090         (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1091       // return a zero value for the load's basic type
1092       // (This is one of the few places where a generic PhaseTransform
1093       // can create new nodes.  Think of it as lazily manifesting
1094       // virtually pre-existing constants.)






1095       return phase->zerocon(memory_type());
1096     }
1097 
1098     // A load from an initialization barrier can match a captured store.
1099     if (st->is_Proj() && st->in(0)->is_Initialize()) {
1100       InitializeNode* init = st->in(0)->as_Initialize();
1101       AllocateNode* alloc = init->allocation();
1102       if ((alloc != NULL) && (alloc == ld_alloc)) {
1103         // examine a captured store value
1104         st = init->find_captured_store(ld_off, memory_size(), phase);
1105         if (st != NULL) {
1106           continue;             // take one more trip around
1107         }
1108       }
1109     }
1110 
1111     // Load boxed value from result of valueOf() call is input parameter.
1112     if (this->is_Load() && ld_adr->is_AddP() &&
1113         (tp != NULL) && tp->is_ptr_to_boxed_value()) {
1114       intptr_t ignore = 0;


1132 //----------------------is_instance_field_load_with_local_phi------------------
1133 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1134   if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1135       in(Address)->is_AddP() ) {
1136     const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1137     // Only instances and boxed values.
1138     if( t_oop != NULL &&
1139         (t_oop->is_ptr_to_boxed_value() ||
1140          t_oop->is_known_instance_field()) &&
1141         t_oop->offset() != Type::OffsetBot &&
1142         t_oop->offset() != Type::OffsetTop) {
1143       return true;
1144     }
1145   }
1146   return false;
1147 }
1148 
1149 //------------------------------Identity---------------------------------------
1150 // Loads are identity if previous store is to same address
1151 Node* LoadNode::Identity(PhaseGVN* phase) {



























1152   // If the previous store-maker is the right kind of Store, and the store is
1153   // to the same address, then we are equal to the value stored.
1154   Node* mem = in(Memory);
1155   Node* value = can_see_stored_value(mem, phase);
1156   if( value ) {
1157     // byte, short & char stores truncate naturally.
1158     // A load has to load the truncated value which requires
1159     // some sort of masking operation and that requires an
1160     // Ideal call instead of an Identity call.
1161     if (memory_size() < BytesPerInt) {
1162       // If the input to the store does not fit with the load's result type,
1163       // it must be truncated via an Ideal call.
1164       if (!phase->type(value)->higher_equal(phase->type(this)))
1165         return this;
1166     }
1167     // (This works even when value is a Con, but LoadNode::Value
1168     // usually runs first, producing the singleton type of the Con.)
1169     return value;
1170   }
1171 


1667   // fold up, do so.
1668   Node* prev_mem = find_previous_store(phase);
1669   if (prev_mem != NULL) {
1670     Node* value = can_see_arraycopy_value(prev_mem, phase);
1671     if (value != NULL) {
1672       return value;
1673     }
1674   }
1675   // Steps (a), (b):  Walk past independent stores to find an exact match.
1676   if (prev_mem != NULL && prev_mem != in(MemNode::Memory)) {
1677     // (c) See if we can fold up on the spot, but don't fold up here.
1678     // Fold-up might require truncation (for LoadB/LoadS/LoadUS) or
1679     // just return a prior value, which is done by Identity calls.
1680     if (can_see_stored_value(prev_mem, phase)) {
1681       // Make ready for step (d):
1682       set_req(MemNode::Memory, prev_mem);
1683       return this;
1684     }
1685   }
1686 











1687   return progress ? this : NULL;
1688 }
1689 
1690 // Helper to recognize certain Klass fields which are invariant across
1691 // some group of array types (e.g., int[] or all T[] where T < Object).
1692 const Type*
1693 LoadNode::load_array_final_field(const TypeKlassPtr *tkls,
1694                                  ciKlass* klass) const {
1695   if (tkls->offset() == in_bytes(Klass::modifier_flags_offset())) {
1696     // The field is Klass::_modifier_flags.  Return its (constant) value.
1697     // (Folds up the 2nd indirection in aClassConstant.getModifiers().)
1698     assert(this->Opcode() == Op_LoadI, "must load an int from _modifier_flags");
1699     return TypeInt::make(klass->modifier_flags());
1700   }
1701   if (tkls->offset() == in_bytes(Klass::access_flags_offset())) {
1702     // The field is Klass::_access_flags.  Return its (constant) value.
1703     // (Folds up the 2nd indirection in Reflection.getClassAccessFlags(aClassConstant).)
1704     assert(this->Opcode() == Op_LoadI, "must load an int from _access_flags");
1705     return TypeInt::make(klass->access_flags());
1706   }


1756       }
1757     }
1758 
1759     // Don't do this for integer types. There is only potential profit if
1760     // the element type t is lower than _type; that is, for int types, if _type is
1761     // more restrictive than t.  This only happens here if one is short and the other
1762     // char (both 16 bits), and in those cases we've made an intentional decision
1763     // to use one kind of load over the other. See AndINode::Ideal and 4965907.
1764     // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
1765     //
1766     // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
1767     // where the _gvn.type of the AddP is wider than 8.  This occurs when an earlier
1768     // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
1769     // subsumed by p1.  If p1 is on the worklist but has not yet been re-transformed,
1770     // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
1771     // In fact, that could have been the original type of p1, and p1 could have
1772     // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
1773     // expression (LShiftL quux 3) independently optimized to the constant 8.
1774     if ((t->isa_int() == NULL) && (t->isa_long() == NULL)
1775         && (_type->isa_vect() == NULL)

1776         && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
1777       // t might actually be lower than _type, if _type is a unique
1778       // concrete subclass of abstract class t.
1779       if (off_beyond_header || off == Type::OffsetBot) {  // is the offset beyond the header?
1780         const Type* jt = t->join_speculative(_type);
1781         // In any case, do not allow the join, per se, to empty out the type.
1782         if (jt->empty() && !t->empty()) {
1783           // This can happen if a interface-typed array narrows to a class type.
1784           jt = _type;
1785         }
1786 #ifdef ASSERT
1787         if (phase->C->eliminate_boxing() && adr->is_AddP()) {
1788           // The pointers in the autobox arrays are always non-null
1789           Node* base = adr->in(AddPNode::Base);
1790           if ((base != NULL) && base->is_DecodeN()) {
1791             // Get LoadN node which loads IntegerCache.cache field
1792             base = base->in(1);
1793           }
1794           if ((base != NULL) && base->is_Con()) {
1795             const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
1796             if ((base_type != NULL) && base_type->is_autobox_cache()) {
1797               // It could be narrow oop
1798               assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
1799             }
1800           }
1801         }
1802 #endif
1803         return jt;
1804       }
1805     }
1806   } else if (tp->base() == Type::InstPtr) {
1807     assert( off != Type::OffsetBot ||
1808             // arrays can be cast to Objects
1809             tp->is_oopptr()->klass()->is_java_lang_Object() ||

1810             // unsafe field access may not have a constant offset
1811             C->has_unsafe_access(),
1812             "Field accesses must be precise" );
1813     // For oop loads, we expect the _type to be precise.
1814 
1815     // Optimize loads from constant fields.
1816     const TypeInstPtr* tinst = tp->is_instptr();



















1817     ciObject* const_oop = tinst->const_oop();
1818     if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != NULL && const_oop->is_instance()) {
1819       const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), memory_type());









1820       if (con_type != NULL) {
1821         return con_type;
1822       }
1823     }
1824   } else if (tp->base() == Type::KlassPtr) {
1825     assert( off != Type::OffsetBot ||
1826             // arrays can be cast to Objects

1827             tp->is_klassptr()->klass()->is_java_lang_Object() ||
1828             // also allow array-loading from the primary supertype
1829             // array during subtype checks
1830             Opcode() == Op_LoadKlass,
1831             "Field accesses must be precise" );
1832     // For klass/static loads, we expect the _type to be precise
1833   } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
1834     /* With mirrors being an indirect in the Klass*
1835      * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
1836      * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
1837      *
1838      * So check the type and klass of the node before the LoadP.
1839      */
1840     Node* adr2 = adr->in(MemNode::Address);
1841     const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
1842     if (tkls != NULL && !StressReflectiveCode) {
1843       ciKlass* klass = tkls->klass();
1844       if (klass->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
1845         assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
1846         assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
1847         return TypeInstPtr::make(klass->java_mirror());
















1848       }
1849     }
1850   }
1851 
1852   const TypeKlassPtr *tkls = tp->isa_klassptr();
1853   if (tkls != NULL && !StressReflectiveCode) {
1854     ciKlass* klass = tkls->klass();
1855     if (klass->is_loaded() && tkls->klass_is_exact()) {
1856       // We are loading a field from a Klass metaobject whose identity
1857       // is known at compile time (the type is "exact" or "precise").
1858       // Check for fields we know are maintained as constants by the VM.
1859       if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
1860         // The field is Klass::_super_check_offset.  Return its (constant) value.
1861         // (Folds up type checking code.)
1862         assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
1863         return TypeInt::make(klass->super_check_offset());
1864       }
1865       // Compute index into primary_supers array
1866       juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
1867       // Check for overflowing; use unsigned compare to handle the negative case.
1868       if( depth < ciKlass::primary_super_limit() ) {
1869         // The field is an element of Klass::_primary_supers.  Return its (constant) value.
1870         // (Folds up type checking code.)
1871         assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
1872         ciKlass *ss = klass->super_of_depth(depth);
1873         return ss ? TypeKlassPtr::make(ss) : TypePtr::NULL_PTR;
1874       }
1875       const Type* aift = load_array_final_field(tkls, klass);
1876       if (aift != NULL)  return aift;
1877     }
1878 
1879     // We can still check if we are loading from the primary_supers array at a
1880     // shallow enough depth.  Even though the klass is not exact, entries less
1881     // than or equal to its super depth are correct.
1882     if (klass->is_loaded() ) {
1883       ciType *inner = klass;
1884       while( inner->is_obj_array_klass() )
1885         inner = inner->as_obj_array_klass()->base_element_type();
1886       if( inner->is_instance_klass() &&
1887           !inner->as_instance_klass()->flags().is_interface() ) {
1888         // Compute index into primary_supers array
1889         juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
1890         // Check for overflowing; use unsigned compare to handle the negative case.
1891         if( depth < ciKlass::primary_super_limit() &&
1892             depth <= klass->super_depth() ) { // allow self-depth checks to handle self-check case
1893           // The field is an element of Klass::_primary_supers.  Return its (constant) value.
1894           // (Folds up type checking code.)
1895           assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
1896           ciKlass *ss = klass->super_of_depth(depth);
1897           return ss ? TypeKlassPtr::make(ss) : TypePtr::NULL_PTR;
1898         }
1899       }
1900     }
1901 
1902     // If the type is enough to determine that the thing is not an array,


2108   // Either input is TOP ==> the result is TOP
2109   const Type *t1 = phase->type( in(MemNode::Memory) );
2110   if (t1 == Type::TOP)  return Type::TOP;
2111   Node *adr = in(MemNode::Address);
2112   const Type *t2 = phase->type( adr );
2113   if (t2 == Type::TOP)  return Type::TOP;
2114   const TypePtr *tp = t2->is_ptr();
2115   if (TypePtr::above_centerline(tp->ptr()) ||
2116       tp->ptr() == TypePtr::Null)  return Type::TOP;
2117 
2118   // Return a more precise klass, if possible
2119   const TypeInstPtr *tinst = tp->isa_instptr();
2120   if (tinst != NULL) {
2121     ciInstanceKlass* ik = tinst->klass()->as_instance_klass();
2122     int offset = tinst->offset();
2123     if (ik == phase->C->env()->Class_klass()
2124         && (offset == java_lang_Class::klass_offset_in_bytes() ||
2125             offset == java_lang_Class::array_klass_offset_in_bytes())) {
2126       // We are loading a special hidden field from a Class mirror object,
2127       // the field which points to the VM's Klass metaobject.
2128       ciType* t = tinst->java_mirror_type();

2129       // java_mirror_type returns non-null for compile-time Class constants.
2130       if (t != NULL) {
2131         // constant oop => constant klass
2132         if (offset == java_lang_Class::array_klass_offset_in_bytes()) {
2133           if (t->is_void()) {
2134             // We cannot create a void array.  Since void is a primitive type return null
2135             // klass.  Users of this result need to do a null check on the returned klass.
2136             return TypePtr::NULL_PTR;
2137           }
2138           return TypeKlassPtr::make(ciArrayKlass::make(t));
2139         }
2140         if (!t->is_klass()) {
2141           // a primitive Class (e.g., int.class) has NULL for a klass field
2142           return TypePtr::NULL_PTR;
2143         }
2144         // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2145         return TypeKlassPtr::make(t->as_klass());
2146       }
2147       // non-constant mirror, so we can't tell what's going on
2148     }
2149     if( !ik->is_loaded() )
2150       return _type;             // Bail out if not loaded
2151     if (offset == oopDesc::klass_offset_in_bytes()) {
2152       if (tinst->klass_is_exact()) {
2153         return TypeKlassPtr::make(ik);
2154       }
2155       // See if we can become precise: no subklasses and no interface
2156       // (Note:  We need to support verified interfaces.)
2157       if (!ik->is_interface() && !ik->has_subklass()) {
2158         //assert(!UseExactTypes, "this code should be useless with exact types");
2159         // Add a dependence; if any subclass added we need to recompile
2160         if (!ik->is_final()) {
2161           // %%% should use stronger assert_unique_concrete_subtype instead
2162           phase->C->dependencies()->assert_leaf_type(ik);
2163         }
2164         // Return precise klass
2165         return TypeKlassPtr::make(ik);
2166       }
2167 
2168       // Return root of possible klass
2169       return TypeKlassPtr::make(TypePtr::NotNull, ik, 0/*offset*/);
2170     }
2171   }
2172 
2173   // Check for loading klass from an array
2174   const TypeAryPtr *tary = tp->isa_aryptr();
2175   if( tary != NULL ) {
2176     ciKlass *tary_klass = tary->klass();
2177     if (tary_klass != NULL   // can be NULL when at BOTTOM or TOP
2178         && tary->offset() == oopDesc::klass_offset_in_bytes()) {



2179       if (tary->klass_is_exact()) {
2180         return TypeKlassPtr::make(tary_klass);
2181       }
2182       ciArrayKlass *ak = tary->klass()->as_array_klass();
2183       // If the klass is an object array, we defer the question to the
2184       // array component klass.
2185       if( ak->is_obj_array_klass() ) {
2186         assert( ak->is_loaded(), "" );
2187         ciKlass *base_k = ak->as_obj_array_klass()->base_element_klass();
2188         if( base_k->is_loaded() && base_k->is_instance_klass() ) {
2189           ciInstanceKlass* ik = base_k->as_instance_klass();
2190           // See if we can become precise: no subklasses and no interface
2191           if (!ik->is_interface() && !ik->has_subklass()) {
2192             //assert(!UseExactTypes, "this code should be useless with exact types");
2193             // Add a dependence; if any subclass added we need to recompile
2194             if (!ik->is_final()) {
2195               phase->C->dependencies()->assert_leaf_type(ik);
2196             }
2197             // Return precise array klass
2198             return TypeKlassPtr::make(ak);
2199           }
2200         }
2201         return TypeKlassPtr::make(TypePtr::NotNull, ak, 0/*offset*/);
2202       } else {                  // Found a type-array?
2203         //assert(!UseExactTypes, "this code should be useless with exact types");
2204         assert( ak->is_type_array_klass(), "" );
2205         return TypeKlassPtr::make(ak); // These are always precise
2206       }
2207     }
2208   }
2209 
2210   // Check for loading klass from an array klass
2211   const TypeKlassPtr *tkls = tp->isa_klassptr();
2212   if (tkls != NULL && !StressReflectiveCode) {
2213     ciKlass* klass = tkls->klass();
2214     if( !klass->is_loaded() )
2215       return _type;             // Bail out if not loaded


2216     if( klass->is_obj_array_klass() &&
2217         tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2218       ciKlass* elem = klass->as_obj_array_klass()->element_klass();
2219       // // Always returning precise element type is incorrect,
2220       // // e.g., element type could be object and array may contain strings
2221       // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2222 
2223       // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2224       // according to the element type's subclassing.
2225       return TypeKlassPtr::make(tkls->ptr(), elem, 0/*offset*/);




2226     }
2227     if( klass->is_instance_klass() && tkls->klass_is_exact() &&
2228         tkls->offset() == in_bytes(Klass::super_offset())) {
2229       ciKlass* sup = klass->as_instance_klass()->super();
2230       // The field is Klass::_super.  Return its (constant) value.
2231       // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2232       return sup ? TypeKlassPtr::make(sup) : TypePtr::NULL_PTR;
2233     }
2234   }
2235 
2236   // Bailout case
2237   return LoadNode::Value(phase);
2238 }
2239 
2240 //------------------------------Identity---------------------------------------
2241 // To clean up reflective code, simplify k.java_mirror.as_klass to plain k.
2242 // Also feed through the klass in Allocate(...klass...)._klass.
2243 Node* LoadKlassNode::Identity(PhaseGVN* phase) {
2244   return klass_identity_common(phase);
2245 }
2246 








































2247 Node* LoadNode::klass_identity_common(PhaseGVN* phase) {
2248   Node* x = LoadNode::Identity(phase);
2249   if (x != this)  return x;
2250 
2251   // Take apart the address into an oop and and offset.
2252   // Return 'this' if we cannot.
2253   Node*    adr    = in(MemNode::Address);
2254   intptr_t offset = 0;
2255   Node*    base   = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2256   if (base == NULL)     return this;
2257   const TypeOopPtr* toop = phase->type(adr)->isa_oopptr();
2258   if (toop == NULL)     return this;
2259 
2260   // Step over potential GC barrier for OopHandle resolve
2261   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
2262   if (bs->is_gc_barrier_node(base)) {
2263     base = bs->step_over_gc_barrier(base);
2264   }
2265 
2266   // We can fetch the klass directly through an AllocateNode.


2413 //=============================================================================
2414 //---------------------------StoreNode::make-----------------------------------
2415 // Polymorphic factory method:
2416 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo) {
2417   assert((mo == unordered || mo == release), "unexpected");
2418   Compile* C = gvn.C;
2419   assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2420          ctl != NULL, "raw memory operations should have control edge");
2421 
2422   switch (bt) {
2423   case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2424   case T_BYTE:    return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2425   case T_INT:     return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2426   case T_CHAR:
2427   case T_SHORT:   return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2428   case T_LONG:    return new StoreLNode(ctl, mem, adr, adr_type, val, mo);
2429   case T_FLOAT:   return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2430   case T_DOUBLE:  return new StoreDNode(ctl, mem, adr, adr_type, val, mo);
2431   case T_METADATA:
2432   case T_ADDRESS:

2433   case T_OBJECT:
2434 #ifdef _LP64
2435     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2436       val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2437       return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2438     } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2439                (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2440                 adr->bottom_type()->isa_rawptr())) {
2441       val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2442       return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2443     }
2444 #endif
2445     {
2446       return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2447     }
2448   default:
2449     ShouldNotReachHere();
2450     return (StoreNode*)NULL;
2451   }
2452 }


2473   //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
2474 
2475   // Since they are not commoned, do not hash them:
2476   return NO_HASH;
2477 }
2478 
2479 //------------------------------Ideal------------------------------------------
2480 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
2481 // When a store immediately follows a relevant allocation/initialization,
2482 // try to capture it into the initialization, or hoist it above.
2483 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2484   Node* p = MemNode::Ideal_common(phase, can_reshape);
2485   if (p)  return (p == NodeSentinel) ? NULL : p;
2486 
2487   Node* mem     = in(MemNode::Memory);
2488   Node* address = in(MemNode::Address);
2489   // Back-to-back stores to same address?  Fold em up.  Generally
2490   // unsafe if I have intervening uses...  Also disallowed for StoreCM
2491   // since they must follow each StoreP operation.  Redundant StoreCMs
2492   // are eliminated just before matching in final_graph_reshape.
2493   {
2494     Node* st = mem;
2495     // If Store 'st' has more than one use, we cannot fold 'st' away.
2496     // For example, 'st' might be the final state at a conditional
2497     // return.  Or, 'st' might be used by some node which is live at
2498     // the same time 'st' is live, which might be unschedulable.  So,
2499     // require exactly ONE user until such time as we clone 'mem' for
2500     // each of 'mem's uses (thus making the exactly-1-user-rule hold
2501     // true).
2502     while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
2503       // Looking at a dead closed cycle of memory?
2504       assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
2505       assert(Opcode() == st->Opcode() ||
2506              st->Opcode() == Op_StoreVector ||
2507              Opcode() == Op_StoreVector ||
2508              phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
2509              (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
2510              (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy

2511              (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
2512              "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
2513 
2514       if (st->in(MemNode::Address)->eqv_uncast(address) &&
2515           st->as_Store()->memory_size() <= this->memory_size()) {
2516         Node* use = st->raw_out(0);
2517         phase->igvn_rehash_node_delayed(use);
2518         if (can_reshape) {
2519           use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase->is_IterGVN());
2520         } else {
2521           // It's OK to do this in the parser, since DU info is always accurate,
2522           // and the parser always refers to nodes via SafePointNode maps.
2523           use->set_req(MemNode::Memory, st->in(MemNode::Memory));
2524         }
2525         return this;
2526       }
2527       st = st->in(MemNode::Memory);
2528     }
2529   }
2530 


2576   // Load then Store?  Then the Store is useless
2577   if (val->is_Load() &&
2578       val->in(MemNode::Address)->eqv_uncast(adr) &&
2579       val->in(MemNode::Memory )->eqv_uncast(mem) &&
2580       val->as_Load()->store_Opcode() == Opcode()) {
2581     result = mem;
2582   }
2583 
2584   // Two stores in a row of the same value?
2585   if (result == this &&
2586       mem->is_Store() &&
2587       mem->in(MemNode::Address)->eqv_uncast(adr) &&
2588       mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2589       mem->Opcode() == Opcode()) {
2590     result = mem;
2591   }
2592 
2593   // Store of zero anywhere into a freshly-allocated object?
2594   // Then the store is useless.
2595   // (It must already have been captured by the InitializeNode.)
2596   if (result == this &&
2597       ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
2598     // a newly allocated object is already all-zeroes everywhere
2599     if (mem->is_Proj() && mem->in(0)->is_Allocate()) {


2600       result = mem;
2601     }
2602 
2603     if (result == this) {
2604       // the store may also apply to zero-bits in an earlier object
2605       Node* prev_mem = find_previous_store(phase);
2606       // Steps (a), (b):  Walk past independent stores to find an exact match.
2607       if (prev_mem != NULL) {
2608         Node* prev_val = can_see_stored_value(prev_mem, phase);
2609         if (prev_val != NULL && phase->eqv(prev_val, val)) {
2610           // prev_val and val might differ by a cast; it would be good
2611           // to keep the more informative of the two.
2612           result = mem;








2613         }
2614       }
2615     }
2616   }
2617 
2618   if (result != this && phase->is_IterGVN() != NULL) {
2619     MemBarNode* trailing = trailing_membar();
2620     if (trailing != NULL) {
2621 #ifdef ASSERT
2622       const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
2623       assert(t_oop == NULL || t_oop->is_known_instance_field(), "only for non escaping objects");
2624 #endif
2625       PhaseIterGVN* igvn = phase->is_IterGVN();
2626       trailing->remove(igvn);
2627     }
2628   }
2629 
2630   return result;
2631 }
2632 


2901 // Clearing a short array is faster with stores
2902 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2903   // Already know this is a large node, do not try to ideal it
2904   if (!IdealizeClearArrayNode || _is_large) return NULL;
2905 
2906   const int unit = BytesPerLong;
2907   const TypeX* t = phase->type(in(2))->isa_intptr_t();
2908   if (!t)  return NULL;
2909   if (!t->is_con())  return NULL;
2910   intptr_t raw_count = t->get_con();
2911   intptr_t size = raw_count;
2912   if (!Matcher::init_array_count_is_in_bytes) size *= unit;
2913   // Clearing nothing uses the Identity call.
2914   // Negative clears are possible on dead ClearArrays
2915   // (see jck test stmt114.stmt11402.val).
2916   if (size <= 0 || size % unit != 0)  return NULL;
2917   intptr_t count = size / unit;
2918   // Length too long; communicate this to matchers and assemblers.
2919   // Assemblers are responsible to produce fast hardware clears for it.
2920   if (size > InitArrayShortSize) {
2921     return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
2922   }
2923   Node *mem = in(1);
2924   if( phase->type(mem)==Type::TOP ) return NULL;
2925   Node *adr = in(3);
2926   const Type* at = phase->type(adr);
2927   if( at==Type::TOP ) return NULL;
2928   const TypePtr* atp = at->isa_ptr();
2929   // adjust atp to be the correct array element address type
2930   if (atp == NULL)  atp = TypePtr::BOTTOM;
2931   else              atp = atp->add_offset(Type::OffsetBot);
2932   // Get base for derived pointer purposes
2933   if( adr->Opcode() != Op_AddP ) Unimplemented();
2934   Node *base = adr->in(1);
2935 
2936   Node *zero = phase->makecon(TypeLong::ZERO);
2937   Node *off  = phase->MakeConX(BytesPerLong);
2938   mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
2939   count--;
2940   while( count-- ) {
2941     mem = phase->transform(mem);
2942     adr = phase->transform(new AddPNode(base,adr,off));
2943     mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
2944   }
2945   return mem;
2946 }
2947 
2948 //----------------------------step_through----------------------------------
2949 // Return allocation input memory edge if it is different instance
2950 // or itself if it is the one we are looking for.
2951 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseTransform* phase) {
2952   Node* n = *np;
2953   assert(n->is_ClearArray(), "sanity");
2954   intptr_t offset;
2955   AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
2956   // This method is called only before Allocate nodes are expanded
2957   // during macro nodes expansion. Before that ClearArray nodes are
2958   // only generated in PhaseMacroExpand::generate_arraycopy() (before
2959   // Allocate nodes are expanded) which follows allocations.
2960   assert(alloc != NULL, "should have allocation");
2961   if (alloc->_idx == instance_id) {
2962     // Can not bypass initialization of the instance we are looking for.
2963     return false;
2964   }
2965   // Otherwise skip it.
2966   InitializeNode* init = alloc->initialization();
2967   if (init != NULL)
2968     *np = init->in(TypeFunc::Memory);
2969   else
2970     *np = alloc->in(TypeFunc::Memory);
2971   return true;
2972 }
2973 
2974 //----------------------------clear_memory-------------------------------------
2975 // Generate code to initialize object storage to zero.
2976 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,


2977                                    intptr_t start_offset,
2978                                    Node* end_offset,
2979                                    PhaseGVN* phase) {
2980   intptr_t offset = start_offset;
2981 
2982   int unit = BytesPerLong;
2983   if ((offset % unit) != 0) {
2984     Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
2985     adr = phase->transform(adr);
2986     const TypePtr* atp = TypeRawPtr::BOTTOM;
2987     mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);






2988     mem = phase->transform(mem);
2989     offset += BytesPerInt;
2990   }
2991   assert((offset % unit) == 0, "");
2992 
2993   // Initialize the remaining stuff, if any, with a ClearArray.
2994   return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, phase);
2995 }
2996 
2997 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,

2998                                    Node* start_offset,
2999                                    Node* end_offset,
3000                                    PhaseGVN* phase) {
3001   if (start_offset == end_offset) {
3002     // nothing to do
3003     return mem;
3004   }
3005 
3006   int unit = BytesPerLong;
3007   Node* zbase = start_offset;
3008   Node* zend  = end_offset;
3009 
3010   // Scale to the unit required by the CPU:
3011   if (!Matcher::init_array_count_is_in_bytes) {
3012     Node* shift = phase->intcon(exact_log2(unit));
3013     zbase = phase->transform(new URShiftXNode(zbase, shift) );
3014     zend  = phase->transform(new URShiftXNode(zend,  shift) );
3015   }
3016 
3017   // Bulk clear double-words
3018   Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3019   Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3020   mem = new ClearArrayNode(ctl, mem, zsize, adr, false);



3021   return phase->transform(mem);
3022 }
3023 
3024 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,


3025                                    intptr_t start_offset,
3026                                    intptr_t end_offset,
3027                                    PhaseGVN* phase) {
3028   if (start_offset == end_offset) {
3029     // nothing to do
3030     return mem;
3031   }
3032 
3033   assert((end_offset % BytesPerInt) == 0, "odd end offset");
3034   intptr_t done_offset = end_offset;
3035   if ((done_offset % BytesPerLong) != 0) {
3036     done_offset -= BytesPerInt;
3037   }
3038   if (done_offset > start_offset) {
3039     mem = clear_memory(ctl, mem, dest,
3040                        start_offset, phase->MakeConX(done_offset), phase);
3041   }
3042   if (done_offset < end_offset) { // emit the final 32-bit store
3043     Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3044     adr = phase->transform(adr);
3045     const TypePtr* atp = TypeRawPtr::BOTTOM;
3046     mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);






3047     mem = phase->transform(mem);
3048     done_offset += BytesPerInt;
3049   }
3050   assert(done_offset == end_offset, "");
3051   return mem;
3052 }
3053 
3054 //=============================================================================
3055 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3056   : MultiNode(TypeFunc::Parms + (precedent == NULL? 0: 1)),
3057     _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3058 #ifdef ASSERT
3059   , _pair_idx(0)
3060 #endif
3061 {
3062   init_class_id(Class_MemBar);
3063   Node* top = C->top();
3064   init_req(TypeFunc::I_O,top);
3065   init_req(TypeFunc::FramePtr,top);
3066   init_req(TypeFunc::ReturnAdr,top);


3165       PhaseIterGVN* igvn = phase->is_IterGVN();
3166       remove(igvn);
3167       // Must return either the original node (now dead) or a new node
3168       // (Do not return a top here, since that would break the uniqueness of top.)
3169       return new ConINode(TypeInt::ZERO);
3170     }
3171   }
3172   return progress ? this : NULL;
3173 }
3174 
3175 //------------------------------Value------------------------------------------
3176 const Type* MemBarNode::Value(PhaseGVN* phase) const {
3177   if( !in(0) ) return Type::TOP;
3178   if( phase->type(in(0)) == Type::TOP )
3179     return Type::TOP;
3180   return TypeTuple::MEMBAR;
3181 }
3182 
3183 //------------------------------match------------------------------------------
3184 // Construct projections for memory.
3185 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
3186   switch (proj->_con) {
3187   case TypeFunc::Control:
3188   case TypeFunc::Memory:
3189     return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3190   }
3191   ShouldNotReachHere();
3192   return NULL;
3193 }
3194 
3195 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3196   trailing->_kind = TrailingStore;
3197   leading->_kind = LeadingStore;
3198 #ifdef ASSERT
3199   trailing->_pair_idx = leading->_idx;
3200   leading->_pair_idx = leading->_idx;
3201 #endif
3202 }
3203 
3204 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3205   trailing->_kind = TrailingLoadStore;


3451   return (req() > RawStores);
3452 }
3453 
3454 void InitializeNode::set_complete(PhaseGVN* phase) {
3455   assert(!is_complete(), "caller responsibility");
3456   _is_complete = Complete;
3457 
3458   // After this node is complete, it contains a bunch of
3459   // raw-memory initializations.  There is no need for
3460   // it to have anything to do with non-raw memory effects.
3461   // Therefore, tell all non-raw users to re-optimize themselves,
3462   // after skipping the memory effects of this initialization.
3463   PhaseIterGVN* igvn = phase->is_IterGVN();
3464   if (igvn)  igvn->add_users_to_worklist(this);
3465 }
3466 
3467 // convenience function
3468 // return false if the init contains any stores already
3469 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
3470   InitializeNode* init = initialization();
3471   if (init == NULL || init->is_complete())  return false;


3472   init->remove_extra_zeroes();
3473   // for now, if this allocation has already collected any inits, bail:
3474   if (init->is_non_zero())  return false;
3475   init->set_complete(phase);
3476   return true;
3477 }
3478 
3479 void InitializeNode::remove_extra_zeroes() {
3480   if (req() == RawStores)  return;
3481   Node* zmem = zero_memory();
3482   uint fill = RawStores;
3483   for (uint i = fill; i < req(); i++) {
3484     Node* n = in(i);
3485     if (n->is_top() || n == zmem)  continue;  // skip
3486     if (fill < i)  set_req(fill, n);          // compact
3487     ++fill;
3488   }
3489   // delete any empty spaces created:
3490   while (fill < req()) {
3491     del_req(fill);


4195         //   z's_done      12  16  16  16    12  16    12
4196         //   z's_needed    12  16  16  16    16  16    16
4197         //   zsize          0   0   0   0     4   0     4
4198         if (next_full_store < 0) {
4199           // Conservative tack:  Zero to end of current word.
4200           zeroes_needed = align_up(zeroes_needed, BytesPerInt);
4201         } else {
4202           // Zero to beginning of next fully initialized word.
4203           // Or, don't zero at all, if we are already in that word.
4204           assert(next_full_store >= zeroes_needed, "must go forward");
4205           assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
4206           zeroes_needed = next_full_store;
4207         }
4208       }
4209 
4210       if (zeroes_needed > zeroes_done) {
4211         intptr_t zsize = zeroes_needed - zeroes_done;
4212         // Do some incremental zeroing on rawmem, in parallel with inits.
4213         zeroes_done = align_down(zeroes_done, BytesPerInt);
4214         rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,


4215                                               zeroes_done, zeroes_needed,
4216                                               phase);
4217         zeroes_done = zeroes_needed;
4218         if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
4219           do_zeroing = false;   // leave the hole, next time
4220       }
4221     }
4222 
4223     // Collect the store and move on:
4224     st->set_req(MemNode::Memory, inits);
4225     inits = st;                 // put it on the linearized chain
4226     set_req(i, zmem);           // unhook from previous position
4227 
4228     if (zeroes_done == st_off)
4229       zeroes_done = next_init_off;
4230 
4231     assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
4232 
4233     #ifdef ASSERT
4234     // Various order invariants.  Weaker than stores_are_sane because


4254   remove_extra_zeroes();        // clear out all the zmems left over
4255   add_req(inits);
4256 
4257   if (!(UseTLAB && ZeroTLAB)) {
4258     // If anything remains to be zeroed, zero it all now.
4259     zeroes_done = align_down(zeroes_done, BytesPerInt);
4260     // if it is the last unused 4 bytes of an instance, forget about it
4261     intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
4262     if (zeroes_done + BytesPerLong >= size_limit) {
4263       AllocateNode* alloc = allocation();
4264       assert(alloc != NULL, "must be present");
4265       if (alloc != NULL && alloc->Opcode() == Op_Allocate) {
4266         Node* klass_node = alloc->in(AllocateNode::KlassNode);
4267         ciKlass* k = phase->type(klass_node)->is_klassptr()->klass();
4268         if (zeroes_done == k->layout_helper())
4269           zeroes_done = size_limit;
4270       }
4271     }
4272     if (zeroes_done < size_limit) {
4273       rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,


4274                                             zeroes_done, size_in_bytes, phase);
4275     }
4276   }
4277 
4278   set_complete(phase);
4279   return rawmem;
4280 }
4281 
4282 
4283 #ifdef ASSERT
4284 bool InitializeNode::stores_are_sane(PhaseTransform* phase) {
4285   if (is_complete())
4286     return true;                // stores could be anything at this point
4287   assert(allocation() != NULL, "must be present");
4288   intptr_t last_off = allocation()->minimum_header_size();
4289   for (uint i = InitializeNode::RawStores; i < req(); i++) {
4290     Node* st = in(i);
4291     intptr_t st_off = get_store_offset(st, phase);
4292     if (st_off < 0)  continue;  // ignore dead garbage
4293     if (last_off > st_off) {




  28 #include "gc/shared/barrierSet.hpp"
  29 #include "gc/shared/c2/barrierSetC2.hpp"
  30 #include "memory/allocation.inline.hpp"
  31 #include "memory/resourceArea.hpp"
  32 #include "oops/objArrayKlass.hpp"
  33 #include "opto/addnode.hpp"
  34 #include "opto/arraycopynode.hpp"
  35 #include "opto/cfgnode.hpp"
  36 #include "opto/compile.hpp"
  37 #include "opto/connode.hpp"
  38 #include "opto/convertnode.hpp"
  39 #include "opto/loopnode.hpp"
  40 #include "opto/machnode.hpp"
  41 #include "opto/matcher.hpp"
  42 #include "opto/memnode.hpp"
  43 #include "opto/mulnode.hpp"
  44 #include "opto/narrowptrnode.hpp"
  45 #include "opto/phaseX.hpp"
  46 #include "opto/regmask.hpp"
  47 #include "opto/rootnode.hpp"
  48 #include "opto/valuetypenode.hpp"
  49 #include "utilities/align.hpp"
  50 #include "utilities/copy.hpp"
  51 #include "utilities/macros.hpp"
  52 #include "utilities/vmError.hpp"
  53 #if INCLUDE_ZGC
  54 #include "gc/z/c2/zBarrierSetC2.hpp"
  55 #endif
  56 
  57 // Portions of code courtesy of Clifford Click
  58 
  59 // Optimization - Graph Style
  60 
  61 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem,  const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
  62 
  63 //=============================================================================
  64 uint MemNode::size_of() const { return sizeof(*this); }
  65 
  66 const TypePtr *MemNode::adr_type() const {
  67   Node* adr = in(Address);
  68   if (adr == NULL)  return NULL; // node is dead


 223       // clone the Phi with our address type
 224       result = mphi->split_out_instance(t_adr, igvn);
 225     } else {
 226       assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
 227     }
 228   }
 229   return result;
 230 }
 231 
 232 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem,  const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
 233   uint alias_idx = phase->C->get_alias_index(tp);
 234   Node *mem = mmem;
 235 #ifdef ASSERT
 236   {
 237     // Check that current type is consistent with the alias index used during graph construction
 238     assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
 239     bool consistent =  adr_check == NULL || adr_check->empty() ||
 240                        phase->C->must_alias(adr_check, alias_idx );
 241     // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
 242     if( !consistent && adr_check != NULL && !adr_check->empty() &&
 243         tp->isa_aryptr() &&        tp->offset() == Type::OffsetBot &&
 244         adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
 245         ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
 246           adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
 247           adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
 248       // don't assert if it is dead code.
 249       consistent = true;
 250     }
 251     if( !consistent ) {
 252       st->print("alias_idx==%d, adr_check==", alias_idx);
 253       if( adr_check == NULL ) {
 254         st->print("NULL");
 255       } else {
 256         adr_check->dump();
 257       }
 258       st->cr();
 259       print_alias_types();
 260       assert(consistent, "adr_check must match alias idx");
 261     }
 262   }
 263 #endif


 821          "use LoadKlassNode instead");
 822   assert(!(adr_type->isa_aryptr() &&
 823            adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
 824          "use LoadRangeNode instead");
 825   // Check control edge of raw loads
 826   assert( ctl != NULL || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
 827           // oop will be recorded in oop map if load crosses safepoint
 828           rt->isa_oopptr() || is_immutable_value(adr),
 829           "raw memory operations should have control edge");
 830   LoadNode* load = NULL;
 831   switch (bt) {
 832   case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 833   case T_BYTE:    load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 834   case T_INT:     load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 835   case T_CHAR:    load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 836   case T_SHORT:   load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 837   case T_LONG:    load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency); break;
 838   case T_FLOAT:   load = new LoadFNode (ctl, mem, adr, adr_type, rt,            mo, control_dependency); break;
 839   case T_DOUBLE:  load = new LoadDNode (ctl, mem, adr, adr_type, rt,            mo, control_dependency); break;
 840   case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(),  mo, control_dependency); break;
 841   case T_VALUETYPE:
 842   case T_OBJECT:
 843 #ifdef _LP64
 844     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
 845       load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
 846     } else
 847 #endif
 848     {
 849       assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
 850       load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
 851     }
 852     break;
 853   default:
 854     ShouldNotReachHere();
 855     break;
 856   }
 857   assert(load != NULL, "LoadNode should have been created");
 858   if (unaligned) {
 859     load->set_unaligned_access();
 860   }
 861   if (mismatched) {


1077         // the same pointer-and-offset that we stored to.
1078         // Casted version may carry a dependency and it is respected.
1079         // Thus, we are able to replace L by V.
1080       }
1081       // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1082       if (store_Opcode() != st->Opcode())
1083         return NULL;
1084       return st->in(MemNode::ValueIn);
1085     }
1086 
1087     // A load from a freshly-created object always returns zero.
1088     // (This can happen after LoadNode::Ideal resets the load's memory input
1089     // to find_captured_store, which returned InitializeNode::zero_memory.)
1090     if (st->is_Proj() && st->in(0)->is_Allocate() &&
1091         (st->in(0) == ld_alloc) &&
1092         (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1093       // return a zero value for the load's basic type
1094       // (This is one of the few places where a generic PhaseTransform
1095       // can create new nodes.  Think of it as lazily manifesting
1096       // virtually pre-existing constants.)
1097       assert(memory_type() != T_VALUETYPE, "should not be used for value types");
1098       Node* default_value = ld_alloc->in(AllocateNode::DefaultValue);
1099       if (default_value != NULL) {
1100         return default_value;
1101       }
1102       assert(ld_alloc->in(AllocateNode::RawDefaultValue) == NULL, "default value may not be null");
1103       return phase->zerocon(memory_type());
1104     }
1105 
1106     // A load from an initialization barrier can match a captured store.
1107     if (st->is_Proj() && st->in(0)->is_Initialize()) {
1108       InitializeNode* init = st->in(0)->as_Initialize();
1109       AllocateNode* alloc = init->allocation();
1110       if ((alloc != NULL) && (alloc == ld_alloc)) {
1111         // examine a captured store value
1112         st = init->find_captured_store(ld_off, memory_size(), phase);
1113         if (st != NULL) {
1114           continue;             // take one more trip around
1115         }
1116       }
1117     }
1118 
1119     // Load boxed value from result of valueOf() call is input parameter.
1120     if (this->is_Load() && ld_adr->is_AddP() &&
1121         (tp != NULL) && tp->is_ptr_to_boxed_value()) {
1122       intptr_t ignore = 0;


1140 //----------------------is_instance_field_load_with_local_phi------------------
1141 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1142   if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1143       in(Address)->is_AddP() ) {
1144     const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1145     // Only instances and boxed values.
1146     if( t_oop != NULL &&
1147         (t_oop->is_ptr_to_boxed_value() ||
1148          t_oop->is_known_instance_field()) &&
1149         t_oop->offset() != Type::OffsetBot &&
1150         t_oop->offset() != Type::OffsetTop) {
1151       return true;
1152     }
1153   }
1154   return false;
1155 }
1156 
1157 //------------------------------Identity---------------------------------------
1158 // Loads are identity if previous store is to same address
1159 Node* LoadNode::Identity(PhaseGVN* phase) {
1160   // Loading from a ValueTypePtr? The ValueTypePtr has the values of
1161   // all fields as input. Look for the field with matching offset.
1162   Node* addr = in(Address);
1163   intptr_t offset;
1164   Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
1165   if (base != NULL && base->is_ValueTypePtr() && offset > oopDesc::klass_offset_in_bytes()) {
1166     Node* value = base->as_ValueTypePtr()->field_value_by_offset((int)offset, true);
1167     if (value->is_ValueType()) {
1168       // Non-flattened value type field
1169       ValueTypeNode* vt = value->as_ValueType();
1170       if (vt->is_allocated(phase)) {
1171         value = vt->get_oop();
1172       } else {
1173         // Not yet allocated, bail out
1174         value = NULL;
1175       }
1176     }
1177     if (value != NULL) {
1178       if (Opcode() == Op_LoadN) {
1179         // Encode oop value if we are loading a narrow oop
1180         assert(!phase->type(value)->isa_narrowoop(), "should already be decoded");
1181         value = phase->transform(new EncodePNode(value, bottom_type()));
1182       }
1183       return value;
1184     }
1185   }
1186 
1187   // If the previous store-maker is the right kind of Store, and the store is
1188   // to the same address, then we are equal to the value stored.
1189   Node* mem = in(Memory);
1190   Node* value = can_see_stored_value(mem, phase);
1191   if( value ) {
1192     // byte, short & char stores truncate naturally.
1193     // A load has to load the truncated value which requires
1194     // some sort of masking operation and that requires an
1195     // Ideal call instead of an Identity call.
1196     if (memory_size() < BytesPerInt) {
1197       // If the input to the store does not fit with the load's result type,
1198       // it must be truncated via an Ideal call.
1199       if (!phase->type(value)->higher_equal(phase->type(this)))
1200         return this;
1201     }
1202     // (This works even when value is a Con, but LoadNode::Value
1203     // usually runs first, producing the singleton type of the Con.)
1204     return value;
1205   }
1206 


1702   // fold up, do so.
1703   Node* prev_mem = find_previous_store(phase);
1704   if (prev_mem != NULL) {
1705     Node* value = can_see_arraycopy_value(prev_mem, phase);
1706     if (value != NULL) {
1707       return value;
1708     }
1709   }
1710   // Steps (a), (b):  Walk past independent stores to find an exact match.
1711   if (prev_mem != NULL && prev_mem != in(MemNode::Memory)) {
1712     // (c) See if we can fold up on the spot, but don't fold up here.
1713     // Fold-up might require truncation (for LoadB/LoadS/LoadUS) or
1714     // just return a prior value, which is done by Identity calls.
1715     if (can_see_stored_value(prev_mem, phase)) {
1716       // Make ready for step (d):
1717       set_req(MemNode::Memory, prev_mem);
1718       return this;
1719     }
1720   }
1721 
1722   AllocateNode* alloc = AllocateNode::Ideal_allocation(address, phase);
1723   if (alloc != NULL && mem->is_Proj() &&
1724       mem->in(0) != NULL &&
1725       mem->in(0) == alloc->initialization() &&
1726       Opcode() == Op_LoadX &&
1727       alloc->initialization()->proj_out_or_null(0) != NULL) {
1728     InitializeNode* init = alloc->initialization();
1729     Node* control = init->proj_out(0);
1730     return alloc->make_ideal_mark(phase, address, control, mem, NULL);
1731   }
1732 
1733   return progress ? this : NULL;
1734 }
1735 
1736 // Helper to recognize certain Klass fields which are invariant across
1737 // some group of array types (e.g., int[] or all T[] where T < Object).
1738 const Type*
1739 LoadNode::load_array_final_field(const TypeKlassPtr *tkls,
1740                                  ciKlass* klass) const {
1741   if (tkls->offset() == in_bytes(Klass::modifier_flags_offset())) {
1742     // The field is Klass::_modifier_flags.  Return its (constant) value.
1743     // (Folds up the 2nd indirection in aClassConstant.getModifiers().)
1744     assert(this->Opcode() == Op_LoadI, "must load an int from _modifier_flags");
1745     return TypeInt::make(klass->modifier_flags());
1746   }
1747   if (tkls->offset() == in_bytes(Klass::access_flags_offset())) {
1748     // The field is Klass::_access_flags.  Return its (constant) value.
1749     // (Folds up the 2nd indirection in Reflection.getClassAccessFlags(aClassConstant).)
1750     assert(this->Opcode() == Op_LoadI, "must load an int from _access_flags");
1751     return TypeInt::make(klass->access_flags());
1752   }


1802       }
1803     }
1804 
1805     // Don't do this for integer types. There is only potential profit if
1806     // the element type t is lower than _type; that is, for int types, if _type is
1807     // more restrictive than t.  This only happens here if one is short and the other
1808     // char (both 16 bits), and in those cases we've made an intentional decision
1809     // to use one kind of load over the other. See AndINode::Ideal and 4965907.
1810     // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
1811     //
1812     // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
1813     // where the _gvn.type of the AddP is wider than 8.  This occurs when an earlier
1814     // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
1815     // subsumed by p1.  If p1 is on the worklist but has not yet been re-transformed,
1816     // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
1817     // In fact, that could have been the original type of p1, and p1 could have
1818     // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
1819     // expression (LShiftL quux 3) independently optimized to the constant 8.
1820     if ((t->isa_int() == NULL) && (t->isa_long() == NULL)
1821         && (_type->isa_vect() == NULL)
1822         && t->isa_valuetype() == NULL
1823         && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
1824       // t might actually be lower than _type, if _type is a unique
1825       // concrete subclass of abstract class t.
1826       if (off_beyond_header || off == Type::OffsetBot) {  // is the offset beyond the header?
1827         const Type* jt = t->join_speculative(_type);
1828         // In any case, do not allow the join, per se, to empty out the type.
1829         if (jt->empty() && !t->empty()) {
1830           // This can happen if a interface-typed array narrows to a class type.
1831           jt = _type;
1832         }
1833 #ifdef ASSERT
1834         if (phase->C->eliminate_boxing() && adr->is_AddP()) {
1835           // The pointers in the autobox arrays are always non-null
1836           Node* base = adr->in(AddPNode::Base);
1837           if ((base != NULL) && base->is_DecodeN()) {
1838             // Get LoadN node which loads IntegerCache.cache field
1839             base = base->in(1);
1840           }
1841           if ((base != NULL) && base->is_Con()) {
1842             const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
1843             if ((base_type != NULL) && base_type->is_autobox_cache()) {
1844               // It could be narrow oop
1845               assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
1846             }
1847           }
1848         }
1849 #endif
1850         return jt;
1851       }
1852     }
1853   } else if (tp->base() == Type::InstPtr) {
1854     assert( off != Type::OffsetBot ||
1855             // arrays can be cast to Objects
1856             tp->is_oopptr()->klass()->is_java_lang_Object() ||
1857             tp->is_oopptr()->klass() == ciEnv::current()->Class_klass() ||
1858             // unsafe field access may not have a constant offset
1859             C->has_unsafe_access(),
1860             "Field accesses must be precise" );
1861     // For oop loads, we expect the _type to be precise.
1862 

1863     const TypeInstPtr* tinst = tp->is_instptr();
1864     BasicType bt = memory_type();
1865 
1866     // Fold component and value mirror loads
1867     ciInstanceKlass* ik = tinst->klass()->as_instance_klass();
1868     if (ik == phase->C->env()->Class_klass() && (off == java_lang_Class::component_mirror_offset_in_bytes() ||
1869                                                  off == java_lang_Class::inline_mirror_offset_in_bytes())) {
1870       ciType* mirror_type = tinst->java_mirror_type();
1871       if (mirror_type != NULL) {
1872         const Type* const_oop = TypePtr::NULL_PTR;
1873         if (mirror_type->is_array_klass()) {
1874           const_oop = TypeInstPtr::make(mirror_type->as_array_klass()->component_mirror_instance());
1875         } else if (mirror_type->is_valuetype()) {
1876           const_oop = TypeInstPtr::make(mirror_type->as_value_klass()->inline_mirror_instance());
1877         }
1878         return (bt == T_NARROWOOP) ? const_oop->make_narrowoop() : const_oop;
1879       }
1880     }
1881 
1882     // Optimize loads from constant fields.
1883     ciObject* const_oop = tinst->const_oop();
1884     if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != NULL && const_oop->is_instance()) {
1885       ciType* mirror_type = const_oop->as_instance()->java_mirror_type();
1886       if (mirror_type != NULL && mirror_type->is_valuetype()) {
1887         ciValueKlass* vk = mirror_type->as_value_klass();
1888         if (off == vk->default_value_offset()) {
1889           // Loading a special hidden field that contains the oop of the default value type
1890           const Type* const_oop = TypeInstPtr::make(vk->default_value_instance());
1891           return (bt == T_NARROWOOP) ? const_oop->make_narrowoop() : const_oop;
1892         }
1893       }
1894       const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
1895       if (con_type != NULL) {
1896         return con_type;
1897       }
1898     }
1899   } else if (tp->base() == Type::KlassPtr) {
1900     assert( off != Type::OffsetBot ||
1901             // arrays can be cast to Objects
1902             tp->is_klassptr()->klass() == NULL ||
1903             tp->is_klassptr()->klass()->is_java_lang_Object() ||
1904             // also allow array-loading from the primary supertype
1905             // array during subtype checks
1906             Opcode() == Op_LoadKlass,
1907             "Field accesses must be precise" );
1908     // For klass/static loads, we expect the _type to be precise
1909   } else if (tp->base() == Type::RawPtr && !StressReflectiveCode) {
1910     if (adr->is_Load() && off == 0) {
1911       /* With mirrors being an indirect in the Klass*
1912        * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
1913        * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
1914        *
1915        * So check the type and klass of the node before the LoadP.
1916        */
1917       Node* adr2 = adr->in(MemNode::Address);
1918       const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
1919       if (tkls != NULL) {
1920         ciKlass* klass = tkls->klass();
1921         if (klass != NULL && klass->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
1922           assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
1923           assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
1924           return TypeInstPtr::make(klass->java_mirror());
1925         }
1926       }
1927     } else {
1928       // Check for a load of the default value offset from the ValueKlassFixedBlock:
1929       // LoadI(LoadP(value_klass, adr_valueklass_fixed_block_offset), default_value_offset_offset)
1930       intptr_t offset = 0;
1931       Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
1932       if (base != NULL && base->is_Load() && offset == in_bytes(ValueKlass::default_value_offset_offset())) {
1933         const TypeKlassPtr* tkls = phase->type(base->in(MemNode::Address))->isa_klassptr();
1934         if (tkls != NULL && tkls->is_loaded() && tkls->klass_is_exact() && tkls->isa_valuetype() &&
1935             tkls->offset() == in_bytes(InstanceKlass::adr_valueklass_fixed_block_offset())) {
1936           assert(base->Opcode() == Op_LoadP, "must load an oop from klass");
1937           assert(Opcode() == Op_LoadI, "must load an int from fixed block");
1938           return TypeInt::make(tkls->klass()->as_value_klass()->default_value_offset());
1939         }
1940       }
1941     }
1942   }
1943 
1944   const TypeKlassPtr *tkls = tp->isa_klassptr();
1945   if (tkls != NULL && !StressReflectiveCode) {
1946     ciKlass* klass = tkls->klass();
1947     if (tkls->is_loaded() && tkls->klass_is_exact()) {
1948       // We are loading a field from a Klass metaobject whose identity
1949       // is known at compile time (the type is "exact" or "precise").
1950       // Check for fields we know are maintained as constants by the VM.
1951       if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
1952         // The field is Klass::_super_check_offset.  Return its (constant) value.
1953         // (Folds up type checking code.)
1954         assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
1955         return TypeInt::make(klass->super_check_offset());
1956       }
1957       // Compute index into primary_supers array
1958       juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
1959       // Check for overflowing; use unsigned compare to handle the negative case.
1960       if( depth < ciKlass::primary_super_limit() ) {
1961         // The field is an element of Klass::_primary_supers.  Return its (constant) value.
1962         // (Folds up type checking code.)
1963         assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
1964         ciKlass *ss = klass->super_of_depth(depth);
1965         return ss ? TypeKlassPtr::make(ss) : TypePtr::NULL_PTR;
1966       }
1967       const Type* aift = load_array_final_field(tkls, klass);
1968       if (aift != NULL)  return aift;
1969     }
1970 
1971     // We can still check if we are loading from the primary_supers array at a
1972     // shallow enough depth.  Even though the klass is not exact, entries less
1973     // than or equal to its super depth are correct.
1974     if (tkls->is_loaded()) {
1975       ciType *inner = klass;
1976       while( inner->is_obj_array_klass() )
1977         inner = inner->as_obj_array_klass()->base_element_type();
1978       if( inner->is_instance_klass() &&
1979           !inner->as_instance_klass()->flags().is_interface() ) {
1980         // Compute index into primary_supers array
1981         juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
1982         // Check for overflowing; use unsigned compare to handle the negative case.
1983         if( depth < ciKlass::primary_super_limit() &&
1984             depth <= klass->super_depth() ) { // allow self-depth checks to handle self-check case
1985           // The field is an element of Klass::_primary_supers.  Return its (constant) value.
1986           // (Folds up type checking code.)
1987           assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
1988           ciKlass *ss = klass->super_of_depth(depth);
1989           return ss ? TypeKlassPtr::make(ss) : TypePtr::NULL_PTR;
1990         }
1991       }
1992     }
1993 
1994     // If the type is enough to determine that the thing is not an array,


2200   // Either input is TOP ==> the result is TOP
2201   const Type *t1 = phase->type( in(MemNode::Memory) );
2202   if (t1 == Type::TOP)  return Type::TOP;
2203   Node *adr = in(MemNode::Address);
2204   const Type *t2 = phase->type( adr );
2205   if (t2 == Type::TOP)  return Type::TOP;
2206   const TypePtr *tp = t2->is_ptr();
2207   if (TypePtr::above_centerline(tp->ptr()) ||
2208       tp->ptr() == TypePtr::Null)  return Type::TOP;
2209 
2210   // Return a more precise klass, if possible
2211   const TypeInstPtr *tinst = tp->isa_instptr();
2212   if (tinst != NULL) {
2213     ciInstanceKlass* ik = tinst->klass()->as_instance_klass();
2214     int offset = tinst->offset();
2215     if (ik == phase->C->env()->Class_klass()
2216         && (offset == java_lang_Class::klass_offset_in_bytes() ||
2217             offset == java_lang_Class::array_klass_offset_in_bytes())) {
2218       // We are loading a special hidden field from a Class mirror object,
2219       // the field which points to the VM's Klass metaobject.
2220       bool is_indirect_type = true;
2221       ciType* t = tinst->java_mirror_type(&is_indirect_type);
2222       // java_mirror_type returns non-null for compile-time Class constants.
2223       if (t != NULL) {
2224         // constant oop => constant klass
2225         if (offset == java_lang_Class::array_klass_offset_in_bytes()) {
2226           if (t->is_void()) {
2227             // We cannot create a void array.  Since void is a primitive type return null
2228             // klass.  Users of this result need to do a null check on the returned klass.
2229             return TypePtr::NULL_PTR;
2230           }
2231           return TypeKlassPtr::make(ciArrayKlass::make(t, /* never_null */ !is_indirect_type));
2232         }
2233         if (!t->is_klass()) {
2234           // a primitive Class (e.g., int.class) has NULL for a klass field
2235           return TypePtr::NULL_PTR;
2236         }
2237         // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2238         return TypeKlassPtr::make(t->as_klass());
2239       }
2240       // non-constant mirror, so we can't tell what's going on
2241     }
2242     if( !ik->is_loaded() )
2243       return _type;             // Bail out if not loaded
2244     if (offset == oopDesc::klass_offset_in_bytes()) {
2245       if (tinst->klass_is_exact()) {
2246         return TypeKlassPtr::make(ik);
2247       }
2248       // See if we can become precise: no subklasses and no interface
2249       // (Note:  We need to support verified interfaces.)
2250       if (!ik->is_interface() && !ik->has_subklass()) {
2251         //assert(!UseExactTypes, "this code should be useless with exact types");
2252         // Add a dependence; if any subclass added we need to recompile
2253         if (!ik->is_final()) {
2254           // %%% should use stronger assert_unique_concrete_subtype instead
2255           phase->C->dependencies()->assert_leaf_type(ik);
2256         }
2257         // Return precise klass
2258         return TypeKlassPtr::make(ik);
2259       }
2260 
2261       // Return root of possible klass
2262       return TypeKlassPtr::make(TypePtr::NotNull, ik, Type::Offset(0));
2263     }
2264   }
2265 
2266   // Check for loading klass from an array
2267   const TypeAryPtr *tary = tp->isa_aryptr();
2268   if (tary != NULL) {
2269     ciKlass *tary_klass = tary->klass();
2270     if (tary_klass != NULL   // can be NULL when at BOTTOM or TOP
2271         && tary->offset() == oopDesc::klass_offset_in_bytes()) {
2272       ciArrayKlass* ak = tary_klass->as_array_klass();
2273       // Do not fold klass loads from [V?. The runtime type might be [V due to [V <: [V?
2274       // and the klass for [V is not equal to the klass for [V?.
2275       if (tary->klass_is_exact()) {
2276         return TypeKlassPtr::make(tary_klass);
2277       }
2278 
2279       // If the klass is an object array, we defer the question to the
2280       // array component klass.
2281       if (ak->is_obj_array_klass()) {
2282         assert(ak->is_loaded(), "");
2283         ciKlass *base_k = ak->as_obj_array_klass()->base_element_klass();
2284         if (base_k->is_loaded() && base_k->is_instance_klass()) {
2285           ciInstanceKlass *ik = base_k->as_instance_klass();
2286           // See if we can become precise: no subklasses and no interface
2287           if (!ik->is_interface() && !ik->has_subklass() && (!ik->is_valuetype() || ak->storage_properties().is_null_free())) {
2288             //assert(!UseExactTypes, "this code should be useless with exact types");
2289             // Add a dependence; if any subclass added we need to recompile
2290             if (!ik->is_final()) {
2291               phase->C->dependencies()->assert_leaf_type(ik);
2292             }
2293             // Return precise array klass
2294             return TypeKlassPtr::make(ak);
2295           }
2296         }
2297         return TypeKlassPtr::make(TypePtr::NotNull, ak, Type::Offset(0));
2298       } else if (ak->is_type_array_klass()) {
2299         //assert(!UseExactTypes, "this code should be useless with exact types");

2300         return TypeKlassPtr::make(ak); // These are always precise
2301       }
2302     }
2303   }
2304 
2305   // Check for loading klass from an array klass
2306   const TypeKlassPtr *tkls = tp->isa_klassptr();
2307   if (tkls != NULL && !StressReflectiveCode) {
2308     if (!tkls->is_loaded()) {

2309       return _type;             // Bail out if not loaded
2310     }
2311     ciKlass* klass = tkls->klass();
2312     if( klass->is_obj_array_klass() &&
2313         tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2314       ciKlass* elem = klass->as_obj_array_klass()->element_klass();
2315       // // Always returning precise element type is incorrect,
2316       // // e.g., element type could be object and array may contain strings
2317       // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2318 
2319       // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2320       // according to the element type's subclassing.
2321       return TypeKlassPtr::make(tkls->ptr(), elem, Type::Offset(0));
2322     } else if (klass->is_value_array_klass() &&
2323                tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2324       ciKlass* elem = klass->as_value_array_klass()->element_klass();
2325       return TypeKlassPtr::make(tkls->ptr(), elem, Type::Offset(0));
2326     }
2327     if( klass->is_instance_klass() && tkls->klass_is_exact() &&
2328         tkls->offset() == in_bytes(Klass::super_offset())) {
2329       ciKlass* sup = klass->as_instance_klass()->super();
2330       // The field is Klass::_super.  Return its (constant) value.
2331       // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2332       return sup ? TypeKlassPtr::make(sup) : TypePtr::NULL_PTR;
2333     }
2334   }
2335 
2336   // Bailout case
2337   return LoadNode::Value(phase);
2338 }
2339 
2340 //------------------------------Identity---------------------------------------
2341 // To clean up reflective code, simplify k.java_mirror.as_klass to plain k.
2342 // Also feed through the klass in Allocate(...klass...)._klass.
2343 Node* LoadKlassNode::Identity(PhaseGVN* phase) {
2344   return klass_identity_common(phase);
2345 }
2346 
2347 const Type* GetNullFreePropertyNode::Value(PhaseGVN* phase) const {
2348   if (in(1) != NULL) {
2349     const Type* in1_t = phase->type(in(1));
2350     if (in1_t == Type::TOP) {
2351       return Type::TOP;
2352     }
2353     const TypeKlassPtr* tk = in1_t->make_ptr()->is_klassptr();
2354     ciArrayKlass* ak = tk->klass()->as_array_klass();
2355     ciKlass* elem = ak->element_klass();
2356     if (tk->klass_is_exact() || (!elem->is_java_lang_Object() && !elem->is_interface() && !elem->is_valuetype())) {
2357       int props_shift = in1_t->isa_narrowklass() ? oopDesc::narrow_storage_props_shift : oopDesc::wide_storage_props_shift;
2358       ArrayStorageProperties props = ak->storage_properties();
2359       intptr_t storage_properties = props.encode<intptr_t>(props_shift);
2360       if (in1_t->isa_narrowklass()) {
2361         return TypeInt::make((int)storage_properties);
2362       }
2363       return TypeX::make(storage_properties);
2364     }
2365   }
2366   return bottom_type();
2367 }
2368 
2369 Node* GetNullFreePropertyNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2370   if (!can_reshape) {
2371     return NULL;
2372   }
2373   if (in(1) != NULL && in(1)->is_Phi()) {
2374     Node* phi = in(1);
2375     Node* r = phi->in(0);
2376     Node* new_phi = new PhiNode(r, bottom_type());
2377     for (uint i = 1; i < r->req(); i++) {
2378       Node* in = phi->in(i);
2379       if (in == NULL) continue;
2380       new_phi->init_req(i, phase->transform(new GetNullFreePropertyNode(in)));
2381     }
2382     return new_phi;
2383   }
2384   return NULL;
2385 }
2386 
2387 Node* LoadNode::klass_identity_common(PhaseGVN* phase) {
2388   Node* x = LoadNode::Identity(phase);
2389   if (x != this)  return x;
2390 
2391   // Take apart the address into an oop and and offset.
2392   // Return 'this' if we cannot.
2393   Node*    adr    = in(MemNode::Address);
2394   intptr_t offset = 0;
2395   Node*    base   = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2396   if (base == NULL)     return this;
2397   const TypeOopPtr* toop = phase->type(adr)->isa_oopptr();
2398   if (toop == NULL)     return this;
2399 
2400   // Step over potential GC barrier for OopHandle resolve
2401   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
2402   if (bs->is_gc_barrier_node(base)) {
2403     base = bs->step_over_gc_barrier(base);
2404   }
2405 
2406   // We can fetch the klass directly through an AllocateNode.


2553 //=============================================================================
2554 //---------------------------StoreNode::make-----------------------------------
2555 // Polymorphic factory method:
2556 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo) {
2557   assert((mo == unordered || mo == release), "unexpected");
2558   Compile* C = gvn.C;
2559   assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2560          ctl != NULL, "raw memory operations should have control edge");
2561 
2562   switch (bt) {
2563   case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2564   case T_BYTE:    return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2565   case T_INT:     return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2566   case T_CHAR:
2567   case T_SHORT:   return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2568   case T_LONG:    return new StoreLNode(ctl, mem, adr, adr_type, val, mo);
2569   case T_FLOAT:   return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2570   case T_DOUBLE:  return new StoreDNode(ctl, mem, adr, adr_type, val, mo);
2571   case T_METADATA:
2572   case T_ADDRESS:
2573   case T_VALUETYPE:
2574   case T_OBJECT:
2575 #ifdef _LP64
2576     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2577       val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2578       return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2579     } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2580                (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2581                 adr->bottom_type()->isa_rawptr())) {
2582       val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2583       return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2584     }
2585 #endif
2586     {
2587       return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2588     }
2589   default:
2590     ShouldNotReachHere();
2591     return (StoreNode*)NULL;
2592   }
2593 }


2614   //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
2615 
2616   // Since they are not commoned, do not hash them:
2617   return NO_HASH;
2618 }
2619 
2620 //------------------------------Ideal------------------------------------------
2621 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
2622 // When a store immediately follows a relevant allocation/initialization,
2623 // try to capture it into the initialization, or hoist it above.
2624 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2625   Node* p = MemNode::Ideal_common(phase, can_reshape);
2626   if (p)  return (p == NodeSentinel) ? NULL : p;
2627 
2628   Node* mem     = in(MemNode::Memory);
2629   Node* address = in(MemNode::Address);
2630   // Back-to-back stores to same address?  Fold em up.  Generally
2631   // unsafe if I have intervening uses...  Also disallowed for StoreCM
2632   // since they must follow each StoreP operation.  Redundant StoreCMs
2633   // are eliminated just before matching in final_graph_reshape.
2634   if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::VALUES) {
2635     Node* st = mem;
2636     // If Store 'st' has more than one use, we cannot fold 'st' away.
2637     // For example, 'st' might be the final state at a conditional
2638     // return.  Or, 'st' might be used by some node which is live at
2639     // the same time 'st' is live, which might be unschedulable.  So,
2640     // require exactly ONE user until such time as we clone 'mem' for
2641     // each of 'mem's uses (thus making the exactly-1-user-rule hold
2642     // true).
2643     while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
2644       // Looking at a dead closed cycle of memory?
2645       assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
2646       assert(Opcode() == st->Opcode() ||
2647              st->Opcode() == Op_StoreVector ||
2648              Opcode() == Op_StoreVector ||
2649              phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
2650              (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
2651              (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
2652              (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
2653              (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
2654              "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
2655 
2656       if (st->in(MemNode::Address)->eqv_uncast(address) &&
2657           st->as_Store()->memory_size() <= this->memory_size()) {
2658         Node* use = st->raw_out(0);
2659         phase->igvn_rehash_node_delayed(use);
2660         if (can_reshape) {
2661           use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase->is_IterGVN());
2662         } else {
2663           // It's OK to do this in the parser, since DU info is always accurate,
2664           // and the parser always refers to nodes via SafePointNode maps.
2665           use->set_req(MemNode::Memory, st->in(MemNode::Memory));
2666         }
2667         return this;
2668       }
2669       st = st->in(MemNode::Memory);
2670     }
2671   }
2672 


2718   // Load then Store?  Then the Store is useless
2719   if (val->is_Load() &&
2720       val->in(MemNode::Address)->eqv_uncast(adr) &&
2721       val->in(MemNode::Memory )->eqv_uncast(mem) &&
2722       val->as_Load()->store_Opcode() == Opcode()) {
2723     result = mem;
2724   }
2725 
2726   // Two stores in a row of the same value?
2727   if (result == this &&
2728       mem->is_Store() &&
2729       mem->in(MemNode::Address)->eqv_uncast(adr) &&
2730       mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2731       mem->Opcode() == Opcode()) {
2732     result = mem;
2733   }
2734 
2735   // Store of zero anywhere into a freshly-allocated object?
2736   // Then the store is useless.
2737   // (It must already have been captured by the InitializeNode.)
2738   if (result == this && ReduceFieldZeroing) {

2739     // a newly allocated object is already all-zeroes everywhere
2740     if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
2741         (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::DefaultValue) == val)) {
2742       assert(!phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::DefaultValue) == NULL, "storing null to value array is forbidden");
2743       result = mem;
2744     }
2745 
2746     if (result == this) {
2747       // the store may also apply to zero-bits in an earlier object
2748       Node* prev_mem = find_previous_store(phase);
2749       // Steps (a), (b):  Walk past independent stores to find an exact match.
2750       if (prev_mem != NULL) {
2751         Node* prev_val = can_see_stored_value(prev_mem, phase);
2752         if (prev_val != NULL && phase->eqv(prev_val, val)) {
2753           // prev_val and val might differ by a cast; it would be good
2754           // to keep the more informative of the two.
2755           if (phase->type(val)->is_zero_type()) {
2756             result = mem;
2757           } else if (prev_mem->is_Proj() && prev_mem->in(0)->is_Initialize()) {
2758             InitializeNode* init = prev_mem->in(0)->as_Initialize();
2759             AllocateNode* alloc = init->allocation();
2760             if (alloc != NULL && alloc->in(AllocateNode::DefaultValue) == val) {
2761               result = mem;
2762             }
2763           }
2764         }
2765       }
2766     }
2767   }
2768 
2769   if (result != this && phase->is_IterGVN() != NULL) {
2770     MemBarNode* trailing = trailing_membar();
2771     if (trailing != NULL) {
2772 #ifdef ASSERT
2773       const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
2774       assert(t_oop == NULL || t_oop->is_known_instance_field(), "only for non escaping objects");
2775 #endif
2776       PhaseIterGVN* igvn = phase->is_IterGVN();
2777       trailing->remove(igvn);
2778     }
2779   }
2780 
2781   return result;
2782 }
2783 


3052 // Clearing a short array is faster with stores
3053 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3054   // Already know this is a large node, do not try to ideal it
3055   if (!IdealizeClearArrayNode || _is_large) return NULL;
3056 
3057   const int unit = BytesPerLong;
3058   const TypeX* t = phase->type(in(2))->isa_intptr_t();
3059   if (!t)  return NULL;
3060   if (!t->is_con())  return NULL;
3061   intptr_t raw_count = t->get_con();
3062   intptr_t size = raw_count;
3063   if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3064   // Clearing nothing uses the Identity call.
3065   // Negative clears are possible on dead ClearArrays
3066   // (see jck test stmt114.stmt11402.val).
3067   if (size <= 0 || size % unit != 0)  return NULL;
3068   intptr_t count = size / unit;
3069   // Length too long; communicate this to matchers and assemblers.
3070   // Assemblers are responsible to produce fast hardware clears for it.
3071   if (size > InitArrayShortSize) {
3072     return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
3073   }
3074   Node *mem = in(1);
3075   if( phase->type(mem)==Type::TOP ) return NULL;
3076   Node *adr = in(3);
3077   const Type* at = phase->type(adr);
3078   if( at==Type::TOP ) return NULL;
3079   const TypePtr* atp = at->isa_ptr();
3080   // adjust atp to be the correct array element address type
3081   if (atp == NULL)  atp = TypePtr::BOTTOM;
3082   else              atp = atp->add_offset(Type::OffsetBot);
3083   // Get base for derived pointer purposes
3084   if( adr->Opcode() != Op_AddP ) Unimplemented();
3085   Node *base = adr->in(1);
3086 
3087   Node *val = in(4);
3088   Node *off  = phase->MakeConX(BytesPerLong);
3089   mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3090   count--;
3091   while( count-- ) {
3092     mem = phase->transform(mem);
3093     adr = phase->transform(new AddPNode(base,adr,off));
3094     mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3095   }
3096   return mem;
3097 }
3098 
3099 //----------------------------step_through----------------------------------
3100 // Return allocation input memory edge if it is different instance
3101 // or itself if it is the one we are looking for.
3102 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseTransform* phase) {
3103   Node* n = *np;
3104   assert(n->is_ClearArray(), "sanity");
3105   intptr_t offset;
3106   AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3107   // This method is called only before Allocate nodes are expanded
3108   // during macro nodes expansion. Before that ClearArray nodes are
3109   // only generated in PhaseMacroExpand::generate_arraycopy() (before
3110   // Allocate nodes are expanded) which follows allocations.
3111   assert(alloc != NULL, "should have allocation");
3112   if (alloc->_idx == instance_id) {
3113     // Can not bypass initialization of the instance we are looking for.
3114     return false;
3115   }
3116   // Otherwise skip it.
3117   InitializeNode* init = alloc->initialization();
3118   if (init != NULL)
3119     *np = init->in(TypeFunc::Memory);
3120   else
3121     *np = alloc->in(TypeFunc::Memory);
3122   return true;
3123 }
3124 
3125 //----------------------------clear_memory-------------------------------------
3126 // Generate code to initialize object storage to zero.
3127 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3128                                    Node* val,
3129                                    Node* raw_val,
3130                                    intptr_t start_offset,
3131                                    Node* end_offset,
3132                                    PhaseGVN* phase) {
3133   intptr_t offset = start_offset;
3134 
3135   int unit = BytesPerLong;
3136   if ((offset % unit) != 0) {
3137     Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3138     adr = phase->transform(adr);
3139     const TypePtr* atp = TypeRawPtr::BOTTOM;
3140     if (val != NULL) {
3141       assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3142       mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3143     } else {
3144       assert(raw_val == NULL, "val may not be null");
3145       mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3146     }
3147     mem = phase->transform(mem);
3148     offset += BytesPerInt;
3149   }
3150   assert((offset % unit) == 0, "");
3151 
3152   // Initialize the remaining stuff, if any, with a ClearArray.
3153   return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, phase);
3154 }
3155 
3156 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3157                                    Node* raw_val,
3158                                    Node* start_offset,
3159                                    Node* end_offset,
3160                                    PhaseGVN* phase) {
3161   if (start_offset == end_offset) {
3162     // nothing to do
3163     return mem;
3164   }
3165 
3166   int unit = BytesPerLong;
3167   Node* zbase = start_offset;
3168   Node* zend  = end_offset;
3169 
3170   // Scale to the unit required by the CPU:
3171   if (!Matcher::init_array_count_is_in_bytes) {
3172     Node* shift = phase->intcon(exact_log2(unit));
3173     zbase = phase->transform(new URShiftXNode(zbase, shift) );
3174     zend  = phase->transform(new URShiftXNode(zend,  shift) );
3175   }
3176 
3177   // Bulk clear double-words
3178   Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3179   Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3180   if (raw_val == NULL) {
3181     raw_val = phase->MakeConX(0);
3182   }
3183   mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
3184   return phase->transform(mem);
3185 }
3186 
3187 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3188                                    Node* val,
3189                                    Node* raw_val,
3190                                    intptr_t start_offset,
3191                                    intptr_t end_offset,
3192                                    PhaseGVN* phase) {
3193   if (start_offset == end_offset) {
3194     // nothing to do
3195     return mem;
3196   }
3197 
3198   assert((end_offset % BytesPerInt) == 0, "odd end offset");
3199   intptr_t done_offset = end_offset;
3200   if ((done_offset % BytesPerLong) != 0) {
3201     done_offset -= BytesPerInt;
3202   }
3203   if (done_offset > start_offset) {
3204     mem = clear_memory(ctl, mem, dest, val, raw_val,
3205                        start_offset, phase->MakeConX(done_offset), phase);
3206   }
3207   if (done_offset < end_offset) { // emit the final 32-bit store
3208     Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3209     adr = phase->transform(adr);
3210     const TypePtr* atp = TypeRawPtr::BOTTOM;
3211     if (val != NULL) {
3212       assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3213       mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3214     } else {
3215       assert(raw_val == NULL, "val may not be null");
3216       mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3217     }
3218     mem = phase->transform(mem);
3219     done_offset += BytesPerInt;
3220   }
3221   assert(done_offset == end_offset, "");
3222   return mem;
3223 }
3224 
3225 //=============================================================================
3226 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3227   : MultiNode(TypeFunc::Parms + (precedent == NULL? 0: 1)),
3228     _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3229 #ifdef ASSERT
3230   , _pair_idx(0)
3231 #endif
3232 {
3233   init_class_id(Class_MemBar);
3234   Node* top = C->top();
3235   init_req(TypeFunc::I_O,top);
3236   init_req(TypeFunc::FramePtr,top);
3237   init_req(TypeFunc::ReturnAdr,top);


3336       PhaseIterGVN* igvn = phase->is_IterGVN();
3337       remove(igvn);
3338       // Must return either the original node (now dead) or a new node
3339       // (Do not return a top here, since that would break the uniqueness of top.)
3340       return new ConINode(TypeInt::ZERO);
3341     }
3342   }
3343   return progress ? this : NULL;
3344 }
3345 
3346 //------------------------------Value------------------------------------------
3347 const Type* MemBarNode::Value(PhaseGVN* phase) const {
3348   if( !in(0) ) return Type::TOP;
3349   if( phase->type(in(0)) == Type::TOP )
3350     return Type::TOP;
3351   return TypeTuple::MEMBAR;
3352 }
3353 
3354 //------------------------------match------------------------------------------
3355 // Construct projections for memory.
3356 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
3357   switch (proj->_con) {
3358   case TypeFunc::Control:
3359   case TypeFunc::Memory:
3360     return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3361   }
3362   ShouldNotReachHere();
3363   return NULL;
3364 }
3365 
3366 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3367   trailing->_kind = TrailingStore;
3368   leading->_kind = LeadingStore;
3369 #ifdef ASSERT
3370   trailing->_pair_idx = leading->_idx;
3371   leading->_pair_idx = leading->_idx;
3372 #endif
3373 }
3374 
3375 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3376   trailing->_kind = TrailingLoadStore;


3622   return (req() > RawStores);
3623 }
3624 
3625 void InitializeNode::set_complete(PhaseGVN* phase) {
3626   assert(!is_complete(), "caller responsibility");
3627   _is_complete = Complete;
3628 
3629   // After this node is complete, it contains a bunch of
3630   // raw-memory initializations.  There is no need for
3631   // it to have anything to do with non-raw memory effects.
3632   // Therefore, tell all non-raw users to re-optimize themselves,
3633   // after skipping the memory effects of this initialization.
3634   PhaseIterGVN* igvn = phase->is_IterGVN();
3635   if (igvn)  igvn->add_users_to_worklist(this);
3636 }
3637 
3638 // convenience function
3639 // return false if the init contains any stores already
3640 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
3641   InitializeNode* init = initialization();
3642   if (init == NULL || init->is_complete()) {
3643     return false;
3644   }
3645   init->remove_extra_zeroes();
3646   // for now, if this allocation has already collected any inits, bail:
3647   if (init->is_non_zero())  return false;
3648   init->set_complete(phase);
3649   return true;
3650 }
3651 
3652 void InitializeNode::remove_extra_zeroes() {
3653   if (req() == RawStores)  return;
3654   Node* zmem = zero_memory();
3655   uint fill = RawStores;
3656   for (uint i = fill; i < req(); i++) {
3657     Node* n = in(i);
3658     if (n->is_top() || n == zmem)  continue;  // skip
3659     if (fill < i)  set_req(fill, n);          // compact
3660     ++fill;
3661   }
3662   // delete any empty spaces created:
3663   while (fill < req()) {
3664     del_req(fill);


4368         //   z's_done      12  16  16  16    12  16    12
4369         //   z's_needed    12  16  16  16    16  16    16
4370         //   zsize          0   0   0   0     4   0     4
4371         if (next_full_store < 0) {
4372           // Conservative tack:  Zero to end of current word.
4373           zeroes_needed = align_up(zeroes_needed, BytesPerInt);
4374         } else {
4375           // Zero to beginning of next fully initialized word.
4376           // Or, don't zero at all, if we are already in that word.
4377           assert(next_full_store >= zeroes_needed, "must go forward");
4378           assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
4379           zeroes_needed = next_full_store;
4380         }
4381       }
4382 
4383       if (zeroes_needed > zeroes_done) {
4384         intptr_t zsize = zeroes_needed - zeroes_done;
4385         // Do some incremental zeroing on rawmem, in parallel with inits.
4386         zeroes_done = align_down(zeroes_done, BytesPerInt);
4387         rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4388                                               allocation()->in(AllocateNode::DefaultValue),
4389                                               allocation()->in(AllocateNode::RawDefaultValue),
4390                                               zeroes_done, zeroes_needed,
4391                                               phase);
4392         zeroes_done = zeroes_needed;
4393         if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
4394           do_zeroing = false;   // leave the hole, next time
4395       }
4396     }
4397 
4398     // Collect the store and move on:
4399     st->set_req(MemNode::Memory, inits);
4400     inits = st;                 // put it on the linearized chain
4401     set_req(i, zmem);           // unhook from previous position
4402 
4403     if (zeroes_done == st_off)
4404       zeroes_done = next_init_off;
4405 
4406     assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
4407 
4408     #ifdef ASSERT
4409     // Various order invariants.  Weaker than stores_are_sane because


4429   remove_extra_zeroes();        // clear out all the zmems left over
4430   add_req(inits);
4431 
4432   if (!(UseTLAB && ZeroTLAB)) {
4433     // If anything remains to be zeroed, zero it all now.
4434     zeroes_done = align_down(zeroes_done, BytesPerInt);
4435     // if it is the last unused 4 bytes of an instance, forget about it
4436     intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
4437     if (zeroes_done + BytesPerLong >= size_limit) {
4438       AllocateNode* alloc = allocation();
4439       assert(alloc != NULL, "must be present");
4440       if (alloc != NULL && alloc->Opcode() == Op_Allocate) {
4441         Node* klass_node = alloc->in(AllocateNode::KlassNode);
4442         ciKlass* k = phase->type(klass_node)->is_klassptr()->klass();
4443         if (zeroes_done == k->layout_helper())
4444           zeroes_done = size_limit;
4445       }
4446     }
4447     if (zeroes_done < size_limit) {
4448       rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4449                                             allocation()->in(AllocateNode::DefaultValue),
4450                                             allocation()->in(AllocateNode::RawDefaultValue),
4451                                             zeroes_done, size_in_bytes, phase);
4452     }
4453   }
4454 
4455   set_complete(phase);
4456   return rawmem;
4457 }
4458 
4459 
4460 #ifdef ASSERT
4461 bool InitializeNode::stores_are_sane(PhaseTransform* phase) {
4462   if (is_complete())
4463     return true;                // stores could be anything at this point
4464   assert(allocation() != NULL, "must be present");
4465   intptr_t last_off = allocation()->minimum_header_size();
4466   for (uint i = InitializeNode::RawStores; i < req(); i++) {
4467     Node* st = in(i);
4468     intptr_t st_off = get_store_offset(st, phase);
4469     if (st_off < 0)  continue;  // ignore dead garbage
4470     if (last_off > st_off) {


< prev index next >